Merge branch 'develop' into upgrade-flask

cli/onionshare_cli/censorship.py

@@ -22,6 +22,12 @@ import requests
 from .meek import MeekNotRunning
 
 
+class CensorshipCircumventionError(Exception):
+    """
+    There was a problem connecting to the Tor CensorshipCircumvention API.
+    """
+
+
 class CensorshipCircumvention(object):
     """
     Connect to the Tor Moat APIs to retrieve censorship
@@ -47,7 +53,7 @@ class CensorshipCircumvention(object):
                 self.common.log(
                     "CensorshipCircumvention",
                     "__init__",
-                    "Using Meek with CensorShipCircumvention API",
+                    "Using Meek with CensorshipCircumvention API",
                 )
                 self.api_proxies = self.meek.meek_proxies
         if onion:
@@ -58,7 +64,7 @@ class CensorshipCircumvention(object):
                 self.common.log(
                     "CensorshipCircumvention",
                     "__init__",
-                    "Using Tor with CensorShipCircumvention API",
+                    "Using Tor with CensorshipCircumvention API",
                 )
                 (socks_address, socks_port) = self.onion.get_tor_socks_port()
                 self.api_proxies = {
@@ -84,31 +90,34 @@ class CensorshipCircumvention(object):
         if country:
             data = {"country": country}
 
-        r = requests.post(
-            endpoint,
-            json=data,
-            headers={"Content-Type": "application/vnd.api+json"},
-            proxies=self.api_proxies,
-        )
-        if r.status_code != 200:
-            self.common.log(
-                "CensorshipCircumvention",
-                "censorship_obtain_map",
-                f"status_code={r.status_code}",
+        try:
+            r = requests.post(
+                endpoint,
+                json=data,
+                headers={"Content-Type": "application/vnd.api+json"},
+                proxies=self.api_proxies,
             )
-            return False
+            if r.status_code != 200:
+                self.common.log(
+                    "CensorshipCircumvention",
+                    "censorship_obtain_map",
+                    f"status_code={r.status_code}",
+                )
+                return False
 
-        result = r.json()
+            result = r.json()
 
-        if "errors" in result:
-            self.common.log(
-                "CensorshipCircumvention",
-                "censorship_obtain_map",
-                f"errors={result['errors']}",
-            )
-            return False
+            if "errors" in result:
+                self.common.log(
+                    "CensorshipCircumvention",
+                    "censorship_obtain_map",
+                    f"errors={result['errors']}",
+                )
+                return False
 
-        return result
+            return result
+        except requests.exceptions.RequestException as e:
+            raise CensorshipCircumventionError(e)
 
     def request_settings(self, country=False, transports=False):
         """
@@ -127,45 +136,53 @@ class CensorshipCircumvention(object):
         endpoint = "https://bridges.torproject.org/moat/circumvention/settings"
         data = {}
         if country:
+            self.common.log(
+                "CensorshipCircumvention",
+                "censorship_obtain_settings",
+                f"Trying to obtain bridges for country={country}",
+            )
             data = {"country": country}
         if transports:
             data.append({"transports": transports})
-        r = requests.post(
-            endpoint,
-            json=data,
-            headers={"Content-Type": "application/vnd.api+json"},
-            proxies=self.api_proxies,
-        )
-        if r.status_code != 200:
-            self.common.log(
-                "CensorshipCircumvention",
-                "censorship_obtain_settings",
-                f"status_code={r.status_code}",
+        try:
+            r = requests.post(
+                endpoint,
+                json=data,
+                headers={"Content-Type": "application/vnd.api+json"},
+                proxies=self.api_proxies,
             )
-            return False
+            if r.status_code != 200:
+                self.common.log(
+                    "CensorshipCircumvention",
+                    "censorship_obtain_settings",
+                    f"status_code={r.status_code}",
+                )
+                return False
 
-        result = r.json()
+            result = r.json()
 
-        if "errors" in result:
-            self.common.log(
-                "CensorshipCircumvention",
-                "censorship_obtain_settings",
-                f"errors={result['errors']}",
-            )
-            return False
+            if "errors" in result:
+                self.common.log(
+                    "CensorshipCircumvention",
+                    "censorship_obtain_settings",
+                    f"errors={result['errors']}",
+                )
+                return False
 
-        # There are no settings - perhaps this country doesn't require censorship circumvention?
-        # This is not really an error, so we can just check if False and assume direct Tor
-        # connection will work.
-        if not "settings" in result:
-            self.common.log(
-                "CensorshipCircumvention",
-                "censorship_obtain_settings",
-                "No settings found for this country",
-            )
-            return False
+            # There are no settings - perhaps this country doesn't require censorship circumvention?
+            # This is not really an error, so we can just check if False and assume direct Tor
+            # connection will work.
+            if not "settings" in result:
+                self.common.log(
+                    "CensorshipCircumvention",
+                    "censorship_obtain_settings",
+                    "No settings found for this country",
+                )
+                return False
 
-        return result
+            return result
+        except requests.exceptions.RequestException as e:
+            raise CensorshipCircumventionError(e)
 
     def request_builtin_bridges(self):
         """
@@ -174,27 +191,103 @@ class CensorshipCircumvention(object):
         if not self.api_proxies:
             return False
         endpoint = "https://bridges.torproject.org/moat/circumvention/builtin"
-        r = requests.post(
-            endpoint,
-            headers={"Content-Type": "application/vnd.api+json"},
-            proxies=self.api_proxies,
+        try:
+            r = requests.post(
+                endpoint,
+                headers={"Content-Type": "application/vnd.api+json"},
+                proxies=self.api_proxies,
+            )
+            if r.status_code != 200:
+                self.common.log(
+                    "CensorshipCircumvention",
+                    "censorship_obtain_builtin_bridges",
+                    f"status_code={r.status_code}",
+                )
+                return False
+
+            result = r.json()
+
+            if "errors" in result:
+                self.common.log(
+                    "CensorshipCircumvention",
+                    "censorship_obtain_builtin_bridges",
+                    f"errors={result['errors']}",
+                )
+                return False
+
+            return result
+        except requests.exceptions.RequestException as e:
+            raise CensorshipCircumventionError(e)
+
+    def save_settings(self, settings, bridge_settings):
+        """
+        Checks the bridges and saves them in settings.
+        """
+        bridges_ok = False
+        self.settings = settings
+
+        # @TODO there might be several bridge types recommended.
+        # Should we attempt to iterate over each type if one of them fails to connect?
+        # But if so, how to stop it starting 3 separate Tor connection threads?
+        # for bridges in request_bridges["settings"]:
+        bridges = bridge_settings["settings"][0]["bridges"]
+        self.common.log(
+            "CensorshipCircumvention",
+            "save_settings",
+            f"Obtained bridges: {bridges}",
         )
-        if r.status_code != 200:
+        bridge_strings = bridges["bridge_strings"]
+        bridge_type = bridges["type"]
+        bridge_source = bridges["source"]
+
+        # If the recommended bridge source is to use the built-in
+        # bridges, set that in our settings, as if the user had
+        # selected the built-in bridges for a specific PT themselves.
+        #
+        if bridge_source == "builtin":
             self.common.log(
                 "CensorshipCircumvention",
-                "censorship_obtain_builtin_bridges",
-                f"status_code={r.status_code}",
+                "save_settings",
+                "Will be using built-in bridges",
             )
-            return False
-
-        result = r.json()
-
-        if "errors" in result:
+            self.settings.set("bridges_type", "built-in")
+            if bridge_type == "obfs4":
+                self.settings.set("bridges_builtin_pt", "obfs4")
+            if bridge_type == "snowflake":
+                self.settings.set("bridges_builtin_pt", "snowflake")
+            if bridge_type == "meek":
+                self.settings.set("bridges_builtin_pt", "meek-azure")
+            bridges_ok = True
+        else:
             self.common.log(
                 "CensorshipCircumvention",
-                "censorship_obtain_builtin_bridges",
-                f"errors={result['errors']}",
+                "save_settings",
+                "Will be using custom bridges",
+            )
+            # Any other type of bridge we can treat as custom.
+            self.settings.set("bridges_type", "custom")
+
+            # Sanity check the bridges provided from the Tor API before saving
+            bridges_checked = self.common.check_bridges_valid(bridge_strings)
+
+            if bridges_checked:
+                self.settings.set("bridges_custom", "\n".join(bridges_checked))
+                bridges_ok = True
+
+        # If we got any good bridges, save them to settings and return.
+        if bridges_ok:
+            self.common.log(
+                "CensorshipCircumvention",
+                "save_settings",
+                "Saving settings with automatically-obtained bridges",
+            )
+            self.settings.set("bridges_enabled", True)
+            self.settings.save()
+            return True
+        else:
+            self.common.log(
+                "CensorshipCircumvention",
+                "save_settings",
+                "Could not use any of the obtained bridges.",
             )
             return False
-
-        return result

cli/onionshare_cli/common.py

@@ -28,6 +28,7 @@ import sys
 import threading
 import time
 import shutil
+import re
 from pkg_resources import resource_filename
 
 import colorama
@@ -312,7 +313,6 @@ class Common:
         """
         Returns the absolute path of a resource
         """
-        self.log("Common", "get_resource_path", f"filename={filename}")
         path = resource_filename("onionshare_cli", os.path.join("resources", filename))
         self.log("Common", "get_resource_path", f"filename={filename}, path={path}")
         return path
@@ -467,6 +467,40 @@ class Common:
         r = random.SystemRandom()
         return "-".join(r.choice(wordlist) for _ in range(word_count))
 
+    def check_bridges_valid(self, bridges):
+        """
+        Does a regex check against a supplied list of bridges, to make sure they
+        are valid strings depending on the bridge type.
+        """
+        valid_bridges = []
+        self.log("Common", "check_bridges_valid", "Checking bridge syntax")
+        for bridge in bridges:
+            if bridge != "":
+                # Check the syntax of the custom bridge to make sure it looks legitimate
+                ipv4_pattern = re.compile(
+                    "(obfs4\s+)?(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]):([0-9]+)(\s+)([A-Z0-9]+)(.+)$"
+                )
+                ipv6_pattern = re.compile(
+                    "(obfs4\s+)?\[(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\]:[0-9]+\s+[A-Z0-9]+(.+)$"
+                )
+                meek_lite_pattern = re.compile(
+                    "(meek_lite)(\s)+([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+)(\s)+([0-9A-Z]+)(\s)+url=(.+)(\s)+front=(.+)"
+                )
+                snowflake_pattern = re.compile(
+                    "(snowflake)(\s)+([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+)(\s)+([0-9A-Z]+)"
+                )
+                if (
+                    ipv4_pattern.match(bridge)
+                    or ipv6_pattern.match(bridge)
+                    or meek_lite_pattern.match(bridge)
+                    or snowflake_pattern.match(bridge)
+                ):
+                    valid_bridges.append(bridge)
+        if valid_bridges:
+            return valid_bridges
+        else:
+           return False
+
     def is_flatpak(self):
         """
         Returns True if OnionShare is running in a Flatpak sandbox
@@ -479,6 +513,7 @@ class Common:
         """
         return os.environ.get("SNAP_INSTANCE_NAME") == "onionshare"
 
+
     @staticmethod
     def random_string(num_bytes, output_len=None):
         """

cli/onionshare_cli/onion.py

@@ -954,20 +954,6 @@ class Onion(object):
                 "update_builtin_bridges",
                 f"Obtained bridges: {builtin_bridges}",
             )
-            if builtin_bridges["meek"]:
-                # Meek bridge needs to be defined as "meek_lite", not "meek",
-                # for it to work with obfs4proxy.
-                # We also refer to this bridge type as 'meek-azure' in our settings.
-                # So first, rename the key in the dict
-                builtin_bridges["meek-azure"] = builtin_bridges.pop("meek")
-                new_meek_bridges = []
-                # Now replace the values. They also need the url/front params appended
-                for item in builtin_bridges["meek-azure"]:
-                    newline = item.replace("meek", "meek_lite")
-                    new_meek_bridges.append(
-                        f"{newline} url=https://meek.azureedge.net/ front=ajax.aspnetcdn.com"
-                    )
-                builtin_bridges["meek-azure"] = new_meek_bridges
             # Save the new settings
             self.settings.set("bridges_builtin", builtin_bridges)
             self.settings.save()

cli/onionshare_cli/settings.py

@@ -103,6 +103,7 @@ class Settings(object):
             "socket_file_path": "/var/run/tor/control",
             "auth_type": "no_auth",
             "auth_password": "",
+            "auto_connect": False,
             "use_autoupdate": True,
             "autoupdate_timestamp": None,
             "bridges_enabled": False,

cli/onionshare_cli/web/send_base_mode.py

@@ -44,8 +44,9 @@ class SendBaseModeWeb:
         self.download_filesize = None
         self.zip_writer = None
 
-        # Store the tempfile objects here, so when they're garbage collected the files are deleted
-        self.gzip_files = []
+        # Create a temporary dir to store gzip files in
+        self.gzip_tmp_dir = tempfile.TemporaryDirectory(dir=self.common.build_tmp_dir())
+        self.gzip_counter = 0
 
         # If autostop_sharing, only allow one download at a time
         self.download_in_progress = False
@@ -193,15 +194,12 @@ class SendBaseModeWeb:
         # gzip compress the individual file, if it hasn't already been compressed
         if use_gzip:
             if filesystem_path not in self.gzip_individual_files:
-                self.gzip_files.append(
-                    tempfile.NamedTemporaryFile("wb+", dir=self.common.build_tmp_dir())
+                gzip_filename = os.path.join(
+                    self.gzip_tmp_dir.name, str(self.gzip_counter)
                 )
-                gzip_file = self.gzip_files[-1]
-                self._gzip_compress(filesystem_path, gzip_file.name, 6, None)
-                self.gzip_individual_files[filesystem_path] = gzip_file.name
-
-                # Cleanup this temp file
-                self.web.cleanup_tempfiles.append(gzip_file)
+                self.gzip_counter += 1
+                self._gzip_compress(filesystem_path, gzip_filename, 6, None)
+                self.gzip_individual_files[filesystem_path] = gzip_filename
 
             file_to_download = self.gzip_individual_files[filesystem_path]
             filesize = os.path.getsize(self.gzip_individual_files[filesystem_path])

cli/onionshare_cli/web/web.py

@@ -171,7 +171,6 @@ class Web:
             self.socketio.init_app(self.app)
             self.chat_mode = ChatModeWeb(self.common, self)
 
-        self.cleanup_tempfiles = []
         self.cleanup_tempdirs = []
 
     def get_mode(self):
@@ -405,13 +404,8 @@ class Web:
         """
         self.common.log("Web", "cleanup")
 
-        # Close all of the tempfile.NamedTemporaryFile
-        for file in self.cleanup_tempfiles:
-            file.close()
-
         # Clean up the tempfile.NamedTemporaryDirectory objects
         for dir in self.cleanup_tempdirs:
             dir.cleanup()
 
-        self.cleanup_tempfiles = []
         self.cleanup_tempdirs = []

cli/tests/test_cli_settings.py

@@ -37,6 +37,7 @@ class TestSettings:
             "bridges_builtin": {},
             "persistent_tabs": [],
             "theme": 0,
+            "auto_connect": False,
         }
         for key in settings_obj._settings:
             # Skip locale, it will not always default to the same thing

cli/tests/test_cli_web.py

@@ -50,7 +50,6 @@ def web_obj(temp_dir, common_obj, mode, num_files=0):
     web = Web(common_obj, False, mode_settings, mode)
     web.running = True
 
-    web.cleanup_tempfiles == []
     web.cleanup_tempdirs == []
     web.app.testing = True
 
@@ -308,17 +307,13 @@ class TestWeb:
     def test_cleanup(self, common_obj, temp_dir_1024):
         web = web_obj(temp_dir_1024, common_obj, "share", 3)
 
-        temp_file = tempfile.NamedTemporaryFile()
         temp_dir = tempfile.TemporaryDirectory()
 
-        web.cleanup_tempfiles = [temp_file]
         web.cleanup_tempdirs = [temp_dir]
         web.cleanup()
 
-        assert os.path.exists(temp_file.name) is False
         assert os.path.exists(temp_dir.name) is False
 
-        assert web.cleanup_tempfiles == []
         assert web.cleanup_tempdirs == []