From c9fa2308a7c9fb99b40d1c7b8c112c5b9f510d75 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Fri, 15 Oct 2021 14:24:27 +1100 Subject: [PATCH 01/10] Add early (non-domain-fronted!) methods for interacting with the planned Tor censorship circumvention moat endpoints. This is based on loose specs from https://gitlab.torproject.org/tpo/anti-censorship/bridgedb/-/issues/40025 --- cli/onionshare_cli/common.py | 69 ++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/cli/onionshare_cli/common.py b/cli/onionshare_cli/common.py index dd92eb0b..195de2fe 100644 --- a/cli/onionshare_cli/common.py +++ b/cli/onionshare_cli/common.py @@ -22,6 +22,7 @@ import hashlib import os import platform import random +import requests import socket import sys import threading @@ -504,6 +505,74 @@ class Common: total_size += os.path.getsize(fp) return total_size + def censorship_obtain_map(self): + """ + Retrieves the Circumvention map from Tor Project and store it + locally for further look-ups if required. + """ + endpoint = "https://bridges.torproject.org/moat/circumvention/map" + # @TODO this needs to be using domain fronting to defeat censorship + # of the lookup itself. + response = requests.get(endpoint) + self.censorship_map = response.json() + self.log("Common", "censorship_obtain_map", self.censorship_map) + + def censorship_obtain_settings_from_api(self): + """ + Retrieves the Circumvention Settings from Tor Project, which + will return recommended settings based on the country code of + the requesting IP. + """ + endpoint = "https://bridges.torproject.org/moat/circumvention/settings" + # @TODO this needs to be using domain fronting to defeat censorship + # of the lookup itself. + response = requests.get(endpoint) + self.censorship_settings = response.json() + self.log( + "Common", "censorship_obtain_settings_from_api", self.censorship_settings + ) + + def censorship_obtain_settings_from_map(self, country): + """ + Retrieves the Circumvention Settings for this country from the + circumvention map we have stored locally, rather than from the + API endpoint. + + This is for when the user has specified the country themselves + rather than requesting auto-detection. + """ + try: + # Fetch the map. + self.censorship_obtain_map() + self.censorship_settings = self.censorship_map[country] + self.log( + "Common", + "censorship_obtain_settings_from_map", + f"Settings are {self.censorship_settings}", + ) + except KeyError: + self.log( + "Common", + "censorship_obtain_settings_from_map", + "No censorship settings found for this country", + ) + return False + + def censorship_obtain_builtin_bridges(self): + """ + Retrieves the list of built-in bridges from the Tor Project. + """ + endpoint = "https://bridges.torproject.org/moat/circumvention/builtin" + # @TODO this needs to be using domain fronting to defeat censorship + # of the lookup itself. + response = requests.get(endpoint) + self.censorship_builtin_bridges = response.json() + self.log( + "Common", + "censorship_obtain_builtin_bridges", + self.censorship_builtin_bridges, + ) + class AutoStopTimer(threading.Thread): """ From 0989f2b133a46f293a65c9e11a01e8a097e479a1 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Mon, 18 Oct 2021 17:17:47 +1100 Subject: [PATCH 02/10] Move Censorship stuff into its own class. Early attempt at subprocessing out to meek (unfinished) --- cli/onionshare_cli/__init__.py | 17 +-- cli/onionshare_cli/censorship.py | 216 +++++++++++++++++++++++++++++++ cli/onionshare_cli/common.py | 73 +---------- cli/onionshare_cli/onion.py | 1 + 4 files changed, 226 insertions(+), 81 deletions(-) create mode 100644 cli/onionshare_cli/censorship.py diff --git a/cli/onionshare_cli/__init__.py b/cli/onionshare_cli/__init__.py index 4bc00929..ddba332e 100644 --- a/cli/onionshare_cli/__init__.py +++ b/cli/onionshare_cli/__init__.py @@ -27,13 +27,9 @@ from datetime import datetime from datetime import timedelta from .common import Common, CannotFindTor +from .censorship import CensorshipCircumvention from .web import Web -from .onion import ( - TorErrorProtocolError, - TorTooOldEphemeral, - TorTooOldStealth, - Onion, -) +from .onion import TorErrorProtocolError, TorTooOldEphemeral, TorTooOldStealth, Onion from .onionshare import OnionShare from .mode_settings import ModeSettings @@ -94,12 +90,7 @@ def main(cwd=None): help="Filename of persistent session", ) # General args - parser.add_argument( - "--title", - metavar="TITLE", - default=None, - help="Set a title", - ) + parser.add_argument("--title", metavar="TITLE", default=None, help="Set a title") parser.add_argument( "--public", action="store_true", @@ -409,7 +400,7 @@ def main(cwd=None): sys.exit(1) # Warn about sending large files over Tor - if web.share_mode.download_filesize >= 157286400: # 150mb + if web.share_mode.download_filesize >= 157_286_400: # 150mb print("") print("Warning: Sending a large share could take hours") print("") diff --git a/cli/onionshare_cli/censorship.py b/cli/onionshare_cli/censorship.py new file mode 100644 index 00000000..176f95e6 --- /dev/null +++ b/cli/onionshare_cli/censorship.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +""" +OnionShare | https://onionshare.org/ + +Copyright (C) 2014-2021 Micah Lee, et al. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" +import requests +import subprocess + + +class CensorshipCircumvention: + """ + The CensorShipCircumvention object contains methods to detect + and offer solutions to censorship when connecting to Tor. + """ + + def __init__(self, common): + + self.common = common + self.common.log("CensorshipCircumvention", "__init__") + + get_tor_paths = self.common.get_tor_paths + ( + self.tor_path, + self.tor_geo_ip_file_path, + self.tor_geo_ipv6_file_path, + self.obfs4proxy_file_path, + self.meek_client_file_path, + ) = get_tor_paths() + + meek_url = "https://moat.torproject.org.global.prod.fastly.net/" + meek_front = "cdn.sstatic.net" + meek_env = { + "TOR_PT_MANAGED_TRANSPORT_VER": "1", + "TOR_PT_CLIENT_TRANSPORTS": "meek", + } + + # @TODO detect the port from the subprocess output + meek_address = "127.0.0.1" + meek_port = "43533" # hardcoded for testing + self.meek_proxies = { + "http": f"socks5h://{meek_address}:{meek_port}", + "https": f"socks5h://{meek_address}:{meek_port}", + } + + # Start the Meek Client as a subprocess. + # This will be used to do domain fronting to the Tor + # Moat API endpoints for censorship circumvention as + # well as BridgeDB lookups. + + if self.common.platform == "Windows": + # In Windows, hide console window when opening tor.exe subprocess + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + self.meek_proc = subprocess.Popen( + [self.meek_client_file_path, "--url", meek_url, "--front", meek_front], + stdout=subprocess.PIPE, + startupinfo=startupinfo, + bufsize=1, + env=meek_env, + text=True, + ) + else: + self.meek_proc = subprocess.Popen( + [self.meek_client_file_path, "--url", meek_url, "--front", meek_front], + stdout=subprocess.PIPE, + bufsize=1, + env=meek_env, + text=True, + ) + + # if "CMETHOD meek socks5" in line: + # self.meek_host = (line.split(" ")[3].split(":")[0]) + # self.meek_port = (line.split(" ")[3].split(":")[1]) + # self.common.log("CensorshipCircumvention", "__init__", f"Meek host is {self.meek_host}") + # self.common.log("CensorshipCircumvention", "__init__", f"Meek port is {self.meek_port}") + + def censorship_obtain_map(self, country=False): + """ + Retrieves the Circumvention map from Tor Project and store it + locally for further look-ups if required. + + Optionally pass a country code in order to get recommended settings + just for that country. + + Note that this API endpoint doesn't return actual bridges, + it just returns the recommended bridge type countries. + """ + endpoint = "https://bridges.torproject.org/moat/circumvention/map" + data = {} + if country: + data = {"country": country} + + r = requests.post( + endpoint, + json=data, + headers={"Content-Type": "application/vnd.api+json"}, + proxies=self.meek_proxies, + ) + if r.status_code != 200: + self.common.log( + "CensorshipCircumvention", + "censorship_obtain_map", + f"status_code={r.status_code}", + ) + return False + + result = r.json() + + if "errors" in result: + self.common.log( + "CensorshipCircumvention", + "censorship_obtain_map", + f"errors={result['errors']}", + ) + return False + + return result + + def censorship_obtain_settings(self, country=False, transports=False): + """ + Retrieves the Circumvention Settings from Tor Project, which + will return recommended settings based on the country code of + the requesting IP. + + Optionally, a country code can be specified in order to override + the IP detection. + + Optionally, a list of transports can be specified in order to + return recommended settings for just that transport type. + """ + endpoint = "https://bridges.torproject.org/moat/circumvention/settings" + data = {} + if country: + data = {"country": country} + if transports: + data.append({"transports": transports}) + r = requests.post( + endpoint, + json=data, + headers={"Content-Type": "application/vnd.api+json"}, + proxies=self.meek_proxies, + ) + if r.status_code != 200: + self.common.log( + "CensorshipCircumvention", + "censorship_obtain_settings", + f"status_code={r.status_code}", + ) + return False + + result = r.json() + + if "errors" in result: + self.common.log( + "CensorshipCircumvention", + "censorship_obtain_settings", + f"errors={result['errors']}", + ) + return False + + # There are no settings - perhaps this country doesn't require censorship circumvention? + # This is not really an error, so we can just check if False and assume direct Tor + # connection will work. + if not "settings" in result: + self.common.log( + "CensorshipCircumvention", + "censorship_obtain_settings", + "No settings found for this country", + ) + return False + + return result + + def censorship_obtain_builtin_bridges(self): + """ + Retrieves the list of built-in bridges from the Tor Project. + """ + endpoint = "https://bridges.torproject.org/moat/circumvention/builtin" + r = requests.post( + endpoint, + headers={"Content-Type": "application/vnd.api+json"}, + proxies=self.meek_proxies, + ) + if r.status_code != 200: + self.common.log( + "CensorshipCircumvention", + "censorship_obtain_builtin_bridges", + f"status_code={r.status_code}", + ) + return False + + result = r.json() + + if "errors" in result: + self.common.log( + "CensorshipCircumvention", + "censorship_obtain_builtin_bridges", + f"errors={result['errors']}", + ) + return False + + return result diff --git a/cli/onionshare_cli/common.py b/cli/onionshare_cli/common.py index 195de2fe..549b1c21 100644 --- a/cli/onionshare_cli/common.py +++ b/cli/onionshare_cli/common.py @@ -314,6 +314,7 @@ class Common: if not tor_path: raise CannotFindTor() obfs4proxy_file_path = shutil.which("obfs4proxy") + meek_client_file_path = shutil.which("meek-client") prefix = os.path.dirname(os.path.dirname(tor_path)) tor_geo_ip_file_path = os.path.join(prefix, "share/tor/geoip") tor_geo_ipv6_file_path = os.path.join(prefix, "share/tor/geoip6") @@ -321,6 +322,7 @@ class Common: base_path = self.get_resource_path("tor") tor_path = os.path.join(base_path, "Tor", "tor.exe") obfs4proxy_file_path = os.path.join(base_path, "Tor", "obfs4proxy.exe") + meek_client_file_path = os.path.join(base_path, "Tor", "meek-client.exe") tor_geo_ip_file_path = os.path.join(base_path, "Data", "Tor", "geoip") tor_geo_ipv6_file_path = os.path.join(base_path, "Data", "Tor", "geoip6") elif self.platform == "Darwin": @@ -328,6 +330,7 @@ class Common: if not tor_path: raise CannotFindTor() obfs4proxy_file_path = shutil.which("obfs4proxy") + meek_client_file_path = shutil.which("meek-client") prefix = os.path.dirname(os.path.dirname(tor_path)) tor_geo_ip_file_path = os.path.join(prefix, "share/tor/geoip") tor_geo_ipv6_file_path = os.path.join(prefix, "share/tor/geoip6") @@ -336,12 +339,14 @@ class Common: tor_geo_ip_file_path = "/usr/local/share/tor/geoip" tor_geo_ipv6_file_path = "/usr/local/share/tor/geoip6" obfs4proxy_file_path = "/usr/local/bin/obfs4proxy" + meek_client_file_path = "/usr/local/bin/meek-client" return ( tor_path, tor_geo_ip_file_path, tor_geo_ipv6_file_path, obfs4proxy_file_path, + meek_client_file_path, ) def build_data_dir(self): @@ -505,74 +510,6 @@ class Common: total_size += os.path.getsize(fp) return total_size - def censorship_obtain_map(self): - """ - Retrieves the Circumvention map from Tor Project and store it - locally for further look-ups if required. - """ - endpoint = "https://bridges.torproject.org/moat/circumvention/map" - # @TODO this needs to be using domain fronting to defeat censorship - # of the lookup itself. - response = requests.get(endpoint) - self.censorship_map = response.json() - self.log("Common", "censorship_obtain_map", self.censorship_map) - - def censorship_obtain_settings_from_api(self): - """ - Retrieves the Circumvention Settings from Tor Project, which - will return recommended settings based on the country code of - the requesting IP. - """ - endpoint = "https://bridges.torproject.org/moat/circumvention/settings" - # @TODO this needs to be using domain fronting to defeat censorship - # of the lookup itself. - response = requests.get(endpoint) - self.censorship_settings = response.json() - self.log( - "Common", "censorship_obtain_settings_from_api", self.censorship_settings - ) - - def censorship_obtain_settings_from_map(self, country): - """ - Retrieves the Circumvention Settings for this country from the - circumvention map we have stored locally, rather than from the - API endpoint. - - This is for when the user has specified the country themselves - rather than requesting auto-detection. - """ - try: - # Fetch the map. - self.censorship_obtain_map() - self.censorship_settings = self.censorship_map[country] - self.log( - "Common", - "censorship_obtain_settings_from_map", - f"Settings are {self.censorship_settings}", - ) - except KeyError: - self.log( - "Common", - "censorship_obtain_settings_from_map", - "No censorship settings found for this country", - ) - return False - - def censorship_obtain_builtin_bridges(self): - """ - Retrieves the list of built-in bridges from the Tor Project. - """ - endpoint = "https://bridges.torproject.org/moat/circumvention/builtin" - # @TODO this needs to be using domain fronting to defeat censorship - # of the lookup itself. - response = requests.get(endpoint) - self.censorship_builtin_bridges = response.json() - self.log( - "Common", - "censorship_obtain_builtin_bridges", - self.censorship_builtin_bridges, - ) - class AutoStopTimer(threading.Thread): """ diff --git a/cli/onionshare_cli/onion.py b/cli/onionshare_cli/onion.py index 7f6faa17..aa5e276b 100644 --- a/cli/onionshare_cli/onion.py +++ b/cli/onionshare_cli/onion.py @@ -153,6 +153,7 @@ class Onion(object): self.tor_geo_ip_file_path, self.tor_geo_ipv6_file_path, self.obfs4proxy_file_path, + self.meek_client_file_path, ) = get_tor_paths() # The tor process From 5b4d77c3634c0c13ae8ab1f27be540260027d0a8 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Tue, 19 Oct 2021 11:36:03 +1100 Subject: [PATCH 03/10] Refactor to CensorshipCircumvention and Meek classes. Use Meek domain fronting when requesting bridges in frontend --- cli/onionshare_cli/__init__.py | 13 ++ cli/onionshare_cli/censorship.py | 87 +++-------- cli/onionshare_cli/meek.py | 144 ++++++++++++++++++ desktop/src/onionshare/gui_common.py | 6 + desktop/src/onionshare/moat_dialog.py | 16 +- desktop/src/onionshare/tor_settings_dialog.py | 15 +- 6 files changed, 202 insertions(+), 79 deletions(-) create mode 100644 cli/onionshare_cli/meek.py diff --git a/cli/onionshare_cli/__init__.py b/cli/onionshare_cli/__init__.py index ddba332e..99992b25 100644 --- a/cli/onionshare_cli/__init__.py +++ b/cli/onionshare_cli/__init__.py @@ -28,6 +28,7 @@ from datetime import timedelta from .common import Common, CannotFindTor from .censorship import CensorshipCircumvention +from .meek import Meek, MeekNotRunning from .web import Web from .onion import TorErrorProtocolError, TorTooOldEphemeral, TorTooOldStealth, Onion from .onionshare import OnionShare @@ -284,6 +285,18 @@ def main(cwd=None): # Create the Web object web = Web(common, False, mode_settings, mode) + # Create the Meek object and start the meek client + meek = Meek(common) + meek.start() + + # Create the CensorshipCircumvention object to make + # API calls to Tor over Meek + censorship = CensorshipCircumvention(common, meek) + # Example: request recommended bridges, pretending to be from China, using + # domain fronting. + # censorship_recommended_settings = censorship.request_settings(country="cn") + # print(censorship_recommended_settings) + # Start the Onion object try: onion = Onion(common, use_tmp_dir=True) diff --git a/cli/onionshare_cli/censorship.py b/cli/onionshare_cli/censorship.py index 176f95e6..f84b1058 100644 --- a/cli/onionshare_cli/censorship.py +++ b/cli/onionshare_cli/censorship.py @@ -18,77 +18,30 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ import requests -import subprocess + +from .meek import MeekNotRunning -class CensorshipCircumvention: +class CensorshipCircumvention(object): """ - The CensorShipCircumvention object contains methods to detect - and offer solutions to censorship when connecting to Tor. + Connect to the Tor Moat APIs to retrieve censorship + circumvention recommendations, over the Meek client. """ - def __init__(self, common): - + def __init__(self, common, meek, domain_fronting=True): + """ + Set up the CensorshipCircumvention object to hold + common and meek objects. + """ self.common = common + self.meek = meek self.common.log("CensorshipCircumvention", "__init__") - get_tor_paths = self.common.get_tor_paths - ( - self.tor_path, - self.tor_geo_ip_file_path, - self.tor_geo_ipv6_file_path, - self.obfs4proxy_file_path, - self.meek_client_file_path, - ) = get_tor_paths() + # Bail out if we requested domain fronting but we can't use meek + if domain_fronting and not self.meek.meek_proxies: + raise MeekNotRunning() - meek_url = "https://moat.torproject.org.global.prod.fastly.net/" - meek_front = "cdn.sstatic.net" - meek_env = { - "TOR_PT_MANAGED_TRANSPORT_VER": "1", - "TOR_PT_CLIENT_TRANSPORTS": "meek", - } - - # @TODO detect the port from the subprocess output - meek_address = "127.0.0.1" - meek_port = "43533" # hardcoded for testing - self.meek_proxies = { - "http": f"socks5h://{meek_address}:{meek_port}", - "https": f"socks5h://{meek_address}:{meek_port}", - } - - # Start the Meek Client as a subprocess. - # This will be used to do domain fronting to the Tor - # Moat API endpoints for censorship circumvention as - # well as BridgeDB lookups. - - if self.common.platform == "Windows": - # In Windows, hide console window when opening tor.exe subprocess - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - self.meek_proc = subprocess.Popen( - [self.meek_client_file_path, "--url", meek_url, "--front", meek_front], - stdout=subprocess.PIPE, - startupinfo=startupinfo, - bufsize=1, - env=meek_env, - text=True, - ) - else: - self.meek_proc = subprocess.Popen( - [self.meek_client_file_path, "--url", meek_url, "--front", meek_front], - stdout=subprocess.PIPE, - bufsize=1, - env=meek_env, - text=True, - ) - - # if "CMETHOD meek socks5" in line: - # self.meek_host = (line.split(" ")[3].split(":")[0]) - # self.meek_port = (line.split(" ")[3].split(":")[1]) - # self.common.log("CensorshipCircumvention", "__init__", f"Meek host is {self.meek_host}") - # self.common.log("CensorshipCircumvention", "__init__", f"Meek port is {self.meek_port}") - - def censorship_obtain_map(self, country=False): + def request_map(self, country=False): """ Retrieves the Circumvention map from Tor Project and store it locally for further look-ups if required. @@ -108,7 +61,7 @@ class CensorshipCircumvention: endpoint, json=data, headers={"Content-Type": "application/vnd.api+json"}, - proxies=self.meek_proxies, + proxies=self.meek.meek_proxies, ) if r.status_code != 200: self.common.log( @@ -130,7 +83,7 @@ class CensorshipCircumvention: return result - def censorship_obtain_settings(self, country=False, transports=False): + def request_settings(self, country=False, transports=False): """ Retrieves the Circumvention Settings from Tor Project, which will return recommended settings based on the country code of @@ -152,7 +105,7 @@ class CensorshipCircumvention: endpoint, json=data, headers={"Content-Type": "application/vnd.api+json"}, - proxies=self.meek_proxies, + proxies=self.meek.meek_proxies, ) if r.status_code != 200: self.common.log( @@ -185,7 +138,7 @@ class CensorshipCircumvention: return result - def censorship_obtain_builtin_bridges(self): + def request_builtin_bridges(self): """ Retrieves the list of built-in bridges from the Tor Project. """ @@ -193,7 +146,7 @@ class CensorshipCircumvention: r = requests.post( endpoint, headers={"Content-Type": "application/vnd.api+json"}, - proxies=self.meek_proxies, + proxies=self.meek.meek_proxies, ) if r.status_code != 200: self.common.log( diff --git a/cli/onionshare_cli/meek.py b/cli/onionshare_cli/meek.py new file mode 100644 index 00000000..4fc42756 --- /dev/null +++ b/cli/onionshare_cli/meek.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +""" +OnionShare | https://onionshare.org/ + +Copyright (C) 2014-2021 Micah Lee, et al. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" +import subprocess +from queue import Queue, Empty +from threading import Thread + + +class Meek(object): + """ + The Meek object starts the meek-client as a subprocess. + This process is used to do domain-fronting to connect to + the Tor APIs for censorship circumvention and retrieving + bridges, before connecting to Tor. + """ + + def __init__(self, common): + """ + Set up the Meek object + """ + + self.common = common + self.common.log("Meek", "__init__") + + get_tor_paths = self.common.get_tor_paths + ( + self.tor_path, + self.tor_geo_ip_file_path, + self.tor_geo_ipv6_file_path, + self.obfs4proxy_file_path, + self.snowflake_file_path, + self.meek_client_file_path, + ) = get_tor_paths() + + self.meek_proxies = {} + self.meek_url = "https://moat.torproject.org.global.prod.fastly.net/" + self.meek_front = "cdn.sstatic.net" + self.meek_env = { + "TOR_PT_MANAGED_TRANSPORT_VER": "1", + "TOR_PT_CLIENT_TRANSPORTS": "meek", + } + self.meek_host = "127.0.0.1" + self.meek_port = None + + def start(self): + """ + Start the Meek Client and populate the SOCKS proxies dict + for use with requests to the Tor Moat API. + """ + # Small method to read stdout from the subprocess. + # We use this to obtain the random port that Meek + # started on + def enqueue_output(out, queue): + for line in iter(out.readline, b""): + queue.put(line) + out.close() + + # Start the Meek Client as a subprocess. + + if self.common.platform == "Windows": + # In Windows, hide console window when opening tor.exe subprocess + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + self.meek_proc = subprocess.Popen( + [ + self.meek_client_file_path, + "--url", + self.meek_url, + "--front", + self.meek_front, + ], + stdout=subprocess.PIPE, + startupinfo=startupinfo, + bufsize=1, + env=self.meek_env, + text=True, + ) + else: + self.meek_proc = subprocess.Popen( + [ + self.meek_client_file_path, + "--url", + self.meek_url, + "--front", + self.meek_front, + ], + stdout=subprocess.PIPE, + bufsize=1, + env=self.meek_env, + text=True, + ) + + # Queue up the stdout from the subprocess for polling later + q = Queue() + t = Thread(target=enqueue_output, args=(self.meek_proc.stdout, q)) + t.daemon = True # thread dies with the program + t.start() + + while True: + # read stdout without blocking + try: + line = q.get_nowait() + except Empty: + # no stdout yet? + pass + else: # we got stdout + if "CMETHOD meek socks5" in line: + self.meek_host = line.split(" ")[3].split(":")[0] + self.meek_port = line.split(" ")[3].split(":")[1] + self.common.log("Meek", "start", f"Meek host is {self.meek_host}") + self.common.log("Meek", "start", f"Meek port is {self.meek_port}") + break + + if self.meek_port: + self.meek_proxies = { + "http": f"socks5h://{self.meek_host}:{self.meek_port}", + "https": f"socks5h://{self.meek_host}:{self.meek_port}", + } + else: + self.common.log("Meek", "start", "Could not obtain the meek port") + raise MeekNotRunning() + + +class MeekNotRunning(Exception): + """ + We were unable to start Meek or obtain the port + number it started on, in order to do domain fronting. + """ diff --git a/desktop/src/onionshare/gui_common.py b/desktop/src/onionshare/gui_common.py index 0f1dd46e..019cf193 100644 --- a/desktop/src/onionshare/gui_common.py +++ b/desktop/src/onionshare/gui_common.py @@ -409,11 +409,13 @@ class GuiCommon: tor_geo_ipv6_file_path = os.path.join(base_path, "geoip6") obfs4proxy_file_path = os.path.join(base_path, "obfs4proxy") snowflake_file_path = os.path.join(base_path, "snowflake-client") + meek_client_file_path = os.path.join(base_path, "meek-client") else: # Fallback to looking in the path tor_path = shutil.which("tor") obfs4proxy_file_path = shutil.which("obfs4proxy") snowflake_file_path = shutil.which("snowflake-client") + meek_client_file_path = shutil.which("meek-client") prefix = os.path.dirname(os.path.dirname(tor_path)) tor_geo_ip_file_path = os.path.join(prefix, "share/tor/geoip") tor_geo_ipv6_file_path = os.path.join(prefix, "share/tor/geoip6") @@ -423,6 +425,7 @@ class GuiCommon: tor_path = os.path.join(base_path, "Tor", "tor.exe") obfs4proxy_file_path = os.path.join(base_path, "Tor", "obfs4proxy.exe") snowflake_file_path = os.path.join(base_path, "Tor", "snowflake-client.exe") + meek_client_file_path = os.path.join(base_path, "Tor", "meek-client.exe") tor_geo_ip_file_path = os.path.join(base_path, "Data", "Tor", "geoip") tor_geo_ipv6_file_path = os.path.join(base_path, "Data", "Tor", "geoip6") elif self.common.platform == "Darwin": @@ -430,6 +433,7 @@ class GuiCommon: tor_path = os.path.join(base_path, "tor") obfs4proxy_file_path = os.path.join(base_path, "obfs4proxy") snowflake_file_path = os.path.join(base_path, "snowflake-client") + meek_client_file_path = os.path.join(base_path, "meek-client") tor_geo_ip_file_path = os.path.join(base_path, "geoip") tor_geo_ipv6_file_path = os.path.join(base_path, "geoip6") elif self.common.platform == "BSD": @@ -437,6 +441,7 @@ class GuiCommon: tor_geo_ip_file_path = "/usr/local/share/tor/geoip" tor_geo_ipv6_file_path = "/usr/local/share/tor/geoip6" obfs4proxy_file_path = "/usr/local/bin/obfs4proxy" + meek_client_file_path = "/usr/local/bin/meek-client" snowflake_file_path = "/usr/local/bin/snowflake-client" return ( @@ -445,6 +450,7 @@ class GuiCommon: tor_geo_ipv6_file_path, obfs4proxy_file_path, snowflake_file_path, + meek_client_file_path, ) @staticmethod diff --git a/desktop/src/onionshare/moat_dialog.py b/desktop/src/onionshare/moat_dialog.py index 2651736e..78a05482 100644 --- a/desktop/src/onionshare/moat_dialog.py +++ b/desktop/src/onionshare/moat_dialog.py @@ -34,13 +34,15 @@ class MoatDialog(QtWidgets.QDialog): got_bridges = QtCore.Signal(str) - def __init__(self, common): + def __init__(self, common, meek): super(MoatDialog, self).__init__() self.common = common self.common.log("MoatDialog", "__init__") + self.meek = meek + self.setModal(True) self.setWindowTitle(strings._("gui_settings_bridge_moat_button")) self.setWindowIcon(QtGui.QIcon(GuiCommon.get_resource_path("images/logo.png"))) @@ -108,7 +110,7 @@ class MoatDialog(QtWidgets.QDialog): self.submit_button.hide() # BridgeDB fetch - self.t_fetch = MoatThread(self.common, "fetch") + self.t_fetch = MoatThread(self.common, self.meek, "fetch") self.t_fetch.bridgedb_error.connect(self.bridgedb_error) self.t_fetch.captcha_ready.connect(self.captcha_ready) self.t_fetch.start() @@ -130,6 +132,7 @@ class MoatDialog(QtWidgets.QDialog): # BridgeDB check self.t_check = MoatThread( self.common, + self.meek, "check", {"challenge": self.challenge, "solution": self.solution_lineedit.text()}, ) @@ -209,17 +212,20 @@ class MoatThread(QtCore.QThread): captcha_ready = QtCore.Signal(str, str) bridges_ready = QtCore.Signal(str) - def __init__(self, common, action, data={}): + def __init__(self, common, meek, action, data={}): super(MoatThread, self).__init__() self.common = common self.common.log("MoatThread", "__init__", f"action={action}") + self.meek = meek self.transport = "obfs4" self.action = action self.data = data def run(self): - # TODO: Do all of this using domain fronting + + # Start Meek so that we can do domain fronting + self.meek.start() if self.action == "fetch": self.common.log("MoatThread", "run", f"starting fetch") @@ -228,6 +234,7 @@ class MoatThread(QtCore.QThread): r = requests.post( "https://bridges.torproject.org/moat/fetch", headers={"Content-Type": "application/vnd.api+json"}, + proxies=self.meek.meek_proxies, json={ "data": [ { @@ -280,6 +287,7 @@ class MoatThread(QtCore.QThread): r = requests.post( "https://bridges.torproject.org/moat/check", headers={"Content-Type": "application/vnd.api+json"}, + proxies=self.meek.meek_proxies, json={ "data": [ { diff --git a/desktop/src/onionshare/tor_settings_dialog.py b/desktop/src/onionshare/tor_settings_dialog.py index adad6931..e92be2aa 100644 --- a/desktop/src/onionshare/tor_settings_dialog.py +++ b/desktop/src/onionshare/tor_settings_dialog.py @@ -24,6 +24,7 @@ import platform import re import os +from onionshare_cli.meek import Meek from onionshare_cli.settings import Settings from onionshare_cli.onion import Onion @@ -48,6 +49,8 @@ class TorSettingsDialog(QtWidgets.QDialog): self.common.log("TorSettingsDialog", "__init__") + self.meek = Meek(common) + self.setModal(True) self.setWindowTitle(strings._("gui_tor_settings_window_title")) self.setWindowIcon(QtGui.QIcon(GuiCommon.get_resource_path("images/logo.png"))) @@ -78,6 +81,7 @@ class TorSettingsDialog(QtWidgets.QDialog): self.tor_geo_ipv6_file_path, self.obfs4proxy_file_path, self.snowflake_file_path, + self.meek_client_file_path, ) = self.common.gui.get_tor_paths() bridges_label = QtWidgets.QLabel(strings._("gui_settings_tor_bridges_label")) @@ -497,7 +501,7 @@ class TorSettingsDialog(QtWidgets.QDialog): """ self.common.log("TorSettingsDialog", "bridge_moat_button_clicked") - moat_dialog = MoatDialog(self.common) + moat_dialog = MoatDialog(self.common, self.meek) moat_dialog.got_bridges.connect(self.bridge_moat_got_bridges) moat_dialog.exec_() @@ -577,9 +581,7 @@ class TorSettingsDialog(QtWidgets.QDialog): return onion = Onion( - self.common, - use_tmp_dir=True, - get_tor_paths=self.common.gui.get_tor_paths, + self.common, use_tmp_dir=True, get_tor_paths=self.common.gui.get_tor_paths ) tor_con = TorConnectionDialog(self.common, settings, True, onion) @@ -781,10 +783,7 @@ class TorSettingsDialog(QtWidgets.QDialog): Alert(self.common, strings._("gui_settings_moat_bridges_invalid")) return False - settings.set( - "tor_bridges_use_moat_bridges", - moat_bridges, - ) + settings.set("tor_bridges_use_moat_bridges", moat_bridges) settings.set("tor_bridges_use_custom_bridges", "") From bd6390042f73ec64be6cbe3eed4cbc67875f9f6d Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Tue, 19 Oct 2021 11:46:21 +1100 Subject: [PATCH 04/10] Try to bail if we are not in local-only mode and couldn't start the Meek client --- desktop/src/onionshare/moat_dialog.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/desktop/src/onionshare/moat_dialog.py b/desktop/src/onionshare/moat_dialog.py index 78a05482..cedc52d8 100644 --- a/desktop/src/onionshare/moat_dialog.py +++ b/desktop/src/onionshare/moat_dialog.py @@ -25,6 +25,7 @@ import base64 from . import strings from .gui_common import GuiCommon +from onionshare_cli.meek import MeekNotRunning class MoatDialog(QtWidgets.QDialog): @@ -227,6 +228,11 @@ class MoatThread(QtCore.QThread): # Start Meek so that we can do domain fronting self.meek.start() + # We should only fetch bridges if we can domain front, + # but we can override this in local-only mode. + if not self.meek.meek_proxies and not self.common.gui.local_only: + raise MeekNotRunning() + if self.action == "fetch": self.common.log("MoatThread", "run", f"starting fetch") From 1fa82818c3d8467238f35691061a8dceaa7cbcd4 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Wed, 20 Oct 2021 15:55:24 +1100 Subject: [PATCH 05/10] Add meek_client stuff to CLI tests --- cli/tests/test_cli_common.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cli/tests/test_cli_common.py b/cli/tests/test_cli_common.py index a4798d1b..9a64d762 100644 --- a/cli/tests/test_cli_common.py +++ b/cli/tests/test_cli_common.py @@ -162,6 +162,9 @@ class TestGetTorPaths: tor_geo_ip_file_path = os.path.join(base_path, "Resources", "Tor", "geoip") tor_geo_ipv6_file_path = os.path.join(base_path, "Resources", "Tor", "geoip6") obfs4proxy_file_path = os.path.join(base_path, "Resources", "Tor", "obfs4proxy") + meek_client_file_path = os.path.join( + base_path, "Resources", "Tor", "meek-client" + ) snowflake_file_path = os.path.join( base_path, "Resources", "Tor", "snowflake-client" ) @@ -171,6 +174,7 @@ class TestGetTorPaths: tor_geo_ipv6_file_path, obfs4proxy_file_path, snowflake_file_path, + meek_client_file_path, ) @pytest.mark.skipif(sys.platform != "linux", reason="requires Linux") @@ -181,6 +185,7 @@ class TestGetTorPaths: tor_geo_ipv6_file_path, _, # obfs4proxy is optional _, # snowflake-client is optional + _, # meek-client is optional ) = common_obj.get_tor_paths() assert os.path.basename(tor_path) == "tor" @@ -207,6 +212,9 @@ class TestGetTorPaths: snowflake_file_path = os.path.join( os.path.join(base_path, "Tor"), "snowflake-client.exe" ) + meek_client_file_path = os.path.join( + os.path.join(base_path, "Tor"), "meek-client.exe" + ) tor_geo_ip_file_path = os.path.join( os.path.join(os.path.join(base_path, "Data"), "Tor"), "geoip" ) @@ -219,6 +227,7 @@ class TestGetTorPaths: tor_geo_ipv6_file_path, obfs4proxy_file_path, snowflake_file_path, + meek_client_file_path, ) From c81862130ba0969228fa61a3aaf9612ff8424971 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Mon, 25 Oct 2021 10:28:06 +1100 Subject: [PATCH 06/10] Fix comment about meek-client.exe subprocess --- cli/onionshare_cli/meek.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/onionshare_cli/meek.py b/cli/onionshare_cli/meek.py index 4fc42756..482deedd 100644 --- a/cli/onionshare_cli/meek.py +++ b/cli/onionshare_cli/meek.py @@ -74,7 +74,7 @@ class Meek(object): # Start the Meek Client as a subprocess. if self.common.platform == "Windows": - # In Windows, hide console window when opening tor.exe subprocess + # In Windows, hide console window when opening meek-client.exe subprocess startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW self.meek_proc = subprocess.Popen( From 3a715346af241707c952ff446734f8c7bfccd21f Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Mon, 25 Oct 2021 10:44:38 +1100 Subject: [PATCH 07/10] Add cleanup method for the Meek class to kill any meek-client subprocesses once done. Hide stderr from the CLI printed output --- cli/onionshare_cli/__init__.py | 6 +++-- cli/onionshare_cli/meek.py | 37 +++++++++++++++++++++++++++ desktop/src/onionshare/moat_dialog.py | 2 ++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/cli/onionshare_cli/__init__.py b/cli/onionshare_cli/__init__.py index 99992b25..4e34a508 100644 --- a/cli/onionshare_cli/__init__.py +++ b/cli/onionshare_cli/__init__.py @@ -286,8 +286,8 @@ def main(cwd=None): web = Web(common, False, mode_settings, mode) # Create the Meek object and start the meek client - meek = Meek(common) - meek.start() + # meek = Meek(common) + # meek.start() # Create the CensorshipCircumvention object to make # API calls to Tor over Meek @@ -296,6 +296,8 @@ def main(cwd=None): # domain fronting. # censorship_recommended_settings = censorship.request_settings(country="cn") # print(censorship_recommended_settings) + # Clean up the meek subprocess once we're done working with the censorship circumvention API + # meek.cleanup() # Start the Onion object try: diff --git a/cli/onionshare_cli/meek.py b/cli/onionshare_cli/meek.py index 482deedd..675402d3 100644 --- a/cli/onionshare_cli/meek.py +++ b/cli/onionshare_cli/meek.py @@ -18,6 +18,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ import subprocess +import time from queue import Queue, Empty from threading import Thread @@ -86,6 +87,7 @@ class Meek(object): self.meek_front, ], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, startupinfo=startupinfo, bufsize=1, env=self.meek_env, @@ -101,6 +103,7 @@ class Meek(object): self.meek_front, ], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=1, env=self.meek_env, text=True, @@ -136,6 +139,40 @@ class Meek(object): self.common.log("Meek", "start", "Could not obtain the meek port") raise MeekNotRunning() + def cleanup(self): + """ + Kill any meek subprocesses. + """ + self.common.log("Meek", "cleanup") + + if self.meek_proc: + self.meek_proc.terminate() + time.sleep(0.2) + if self.meek_proc.poll() is None: + self.common.log( + "Meek", + "cleanup", + "Tried to terminate meek-client process but it's still running", + ) + try: + self.meek_proc.kill() + time.sleep(0.2) + if self.meek_proc.poll() is None: + self.common.log( + "Meek", + "cleanup", + "Tried to kill meek-client process but it's still running", + ) + except Exception: + self.common.log( + "Meek", "cleanup", "Exception while killing meek-client process" + ) + self.meek_proc = None + + # Reset other Meek settings + self.meek_proxies = {} + self.meek_port = None + class MeekNotRunning(Exception): """ diff --git a/desktop/src/onionshare/moat_dialog.py b/desktop/src/onionshare/moat_dialog.py index af7b70b6..9046c989 100644 --- a/desktop/src/onionshare/moat_dialog.py +++ b/desktop/src/onionshare/moat_dialog.py @@ -264,6 +264,7 @@ class MoatThread(QtCore.QThread): ] }, ) + self.meek.cleanup() if r.status_code != 200: self.common.log("MoatThread", "run", f"status_code={r.status_code}") self.bridgedb_error.emit() @@ -316,6 +317,7 @@ class MoatThread(QtCore.QThread): ] }, ) + self.meek.cleanup() if r.status_code != 200: self.common.log("MoatThread", "run", f"status_code={r.status_code}") self.bridgedb_error.emit() From 6f0674afd8c6b39818dd6ddda417db69d458f68f Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Mon, 25 Oct 2021 11:12:38 +1100 Subject: [PATCH 08/10] React to Meek client binary not found --- cli/onionshare_cli/meek.py | 11 +++++++++++ desktop/src/onionshare/moat_dialog.py | 22 +++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/cli/onionshare_cli/meek.py b/cli/onionshare_cli/meek.py index 675402d3..ff44cb13 100644 --- a/cli/onionshare_cli/meek.py +++ b/cli/onionshare_cli/meek.py @@ -72,6 +72,12 @@ class Meek(object): queue.put(line) out.close() + # Abort early if we can't find the Meek client + # common.get_tor_paths() has already checked it's a file + # so just abort if it's a NoneType object + if self.meek_client_file_path is None: + raise MeekNotFound() + # Start the Meek Client as a subprocess. if self.common.platform == "Windows": @@ -179,3 +185,8 @@ class MeekNotRunning(Exception): We were unable to start Meek or obtain the port number it started on, in order to do domain fronting. """ + +class MeekNotFound(Exception): + """ + We were unable to find the Meek Client binary. + """ diff --git a/desktop/src/onionshare/moat_dialog.py b/desktop/src/onionshare/moat_dialog.py index 9046c989..2821bb1e 100644 --- a/desktop/src/onionshare/moat_dialog.py +++ b/desktop/src/onionshare/moat_dialog.py @@ -26,7 +26,7 @@ import json from . import strings from .gui_common import GuiCommon -from onionshare_cli.meek import MeekNotRunning +from onionshare_cli.meek import MeekNotFound class MoatDialog(QtWidgets.QDialog): @@ -234,12 +234,19 @@ class MoatThread(QtCore.QThread): def run(self): # Start Meek so that we can do domain fronting - self.meek.start() + try: + self.meek.start() + except MeekNotFound: + self.common.log("MoatThread", "run", f"Could not find the Meek Client") + self.bridgedb_error.emit() + return # We should only fetch bridges if we can domain front, # but we can override this in local-only mode. if not self.meek.meek_proxies and not self.common.gui.local_only: - self.common.log("MoatThread", "run", f"Could not identify meek proxies to make request") + self.common.log( + "MoatThread", "run", f"Could not identify meek proxies to make request" + ) self.bridgedb_error.emit() return @@ -256,15 +263,14 @@ class MoatThread(QtCore.QThread): { "version": "0.1.0", "type": "client-transports", - "supported": [ - "obfs4", - "snowflake", - ], + "supported": ["obfs4", "snowflake"], } ] }, ) + self.meek.cleanup() + if r.status_code != 200: self.common.log("MoatThread", "run", f"status_code={r.status_code}") self.bridgedb_error.emit() @@ -317,7 +323,9 @@ class MoatThread(QtCore.QThread): ] }, ) + self.meek.cleanup() + if r.status_code != 200: self.common.log("MoatThread", "run", f"status_code={r.status_code}") self.bridgedb_error.emit() From 8543d215dcdc92fb621a55299a04d8fcef738223 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Mon, 25 Oct 2021 11:45:50 +1100 Subject: [PATCH 09/10] Fix-ups for detecting if the meek binary doesn't exist. Pass the GUI's get_tor_paths down to the CLI when instantiating Meek object --- cli/onionshare_cli/meek.py | 16 +++++++++++----- desktop/src/onionshare/tor_settings_dialog.py | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cli/onionshare_cli/meek.py b/cli/onionshare_cli/meek.py index ff44cb13..762a454c 100644 --- a/cli/onionshare_cli/meek.py +++ b/cli/onionshare_cli/meek.py @@ -17,6 +17,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . """ +import os import subprocess import time from queue import Queue, Empty @@ -31,7 +32,7 @@ class Meek(object): bridges, before connecting to Tor. """ - def __init__(self, common): + def __init__(self, common, get_tor_paths=None): """ Set up the Meek object """ @@ -39,7 +40,9 @@ class Meek(object): self.common = common self.common.log("Meek", "__init__") - get_tor_paths = self.common.get_tor_paths + # Set the path of the meek binary + if not get_tor_paths: + get_tor_paths = self.common.get_tor_paths ( self.tor_path, self.tor_geo_ip_file_path, @@ -72,10 +75,12 @@ class Meek(object): queue.put(line) out.close() + self.common.log("Meek", "start", self.meek_client_file_path) + # Abort early if we can't find the Meek client - # common.get_tor_paths() has already checked it's a file - # so just abort if it's a NoneType object - if self.meek_client_file_path is None: + if self.meek_client_file_path is None or not os.path.exists( + self.meek_client_file_path + ): raise MeekNotFound() # Start the Meek Client as a subprocess. @@ -186,6 +191,7 @@ class MeekNotRunning(Exception): number it started on, in order to do domain fronting. """ + class MeekNotFound(Exception): """ We were unable to find the Meek Client binary. diff --git a/desktop/src/onionshare/tor_settings_dialog.py b/desktop/src/onionshare/tor_settings_dialog.py index 13edd112..6737ae4b 100644 --- a/desktop/src/onionshare/tor_settings_dialog.py +++ b/desktop/src/onionshare/tor_settings_dialog.py @@ -49,7 +49,7 @@ class TorSettingsDialog(QtWidgets.QDialog): self.common.log("TorSettingsDialog", "__init__") - self.meek = Meek(common) + self.meek = Meek(common, get_tor_paths=self.common.gui.get_tor_paths) self.setModal(True) self.setWindowTitle(strings._("gui_tor_settings_window_title")) From 54bfca5f4bfe50587e356f26f48a44cb203fc730 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Mon, 25 Oct 2021 11:56:33 +1100 Subject: [PATCH 10/10] Move debug log call in meek.start() --- cli/onionshare_cli/meek.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/onionshare_cli/meek.py b/cli/onionshare_cli/meek.py index 762a454c..6b31a584 100644 --- a/cli/onionshare_cli/meek.py +++ b/cli/onionshare_cli/meek.py @@ -75,8 +75,6 @@ class Meek(object): queue.put(line) out.close() - self.common.log("Meek", "start", self.meek_client_file_path) - # Abort early if we can't find the Meek client if self.meek_client_file_path is None or not os.path.exists( self.meek_client_file_path @@ -84,6 +82,7 @@ class Meek(object): raise MeekNotFound() # Start the Meek Client as a subprocess. + self.common.log("Meek", "start", "Starting meek client") if self.common.platform == "Windows": # In Windows, hide console window when opening meek-client.exe subprocess