From 1be53c633a3bf65ad7adf2eb1bf869660e8bf91c Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 11 Oct 2018 19:38:05 -0700 Subject: [PATCH 1/7] Remove the whole onionkey module, and generate onion keys using stem 1.7.0 --- install/requirements.txt | 2 +- onionshare/onion.py | 40 ++++++++---- onionshare/onionkey.py | 129 --------------------------------------- 3 files changed, 30 insertions(+), 141 deletions(-) delete mode 100644 onionshare/onionkey.py diff --git a/install/requirements.txt b/install/requirements.txt index 32ec6887..9c8a4a5a 100644 --- a/install/requirements.txt +++ b/install/requirements.txt @@ -22,6 +22,6 @@ PyQt5-sip==4.19.12 PySocks==1.6.8 requests==2.19.1 six==1.11.0 -stem==1.6.0 +stem==1.7.0 urllib3==1.23 Werkzeug==0.14.1 diff --git a/onionshare/onion.py b/onionshare/onion.py index cb73e976..7122c208 100644 --- a/onionshare/onion.py +++ b/onionshare/onion.py @@ -21,10 +21,10 @@ along with this program. If not, see . from stem.control import Controller from stem import ProtocolError, SocketClosed from stem.connection import MissingPassword, UnreadableCookieFile, AuthenticationFailure +from Crypto.PublicKey import RSA import base64, os, sys, tempfile, shutil, urllib, platform, subprocess, time, shlex from distutils.version import LooseVersion as Version -from . import onionkey from . import common, strings from .settings import Settings @@ -441,8 +441,7 @@ class Onion(object): if self.settings.get('private_key'): key_content = self.settings.get('private_key') - # is the key a v2 key? - if onionkey.is_v2_key(key_content): + if self.is_v2_key(key_content): key_type = "RSA1024" # The below section is commented out because re-publishing # a pre-prepared v3 private key is currently unstable in Tor. @@ -458,29 +457,33 @@ class Onion(object): # key_type = "ED25519-V3" else: raise TorErrorProtocolError(strings._('error_invalid_private_key')) + else: + key_type = "NEW" # Work out if we can support v3 onion services, which are preferred if Version(self.tor_version) >= Version('0.3.3.1') and not self.settings.get('use_legacy_v2_onions'): - key_type = "ED25519-V3" - key_content = onionkey.generate_v3_private_key()[0] + key_content = "ED25519-V3" else: # fall back to v2 onion services - key_type = "RSA1024" - key_content = onionkey.generate_v2_private_key()[0] + key_content = "RSA1024" # v3 onions don't yet support basic auth. Our ticket: # https://github.com/micahflee/onionshare/issues/697 - if key_type == "ED25519-V3" and not self.settings.get('use_legacy_v2_onions'): + if key_type == "NEW" and key_content == "ED25519-V3" and not self.settings.get('use_legacy_v2_onions'): basic_auth = None self.stealth = False - self.common.log('Onion', 'start_onion_service', 'key_type={}'.format(key_type)) + debug_message = 'key_type={}'.format(key_type) + if key_type == "NEW": + debug_message += ', key_content={}'.format(key_content) + self.common.log('Onion', 'start_onion_service', '{}'.format(debug_message)) + await_publication = True try: if basic_auth != None: - res = self.c.create_ephemeral_hidden_service({ 80: port }, await_publication=True, basic_auth=basic_auth, key_type=key_type, key_content=key_content) + res = self.c.create_ephemeral_hidden_service({ 80: port }, await_publication=await_publication, basic_auth=basic_auth, key_type=key_type, key_content=key_content) else: # if the stem interface is older than 1.5.0, basic_auth isn't a valid keyword arg - res = self.c.create_ephemeral_hidden_service({ 80: port }, await_publication=True, key_type=key_type, key_content=key_content) + res = self.c.create_ephemeral_hidden_service({ 80: port }, await_publication=await_publication, key_type=key_type, key_content=key_content) except ProtocolError as e: raise TorErrorProtocolError(strings._('error_tor_protocol_error').format(e.args[0])) @@ -575,3 +578,18 @@ class Onion(object): return ('127.0.0.1', 9150) else: return (self.settings.get('socks_address'), self.settings.get('socks_port')) + + def is_v2_key(key): + """ + Helper function for determining if a key is RSA1024 (v2) or not. + """ + try: + # Import the key + key = RSA.importKey(base64.b64decode(key)) + # Is this a v2 Onion key? (1024 bits) If so, we should keep using it. + if key.n.bit_length() == 1024: + return True + else: + return False + except: + return False diff --git a/onionshare/onionkey.py b/onionshare/onionkey.py deleted file mode 100644 index d2c6ad17..00000000 --- a/onionshare/onionkey.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- coding: utf-8 -*- -""" -OnionShare | https://onionshare.org/ - -Copyright (C) 2017 Micah Lee - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -import os -import sys - -import base64 -import hashlib -# Need sha3 if python version is older than 3.6, otherwise -# we can't use hashlib.sha3_256 -if sys.version_info < (3, 6): - import sha3 - -import nacl.signing - -from Crypto.PublicKey import RSA - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa - - -def stem_compatible_base64_blob_from_private_key(private_key_seed: bytes) -> str: - """ - Provides a base64-encoded private key for v3-style Onions. - """ - b = 256 - - def bit(h: bytes, i: int) -> int: - return (h[i // 8] >> (i % 8)) & 1 - - def encode_int(y: int) -> bytes: - bits = [(y >> i) & 1 for i in range(b)] - return b''.join([bytes([(sum([bits[i * 8 + j] << j for j in range(8)]))]) for i in range(b // 8)]) - - def expand_private_key(sk: bytes) -> bytes: - h = hashlib.sha512(sk).digest() - a = 2 ** (b - 2) + sum(2 ** i * bit(h, i) for i in range(3, b - 2)) - k = b''.join([bytes([h[i]]) for i in range(b // 8, b // 4)]) - assert len(k) == 32 - return encode_int(a) + k - - expanded_private_key = expand_private_key(private_key_seed) - return base64.b64encode(expanded_private_key).decode() - - -def onion_url_from_private_key(private_key_seed: bytes) -> str: - """ - Derives the public key (.onion hostname) from a v3-style - Onion private key. - """ - signing_key = nacl.signing.SigningKey(seed=private_key_seed) - public_key = bytes(signing_key.verify_key) - version = b'\x03' - checksum = hashlib.sha3_256(b".onion checksum" + public_key + version).digest()[:2] - onion_address = "http://{}.onion".format(base64.b32encode(public_key + checksum + version).decode().lower()) - return onion_address - - -def generate_v3_private_key(): - """ - Generates a private and public key for use with v3 style Onions. - Returns both the private key as well as the public key (.onion hostname) - """ - private_key_seed = os.urandom(32) - private_key = stem_compatible_base64_blob_from_private_key(private_key_seed) - onion_url = onion_url_from_private_key(private_key_seed) - return (private_key, onion_url) - - -def generate_v2_private_key(): - """ - Generates a private and public key for use with v2 style Onions. - Returns both the serialized private key (compatible with Stem) - as well as the public key (.onion hostname) - """ - # Generate v2 Onion Service private key - private_key = rsa.generate_private_key(public_exponent=65537, - key_size=1024, - backend=default_backend()) - hs_public_key = private_key.public_key() - - # Pre-generate the public key (.onion hostname) - der_format = hs_public_key.public_bytes(encoding=serialization.Encoding.DER, - format=serialization.PublicFormat.PKCS1) - - onion_url = base64.b32encode(hashlib.sha1(der_format).digest()[:-10]).lower().decode() - - # Generate Stem-compatible key content - pem_format = private_key.private_bytes(encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption()) - serialized_key = ''.join(pem_format.decode().split('\n')[1:-2]) - - return (serialized_key, onion_url) - - -def is_v2_key(key): - """ - Helper function for determining if a key is RSA1024 (v2) or not. - """ - try: - # Import the key - key = RSA.importKey(base64.b64decode(key)) - # Is this a v2 Onion key? (1024 bits) If so, we should keep using it. - if key.n.bit_length() == 1024: - return True - else: - return False - except: - return False - From f8f705313b01386c5d8ab708d8d4034bf19f5d8c Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 11 Oct 2018 19:55:53 -0700 Subject: [PATCH 2/7] Drop the PyNaCl dependency, because we are no longer generating v3 onion keys --- install/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/install/requirements.txt b/install/requirements.txt index 9c8a4a5a..20811f7b 100644 --- a/install/requirements.txt +++ b/install/requirements.txt @@ -16,7 +16,6 @@ pefile==2018.8.8 pycparser==2.18 pycryptodome==3.6.6 PyInstaller==3.4 -PyNaCl==1.2.1 PyQt5==5.11.2 PyQt5-sip==4.19.12 PySocks==1.6.8 From b91f1d71bb9f3c5376b43011d948112e86eed50b Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 11 Oct 2018 20:08:23 -0700 Subject: [PATCH 3/7] Bump python required version down to 3.5.3, and finish removing python3-nacl as a dependency --- .travis.yml | 1 + BUILD.md | 12 +++++++++--- install/build_rpm.sh | 2 +- stdeb.cfg | 6 +++--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index e0b5b822..53fd661c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python dist: trusty sudo: required python: + - "3.5.3" - "3.6" - "3.6-dev" - "3.7-dev" diff --git a/BUILD.md b/BUILD.md index 00d24cd2..c47dc427 100644 --- a/BUILD.md +++ b/BUILD.md @@ -11,11 +11,17 @@ cd onionshare Install the needed dependencies: -For Debian-like distros: `apt install -y python3-flask python3-stem python3-pyqt5 python3-cryptography python3-crypto python3-nacl python3-socks python-nautilus tor obfs4proxy python3-pytest build-essential fakeroot python3-all python3-stdeb dh-python` +For Debian-like distros: -On some older versions of Debian you may need to install pysha3 with `pip3 install pysha3` if python3-sha3 is not available. +``` +apt install -y python3-flask python3-stem python3-pyqt5 python3-cryptography python3-crypto python3-socks python-nautilus tor obfs4proxy python3-pytest build-essential fakeroot python3-all python3-stdeb dh-python +``` -For Fedora-like distros: `dnf install -y python3-flask python3-stem python3-qt5 python3-pynacl python3-cryptography python3-crypto python3-pysocks nautilus-python tor obfs4 python3-pytest rpm-build` +For Fedora-like distros: + +``` +dnf install -y python3-flask python3-stem python3-qt5 python3-cryptography python3-crypto python3-pysocks nautilus-python tor obfs4 python3-pytest rpm-build +``` After that you can try both the CLI and the GUI version of OnionShare: diff --git a/install/build_rpm.sh b/install/build_rpm.sh index 3f7a68ac..7a34b271 100755 --- a/install/build_rpm.sh +++ b/install/build_rpm.sh @@ -9,7 +9,7 @@ VERSION=`cat share/version.txt` rm -r build dist >/dev/null 2>&1 # build binary package -python3 setup.py bdist_rpm --requires="python3-flask, python3-stem, python3-qt5, python3-pynacl, python3-cryptography, python3-crypto, python3-pysocks, nautilus-python, tor, obfs4" +python3 setup.py bdist_rpm --requires="python3-flask, python3-stem, python3-qt5, python3-cryptography, python3-crypto, python3-pysocks, nautilus-python, tor, obfs4" # install it echo "" diff --git a/stdeb.cfg b/stdeb.cfg index 2fc3d3bf..0980ef54 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,6 +1,6 @@ [DEFAULT] Package3: onionshare -Depends3: python3-flask, python3-stem, python3-pyqt5, python3-cryptography, python3-crypto, python3-nacl, python3-socks, python-nautilus, tor, obfs4proxy -Build-Depends: python3-pytest, python3-flask, python3-stem, python3-pyqt5, python3-cryptography, python3-crypto, python3-nacl, python3-socks, python-nautilus, tor, obfs4proxy +Depends3: python3-flask, python3-stem, python3-pyqt5, python3-cryptography, python3-crypto, python3-socks, python-nautilus, tor, obfs4proxy +Build-Depends: python3-pytest, python3-flask, python3-stem, python3-pyqt5, python3-cryptography, python3-crypto, python3-socks, python-nautilus, tor, obfs4proxy Suite: bionic -X-Python3-Version: >= 3.6 +X-Python3-Version: >= 3.5.3 From 61a98fbbc1a0b17bfe81c8fe74517ce7285c7b44 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 11 Oct 2018 20:18:23 -0700 Subject: [PATCH 4/7] Add instructions for making sure you have pytest in your path (which you don't by default in debian stretch) --- BUILD.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/BUILD.md b/BUILD.md index c47dc427..3061c4a7 100644 --- a/BUILD.md +++ b/BUILD.md @@ -149,6 +149,12 @@ OnionShare includes PyTest unit tests. To run the tests, first install some depe pip3 install -r install/requirements-tests.txt ``` +You must have `pytest` in your path point to python 3's pytest. If necessary, you might need to make a symlink, like: + +```sh +ln -s /usr/bin/pytest-3 /usr/local/bin/pytest +``` + If you'd like to run the CLI-based tests that Travis runs: ```sh From 12180f08dc117ce1dbd72017b8af97cc77891c9a Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 11 Oct 2018 20:48:46 -0700 Subject: [PATCH 5/7] Remove python3-cryptography dependency --- BUILD.md | 4 ++-- stdeb.cfg | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/BUILD.md b/BUILD.md index 3061c4a7..d8150799 100644 --- a/BUILD.md +++ b/BUILD.md @@ -14,13 +14,13 @@ Install the needed dependencies: For Debian-like distros: ``` -apt install -y python3-flask python3-stem python3-pyqt5 python3-cryptography python3-crypto python3-socks python-nautilus tor obfs4proxy python3-pytest build-essential fakeroot python3-all python3-stdeb dh-python +apt install -y python3-flask python3-stem python3-pyqt5 python3-crypto python3-socks python-nautilus tor obfs4proxy python3-pytest build-essential fakeroot python3-all python3-stdeb dh-python ``` For Fedora-like distros: ``` -dnf install -y python3-flask python3-stem python3-qt5 python3-cryptography python3-crypto python3-pysocks nautilus-python tor obfs4 python3-pytest rpm-build +dnf install -y python3-flask python3-stem python3-qt5 python3-crypto python3-pysocks nautilus-python tor obfs4 python3-pytest rpm-build ``` After that you can try both the CLI and the GUI version of OnionShare: diff --git a/stdeb.cfg b/stdeb.cfg index 0980ef54..6729d1b7 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,6 +1,6 @@ [DEFAULT] Package3: onionshare -Depends3: python3-flask, python3-stem, python3-pyqt5, python3-cryptography, python3-crypto, python3-socks, python-nautilus, tor, obfs4proxy -Build-Depends: python3-pytest, python3-flask, python3-stem, python3-pyqt5, python3-cryptography, python3-crypto, python3-socks, python-nautilus, tor, obfs4proxy +Depends3: python3-flask, python3-stem, python3-pyqt5, python3-crypto, python3-socks, python-nautilus, tor, obfs4proxy +Build-Depends: python3-pytest, python3-flask, python3-stem, python3-pyqt5, python3-crypto, python3-socks, python-nautilus, tor, obfs4proxy Suite: bionic X-Python3-Version: >= 3.5.3 From 5f011e9ed652ab66b5449734e73478de9c51f7e8 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 25 Nov 2018 13:14:56 -0800 Subject: [PATCH 6/7] Fix bug related to persistent addresses when using v2 onion services --- onionshare/onion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionshare/onion.py b/onionshare/onion.py index 7122c208..6066f059 100644 --- a/onionshare/onion.py +++ b/onionshare/onion.py @@ -494,7 +494,7 @@ class Onion(object): # A new private key was generated and is in the Control port response. if self.settings.get('save_private_key'): if not self.settings.get('private_key'): - self.settings.set('private_key', key_content) + self.settings.set('private_key', res.private_key) if self.stealth: # Similar to the PrivateKey, the Control port only returns the ClientAuth @@ -579,7 +579,7 @@ class Onion(object): else: return (self.settings.get('socks_address'), self.settings.get('socks_port')) - def is_v2_key(key): + def is_v2_key(self, key): """ Helper function for determining if a key is RSA1024 (v2) or not. """ From b1245dbba0d6cfa6663a2b3c2dca432733149407 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Mon, 26 Nov 2018 08:20:05 +1100 Subject: [PATCH 7/7] Run tests on multiple python versions in CircleCI --- .circleci/config.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b8758353..70fa3b7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,12 +3,18 @@ # Check https://circleci.com/docs/2.0/language-python/ for more details # version: 2 +workflows: + version: 2 + test: + jobs: + - test-3.5 + - test-3.6 + - test-3.7 + jobs: - build: + test-3.5: &test-template docker: - # specify the version you desire here - # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` - - image: circleci/python:3.6.6 + - image: circleci/python:3.5.6 working_directory: ~/repo @@ -18,7 +24,8 @@ jobs: - run: name: install dependencies command: | - sudo apt-get update && sudo apt-get install -y python3-pip python3-flask python3-stem python3-pyqt5 python3-cryptography python3-crypto python3-nacl python3-socks python3-stdeb python3-all python-nautilus xvfb obfs4proxy + sudo apt-get update + sudo apt-get install -y python3-pip python3-flask python3-stem python3-pyqt5 python3-crypto python3-socks python3-stdeb python3-all python-nautilus xvfb obfs4proxy sudo pip3 install -r install/requirements.txt sudo pip3 install -r install/requirements-tests.txt sudo pip3 install pytest-cov flake8 @@ -37,3 +44,12 @@ jobs: command: | xvfb-run pytest --cov=onionshare --cov=onionshare_gui --cov-report=term-missing -vvv tests/ + test-3.6: + <<: *test-template + docker: + - image: circleci/python:3.6.6 + + test-3.7: + <<: *test-template + docker: + - image: circleci/python:3.7.1