From 8fefa4d6b39ab471c8c8ec46dee14421779fe363 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 29 Mar 2022 21:02:03 -0700 Subject: [PATCH 001/154] Try CircleCI Windows orb --- .circleci/config.yml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b03c22f7..63bf394c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,10 +1,21 @@ -version: 2 +version: 2.1 + +orbs: + win: circleci/windows@4.0.0 + workflows: version: 2 test: jobs: - test-cli - test-gui + build: + jobs: + - build-win32 + requires: + - test-cli + - test-gui + jobs: test-cli: @@ -34,7 +45,7 @@ jobs: poetry run onionshare-cli --local-only --receive --auto-stop-timer 2 poetry run onionshare-cli --local-only --website ../docs --auto-stop-timer 2 poetry run onionshare-cli --local-only --chat --auto-stop-timer 2 - + test-gui: docker: - image: cimg/python:3.9 @@ -58,3 +69,13 @@ jobs: command: | cd ~/repo/desktop QT_DEBUG_PLUGINS=1 xvfb-run poetry run pytest -v ./tests/test_gui_*.py + + build-win32: + executor: + name: win/default + shell: powershell.exe + + steps: + - checkout + - run: systeminfo + - run: Write-Host 'Hello, Windows' From ed04214ae628b713c0b1dc150d6a3057905e6047 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 29 Mar 2022 21:04:39 -0700 Subject: [PATCH 002/154] Fix YAML --- .circleci/config.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 63bf394c..7454a513 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,11 +11,10 @@ workflows: - test-gui build: jobs: - - build-win32 - requires: - - test-cli - - test-gui - + - build-win32: + requires: + - test-cli + - test-gui jobs: test-cli: From 499923f4afda65ac85f2ea97b85ba733fbd3b12f Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 29 Mar 2022 21:05:23 -0700 Subject: [PATCH 003/154] Only try building after tests pass --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7454a513..7a12fdd3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,12 +5,10 @@ orbs: workflows: version: 2 - test: + test_and_build: jobs: - test-cli - test-gui - build: - jobs: - build-win32: requires: - test-cli From 631aff7cce89084c3025b96c4684858708c82b20 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 29 Mar 2022 21:11:46 -0700 Subject: [PATCH 004/154] Don't require gui tests because they crash --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a12fdd3..f0f3f550 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ workflows: - build-win32: requires: - test-cli - - test-gui + # - test-gui jobs: test-cli: From 724e48053cdded29c18fc55ef85b4ffdb992a5cf Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 29 Mar 2022 21:59:39 -0700 Subject: [PATCH 005/154] Try installing some dependencies with chocolatey --- .circleci/config.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f0f3f550..44cde11f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ workflows: jobs: - test-cli - test-gui - - build-win32: + - build-win64: requires: - test-cli # - test-gui @@ -67,12 +67,24 @@ jobs: cd ~/repo/desktop QT_DEBUG_PLUGINS=1 xvfb-run poetry run pytest -v ./tests/test_gui_*.py - build-win32: + build-win64: executor: name: win/default shell: powershell.exe steps: - checkout - - run: systeminfo - - run: Write-Host 'Hello, Windows' + - run: + name: Install chocolately dependencies + command: | + choco install python --version=3.9.12 + choco install 7zip + choco install go + - run: + name: Install poetry + command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - + - run: + name: Install poetry dependencies + command: | + cd ~\project + poetry install From c5e73076a4e73a3b254818fa413969c7dd457c32 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 30 Mar 2022 08:29:16 -0700 Subject: [PATCH 006/154] Add --ci-build to Windows build script, and do more to make the build in CircleCI --- .circleci/config.yml | 9 +- desktop/package/build-windows.py | 313 +++++++++++++++++-------------- 2 files changed, 176 insertions(+), 146 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 44cde11f..7e479b8d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,11 +80,18 @@ jobs: choco install python --version=3.9.12 choco install 7zip choco install go + choco install dotnet3.5 + choco install wixtoolset - run: name: Install poetry command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - - run: name: Install poetry dependencies command: | - cd ~\project + cd ~\project\desktop poetry install + - run: + name: Build + command: | + cd ~\project\desktop + poetry run python .\package\build-windows.py --ci-build diff --git a/desktop/package/build-windows.py b/desktop/package/build-windows.py index 9984dd4f..5ae36e19 100644 --- a/desktop/package/build-windows.py +++ b/desktop/package/build-windows.py @@ -4,6 +4,7 @@ import inspect import subprocess import shutil import uuid +import argparse import xml.etree.ElementTree as ET @@ -160,6 +161,18 @@ def wix_build_components_xml(root, data): def main(): + parser = argparse.ArgumentParser( + formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=48) + ) + parser.add_argument( + "--ci-build", + action="store_true", + dest="ci_build", + help="Build in CI, don't code sign", + ) + args = parser.parse_args() + ci_build = bool(args.ci_build) + desktop_dir = os.path.join(root, "desktop") print("○ Clean up from last build") @@ -426,161 +439,171 @@ def main(): freed_mb = int(freed_bytes / 1024 / 1024) print(f"○ Freed {freed_mb} mb") - print(f"○ Signing onionshare.exe") - sign(os.path.join("build", "exe.win32-3.9", "onionshare.exe"), desktop_dir) + if ci_build: + print("Doing a CI build, skipping code signing and msi packaging") - print(f"○ Signing onionshare-cli.exe") - sign(os.path.join("build", "exe.win32-3.9", "onionshare-cli.exe"), desktop_dir) + else: + print(f"○ Signing onionshare.exe") + sign(os.path.join("build", "exe.win32-3.9", "onionshare.exe"), desktop_dir) - print(f"○ Build the WiX file") - version_filename = os.path.join( - root, "cli", "onionshare_cli", "resources", "version.txt" - ) - with open(version_filename) as f: - version = f.read().strip() + print(f"○ Signing onionshare-cli.exe") + sign(os.path.join("build", "exe.win32-3.9", "onionshare-cli.exe"), desktop_dir) - dist_dir = os.path.join( - root, - "desktop", - "build", - "exe.win32-3.9", - ) + print(f"○ Build the WiX file") + version_filename = os.path.join( + root, "cli", "onionshare_cli", "resources", "version.txt" + ) + with open(version_filename) as f: + version = f.read().strip() - data = { - "id": "TARGETDIR", - "name": "SourceDir", - "dirs": [ - { - "id": "ProgramFilesFolder", - "dirs": [], - }, - { - "id": "ProgramMenuFolder", - "dirs": [], - }, - ], - } - - data["dirs"][0]["dirs"].append( - wix_build_data( - dist_dir, + dist_dir = os.path.join( + root, + "desktop", + "build", "exe.win32-3.9", - "INSTALLDIR", - "OnionShare", - ) - ) - - root_el = ET.Element("Wix", xmlns="http://schemas.microsoft.com/wix/2006/wi") - product_el = ET.SubElement( - root_el, - "Product", - Name="OnionShare", - Manufacturer="Micah Lee, et al.", - Id="*", - UpgradeCode="$(var.ProductUpgradeCode)", - Language="1033", - Codepage="1252", - Version="$(var.ProductVersion)", - ) - ET.SubElement( - product_el, - "Package", - Id="*", - Keywords="Installer", - Description="OnionShare $(var.ProductVersion) Installer", - Manufacturer="Micah Lee, et al.", - InstallerVersion="100", - Languages="1033", - Compressed="yes", - SummaryCodepage="1252", - ) - ET.SubElement(product_el, "Media", Id="1", Cabinet="product.cab", EmbedCab="yes") - ET.SubElement( - product_el, - "Icon", - Id="ProductIcon", - SourceFile="..\\onionshare\\resources\\onionshare.ico", - ) - ET.SubElement(product_el, "Property", Id="ARPPRODUCTICON", Value="ProductIcon") - ET.SubElement( - product_el, - "Property", - Id="ARPHELPLINK", - Value="https://docs.onionshare.org", - ) - ET.SubElement( - product_el, - "Property", - Id="ARPURLINFOABOUT", - Value="https://onionshare.org", - ) - ET.SubElement(product_el, "UIRef", Id="WixUI_Minimal") - ET.SubElement(product_el, "UIRef", Id="WixUI_ErrorProgressText") - ET.SubElement( - product_el, - "WixVariable", - Id="WixUILicenseRtf", - Value="..\\package\\license.rtf", - ) - ET.SubElement( - product_el, - "WixVariable", - Id="WixUIDialogBmp", - Value="..\\package\\dialog.bmp", - ) - ET.SubElement( - product_el, - "MajorUpgrade", - AllowSameVersionUpgrades="yes", - DowngradeErrorMessage="A newer version of [ProductName] is already installed. If you are sure you want to downgrade, remove the existing installation via Programs and Features.", - ) - - wix_build_dir_xml(product_el, data) - component_ids = wix_build_components_xml(product_el, data) - - feature_el = ET.SubElement(product_el, "Feature", Id="DefaultFeature", Level="1") - for component_id in component_ids: - ET.SubElement(feature_el, "ComponentRef", Id=component_id) - ET.SubElement(feature_el, "ComponentRef", Id="ApplicationShortcuts") - - with open(os.path.join(root, "desktop", "build", "OnionShare.wxs"), "w") as f: - f.write('\n') - f.write(f'\n') - f.write( - '\n' ) - ET.indent(root_el) - f.write(ET.tostring(root_el).decode()) + data = { + "id": "TARGETDIR", + "name": "SourceDir", + "dirs": [ + { + "id": "ProgramFilesFolder", + "dirs": [], + }, + { + "id": "ProgramMenuFolder", + "dirs": [], + }, + ], + } - print(f"○ Build the MSI") - run( - [shutil.which("candle.exe"), "OnionShare.wxs"], - os.path.join(desktop_dir, "build"), - ) - run( - [shutil.which("light.exe"), "-ext", "WixUIExtension", "OnionShare.wixobj"], - os.path.join(desktop_dir, "build"), - ) + data["dirs"][0]["dirs"].append( + wix_build_data( + dist_dir, + "exe.win32-3.9", + "INSTALLDIR", + "OnionShare", + ) + ) - print(f"○ Prepare OnionShare.msi for signing") - run( - [ - shutil.which("insignia.exe"), - "-im", + root_el = ET.Element("Wix", xmlns="http://schemas.microsoft.com/wix/2006/wi") + product_el = ET.SubElement( + root_el, + "Product", + Name="OnionShare", + Manufacturer="Micah Lee, et al.", + Id="*", + UpgradeCode="$(var.ProductUpgradeCode)", + Language="1033", + Codepage="1252", + Version="$(var.ProductVersion)", + ) + ET.SubElement( + product_el, + "Package", + Id="*", + Keywords="Installer", + Description="OnionShare $(var.ProductVersion) Installer", + Manufacturer="Micah Lee, et al.", + InstallerVersion="100", + Languages="1033", + Compressed="yes", + SummaryCodepage="1252", + ) + ET.SubElement( + product_el, "Media", Id="1", Cabinet="product.cab", EmbedCab="yes" + ) + ET.SubElement( + product_el, + "Icon", + Id="ProductIcon", + SourceFile="..\\onionshare\\resources\\onionshare.ico", + ) + ET.SubElement(product_el, "Property", Id="ARPPRODUCTICON", Value="ProductIcon") + ET.SubElement( + product_el, + "Property", + Id="ARPHELPLINK", + Value="https://docs.onionshare.org", + ) + ET.SubElement( + product_el, + "Property", + Id="ARPURLINFOABOUT", + Value="https://onionshare.org", + ) + ET.SubElement(product_el, "UIRef", Id="WixUI_Minimal") + ET.SubElement(product_el, "UIRef", Id="WixUI_ErrorProgressText") + ET.SubElement( + product_el, + "WixVariable", + Id="WixUILicenseRtf", + Value="..\\package\\license.rtf", + ) + ET.SubElement( + product_el, + "WixVariable", + Id="WixUIDialogBmp", + Value="..\\package\\dialog.bmp", + ) + ET.SubElement( + product_el, + "MajorUpgrade", + AllowSameVersionUpgrades="yes", + DowngradeErrorMessage="A newer version of [ProductName] is already installed. If you are sure you want to downgrade, remove the existing installation via Programs and Features.", + ) + + wix_build_dir_xml(product_el, data) + component_ids = wix_build_components_xml(product_el, data) + + feature_el = ET.SubElement( + product_el, "Feature", Id="DefaultFeature", Level="1" + ) + for component_id in component_ids: + ET.SubElement(feature_el, "ComponentRef", Id=component_id) + ET.SubElement(feature_el, "ComponentRef", Id="ApplicationShortcuts") + + with open(os.path.join(root, "desktop", "build", "OnionShare.wxs"), "w") as f: + f.write('\n') + f.write(f'\n') + f.write( + '\n' + ) + + ET.indent(root_el) + f.write(ET.tostring(root_el).decode()) + + print(f"○ Build the MSI") + run( + [shutil.which("candle.exe"), "OnionShare.wxs"], + os.path.join(desktop_dir, "build"), + ) + run( + [shutil.which("light.exe"), "-ext", "WixUIExtension", "OnionShare.wixobj"], + os.path.join(desktop_dir, "build"), + ) + + print(f"○ Prepare OnionShare.msi for signing") + run( + [ + shutil.which("insignia.exe"), + "-im", + os.path.join(desktop_dir, "build", "OnionShare.msi"), + ], + error_ok=True, + ) + sign(os.path.join(desktop_dir, "build", "OnionShare.msi")) + + final_msi_filename = os.path.join( + desktop_dir, "dist", f"OnionShare-{version}.msi" + ) + print(f"○ Final MSI: {final_msi_filename}") + os.makedirs(os.path.join(desktop_dir, "dist"), exist_ok=True) + os.rename( os.path.join(desktop_dir, "build", "OnionShare.msi"), - ], - error_ok=True, - ) - sign(os.path.join(desktop_dir, "build", "OnionShare.msi")) - - final_msi_filename = os.path.join(desktop_dir, "dist", f"OnionShare-{version}.msi") - print(f"○ Final MSI: {final_msi_filename}") - os.makedirs(os.path.join(desktop_dir, "dist"), exist_ok=True) - os.rename( - os.path.join(desktop_dir, "build", "OnionShare.msi"), - final_msi_filename, - ) + final_msi_filename, + ) if __name__ == "__main__": From 8d14f53a67835c0b374e4266b0022f1eacb34adf Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 30 Mar 2022 08:55:04 -0700 Subject: [PATCH 007/154] Make build-windows.py not use special characters, and make it detect 32-bit or 64-bit python --- .circleci/config.yml | 16 ++++++++- desktop/package/build-windows.py | 58 +++++++++++++++----------------- 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e479b8d..199273d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,7 +81,6 @@ jobs: choco install 7zip choco install go choco install dotnet3.5 - choco install wixtoolset - run: name: Install poetry command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - @@ -90,8 +89,23 @@ jobs: command: | cd ~\project\desktop poetry install + - run: + name: Get tor + command: | + cd ~\project\desktop + poetry run python .\scripts\get-tor-windows.py + - run: + name: Build meek + command: | + cd ~\project\desktop + python .\scripts\build-meek-client.py - run: name: Build command: | cd ~\project\desktop poetry run python .\package\build-windows.py --ci-build + - run: + name: Compress + command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win-amd64-3.9 -DestinationPath ~\win64-build.zip + - store_artifacts: + path: ~\win64-build.zip diff --git a/desktop/package/build-windows.py b/desktop/package/build-windows.py index 5ae36e19..837b70c6 100644 --- a/desktop/package/build-windows.py +++ b/desktop/package/build-windows.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import os +import sys import inspect import subprocess import shutil @@ -74,7 +75,7 @@ def wix_build_data(dirname, dir_prefix, id_, name): id_prefix = id_ # Skip lib/Pyside2/Examples folder - if "\\build\\exe.win32-3.9\\lib\\PySide2\\examples" in dirname: + if "\\lib\\PySide2\\examples" in dirname: continue id_value = f"{id_prefix}{basename.capitalize().replace('-', '_')}" @@ -175,13 +176,13 @@ def main(): desktop_dir = os.path.join(root, "desktop") - print("○ Clean up from last build") + print("> Clean up from last build") if os.path.exists(os.path.join(desktop_dir, "build")): shutil.rmtree(os.path.join(desktop_dir, "build")) if os.path.exists(os.path.join(desktop_dir, "dist")): shutil.rmtree(os.path.join(desktop_dir, "dist")) - print("○ Building binaries") + print("> Building binaries") run( [ shutil.which("python"), @@ -190,15 +191,18 @@ def main(): ], desktop_dir, ) - before_size = get_size(os.path.join(desktop_dir, "build", "exe.win32-3.9")) - print("○ Delete unused PySide2 stuff to save space") + if "64 bit" in sys.version: + python_arch = "win-amd64" + else: + python_arch = "win32" + build_path = os.path.join(desktop_dir, "build", f"exe.{python_arch}-3.9") + + before_size = get_size(build_path) + + print("> Delete unused PySide2 stuff to save space") for dirname in ["examples", "qml"]: - shutil.rmtree( - os.path.join( - desktop_dir, "build", "exe.win32-3.9", "lib", "PySide2", dirname - ) - ) + shutil.rmtree(os.path.join(build_path, "lib", "PySide2", dirname)) for filename in [ "lconvert.exe", "linguist.exe", @@ -425,43 +429,36 @@ def main(): ]: os.remove( os.path.join( - desktop_dir, - "build", - "exe.win32-3.9", + build_path, "lib", "PySide2", filename.replace("/", "\\"), ) ) - after_size = get_size(os.path.join(desktop_dir, "build", "exe.win32-3.9")) + after_size = get_size(build_path) freed_bytes = before_size - after_size freed_mb = int(freed_bytes / 1024 / 1024) - print(f"○ Freed {freed_mb} mb") + print(f"> Freed {freed_mb} mb") if ci_build: print("Doing a CI build, skipping code signing and msi packaging") else: - print(f"○ Signing onionshare.exe") - sign(os.path.join("build", "exe.win32-3.9", "onionshare.exe"), desktop_dir) + print(f"> Signing onionshare.exe") + sign(os.path.join(build_path, "onionshare.exe"), desktop_dir) - print(f"○ Signing onionshare-cli.exe") - sign(os.path.join("build", "exe.win32-3.9", "onionshare-cli.exe"), desktop_dir) + print(f"> Signing onionshare-cli.exe") + sign(os.path.join(build_path, "onionshare-cli.exe"), desktop_dir) - print(f"○ Build the WiX file") + print(f"> Build the WiX file") version_filename = os.path.join( root, "cli", "onionshare_cli", "resources", "version.txt" ) with open(version_filename) as f: version = f.read().strip() - dist_dir = os.path.join( - root, - "desktop", - "build", - "exe.win32-3.9", - ) + dist_dir = os.path.join(root, build_path) data = { "id": "TARGETDIR", @@ -480,8 +477,7 @@ def main(): data["dirs"][0]["dirs"].append( wix_build_data( - dist_dir, - "exe.win32-3.9", + build_path, "INSTALLDIR", "OnionShare", ) @@ -574,7 +570,7 @@ def main(): ET.indent(root_el) f.write(ET.tostring(root_el).decode()) - print(f"○ Build the MSI") + print(f"> Build the MSI") run( [shutil.which("candle.exe"), "OnionShare.wxs"], os.path.join(desktop_dir, "build"), @@ -584,7 +580,7 @@ def main(): os.path.join(desktop_dir, "build"), ) - print(f"○ Prepare OnionShare.msi for signing") + print(f"> Prepare OnionShare.msi for signing") run( [ shutil.which("insignia.exe"), @@ -598,7 +594,7 @@ def main(): final_msi_filename = os.path.join( desktop_dir, "dist", f"OnionShare-{version}.msi" ) - print(f"○ Final MSI: {final_msi_filename}") + print(f"> Final MSI: {final_msi_filename}") os.makedirs(os.path.join(desktop_dir, "dist"), exist_ok=True) os.rename( os.path.join(desktop_dir, "build", "OnionShare.msi"), From dcfb9640cfefb7335dbf100313e195a291319ee0 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 30 Mar 2022 12:26:00 -0700 Subject: [PATCH 008/154] Comment out deleting files, to see if this makes binary run --- .circleci/config.yml | 5 +- desktop/package/build-windows.py | 480 +++++++++++++++---------------- 2 files changed, 242 insertions(+), 243 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 199273d4..330b4b53 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,7 +80,6 @@ jobs: choco install python --version=3.9.12 choco install 7zip choco install go - choco install dotnet3.5 - run: name: Install poetry command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - @@ -106,6 +105,6 @@ jobs: poetry run python .\package\build-windows.py --ci-build - run: name: Compress - command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win-amd64-3.9 -DestinationPath ~\win64-build.zip + command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win-amd64-3.9 -DestinationPath ~\onionshare-win64.zip - store_artifacts: - path: ~\win64-build.zip + path: ~\onionshare-win64.zip diff --git a/desktop/package/build-windows.py b/desktop/package/build-windows.py index 837b70c6..61ad692e 100644 --- a/desktop/package/build-windows.py +++ b/desktop/package/build-windows.py @@ -198,248 +198,248 @@ def main(): python_arch = "win32" build_path = os.path.join(desktop_dir, "build", f"exe.{python_arch}-3.9") - before_size = get_size(build_path) + # before_size = get_size(build_path) - print("> Delete unused PySide2 stuff to save space") - for dirname in ["examples", "qml"]: - shutil.rmtree(os.path.join(build_path, "lib", "PySide2", dirname)) - for filename in [ - "lconvert.exe", - "linguist.exe", - "lrelease.exe", - "lupdate.exe", - "plugins/assetimporters/assimp.dll", - "plugins/assetimporters/uip.dll", - "plugins/audio/qtaudio_wasapi.dll", - "plugins/audio/qtaudio_windows.dll", - "plugins/bearer/qgenericbearer.dll", - "plugins/canbus/qtpassthrucanbus.dll", - "plugins/canbus/qtpeakcanbus.dll", - "plugins/canbus/qtsysteccanbus.dll", - "plugins/canbus/qttinycanbus.dll", - "plugins/canbus/qtvectorcanbus.dll", - "plugins/canbus/qtvirtualcanbus.dll", - "plugins/gamepads/xinputgamepad.dll", - "plugins/generic/qtuiotouchplugin.dll", - "plugins/geometryloaders/defaultgeometryloader.dll", - "plugins/geometryloaders/gltfgeometryloader.dll", - "plugins/geoservices/qtgeoservices_esri.dll", - "plugins/geoservices/qtgeoservices_itemsoverlay.dll", - "plugins/geoservices/qtgeoservices_mapbox.dll", - "plugins/geoservices/qtgeoservices_nokia.dll", - "plugins/geoservices/qtgeoservices_osm.dll", - "plugins/mediaservice/dsengine.dll", - "plugins/mediaservice/qtmedia_audioengine.dll", - "plugins/mediaservice/wmfengine.dll", - "plugins/platforminputcontexts/qtvirtualkeyboardplugin.dll", - "plugins/platforms/qdirect2d.dll", - "plugins/platforms/qoffscreen.dll", - "plugins/platforms/qwebgl.dll", - "plugins/platformthemes/qxdgdesktopportal.dll", - "plugins/playlistformats/qtmultimedia_m3u.dll", - "plugins/position/qtposition_positionpoll.dll", - "plugins/position/qtposition_serialnmea.dll", - "plugins/position/qtposition_winrt.dll", - "plugins/printsupport/windowsprintersupport.dll", - "plugins/qmltooling/qmldbg_debugger.dll", - "plugins/qmltooling/qmldbg_inspector.dll", - "plugins/qmltooling/qmldbg_local.dll", - "plugins/qmltooling/qmldbg_messages.dll", - "plugins/qmltooling/qmldbg_native.dll", - "plugins/qmltooling/qmldbg_nativedebugger.dll", - "plugins/qmltooling/qmldbg_preview.dll", - "plugins/qmltooling/qmldbg_profiler.dll", - "plugins/qmltooling/qmldbg_quickprofiler.dll", - "plugins/qmltooling/qmldbg_server.dll", - "plugins/qmltooling/qmldbg_tcp.dll", - "plugins/renderers/openglrenderer.dll", - "plugins/renderplugins/scene2d.dll", - "plugins/scenegraph/qsgd3d12backend.dll", - "plugins/sceneparsers/gltfsceneexport.dll", - "plugins/sceneparsers/gltfsceneimport.dll", - "plugins/sensorgestures/qtsensorgestures_plugin.dll", - "plugins/sensorgestures/qtsensorgestures_shakeplugin.dll", - "plugins/sensors/qtsensors_generic.dll", - "plugins/sqldrivers/qsqlite.dll", - "plugins/sqldrivers/qsqlodbc.dll", - "plugins/sqldrivers/qsqlpsql.dll", - "plugins/styles/qwindowsvistastyle.dll", - "plugins/texttospeech/qtexttospeech_sapi.dll", - "plugins/virtualkeyboard/qtvirtualkeyboard_hangul.dll", - "plugins/virtualkeyboard/qtvirtualkeyboard_openwnn.dll", - "plugins/virtualkeyboard/qtvirtualkeyboard_pinyin.dll", - "plugins/virtualkeyboard/qtvirtualkeyboard_tcime.dll", - "plugins/virtualkeyboard/qtvirtualkeyboard_thai.dll", - "plugins/webview/qtwebview_webengine.dll", - "pyside2-lupdate.exe", - "Qt3DAnimation.pyd", - "Qt3DAnimation.pyi", - "Qt3DCore.pyd", - "Qt3DCore.pyi", - "Qt3DExtras.pyd", - "Qt3DExtras.pyi", - "Qt3DInput.pyd", - "Qt3DInput.pyi", - "Qt3DLogic.pyd", - "Qt3DLogic.pyi", - "Qt3DRender.pyd", - "Qt3DRender.pyi", - "Qt53DAnimation.dll", - "Qt53DCore.dll", - "Qt53DExtras.dll", - "Qt53DInput.dll", - "Qt53DLogic.dll", - "Qt53DQuick.dll", - "Qt53DQuickAnimation.dll", - "Qt53DQuickExtras.dll", - "Qt53DQuickInput.dll", - "Qt53DQuickRender.dll", - "Qt53DQuickScene2D.dll", - "Qt53DRender.dll", - "Qt5Bluetooth.dll", - "Qt5Bodymovin.dll", - "Qt5Charts.dll", - "Qt5Concurrent.dll", - "Qt5DataVisualization.dll", - "Qt5DBus.dll", - "Qt5Designer.dll", - "Qt5DesignerComponents.dll", - "Qt5Gamepad.dll", - "Qt5Help.dll", - "Qt5Location.dll", - "Qt5Multimedia.dll", - "Qt5MultimediaQuick.dll", - "Qt5MultimediaWidgets.dll", - "Qt5Nfc.dll", - "Qt5OpenGL.dll", - "Qt5Pdf.dll", - "Qt5PdfWidgets.dll", - "Qt5Positioning.dll", - "Qt5PositioningQuick.dll", - "Qt5PrintSupport.dll", - "Qt5Purchasing.dll", - "Qt5Quick.dll", - "Qt5Quick3D.dll", - "Qt5Quick3DAssetImport.dll", - "Qt5Quick3DRender.dll", - "Qt5Quick3DRuntimeRender.dll", - "Qt5Quick3DUtils.dll", - "Qt5QuickControls2.dll", - "Qt5QuickParticles.dll", - "Qt5QuickShapes.dll", - "Qt5QuickTemplates2.dll", - "Qt5QuickTest.dll", - "Qt5QuickWidgets.dll", - "Qt5RemoteObjects.dll", - "Qt5Script.dll", - "Qt5ScriptTools.dll", - "Qt5Scxml.dll", - "Qt5Sensors.dll", - "Qt5SerialBus.dll", - "Qt5SerialPort.dll", - "Qt5Sql.dll", - "Qt5Svg.dll", - "Qt5Test.dll", - "Qt5TextToSpeech.dll", - "Qt5VirtualKeyboard.dll", - "Qt5WebChannel.dll", - "Qt5WebEngine.dll", - "Qt5WebEngineCore.dll", - "Qt5WebEngineWidgets.dll", - "Qt5WebSockets.dll", - "Qt5WebView.dll", - "Qt5Xml.dll", - "Qt5XmlPatterns.dll", - "QtAxContainer.pyd", - "QtAxContainer.pyi", - "QtCharts.pyd", - "QtCharts.pyi", - "QtConcurrent.pyd", - "QtConcurrent.pyi", - "QtDataVisualization.pyd", - "QtDataVisualization.pyi", - "qtdiag.exe", - "QtHelp.pyd", - "QtHelp.pyi", - "QtLocation.pyd", - "QtLocation.pyi", - "QtMultimedia.pyd", - "QtMultimedia.pyi", - "QtMultimediaWidgets.pyd", - "QtMultimediaWidgets.pyi", - "QtNetwork.pyd", - "QtNetwork.pyi", - "QtOpenGL.pyd", - "QtOpenGL.pyi", - "QtOpenGLFunctions.pyd", - "QtOpenGLFunctions.pyi", - "QtPositioning.pyd", - "QtPositioning.pyi", - "QtPrintSupport.pyd", - "QtPrintSupport.pyi", - "QtQml.pyd", - "QtQml.pyi", - "QtQuick.pyd", - "QtQuick.pyi", - "QtQuickControls2.pyd", - "QtQuickControls2.pyi", - "QtQuickWidgets.pyd", - "QtQuickWidgets.pyi", - "QtRemoteObjects.pyd", - "QtRemoteObjects.pyi", - "QtScript.pyd", - "QtScript.pyi", - "QtScriptTools.pyd", - "QtScriptTools.pyi", - "QtScxml.pyd", - "QtScxml.pyi", - "QtSensors.pyd", - "QtSensors.pyi", - "QtSerialPort.pyd", - "QtSerialPort.pyi", - "QtSql.pyd", - "QtSql.pyi", - "QtSvg.pyd", - "QtSvg.pyi", - "QtTest.pyd", - "QtTest.pyi", - "QtTextToSpeech.pyd", - "QtTextToSpeech.pyi", - "QtUiTools.pyd", - "QtUiTools.pyi", - "QtWebChannel.pyd", - "QtWebChannel.pyi", - "QtWebEngine.pyd", - "QtWebEngine.pyi", - "QtWebEngineCore.pyd", - "QtWebEngineCore.pyi", - "QtWebEngineProcess.exe", - "QtWebEngineWidgets.pyd", - "QtWebEngineWidgets.pyi", - "QtWebSockets.pyd", - "QtWebSockets.pyi", - "QtWinExtras.pyd", - "QtWinExtras.pyi", - "QtXml.pyd", - "QtXml.pyi", - "QtXmlPatterns.pyd", - "QtXmlPatterns.pyi", - "rcc.exe", - "uic.exe", - ]: - os.remove( - os.path.join( - build_path, - "lib", - "PySide2", - filename.replace("/", "\\"), - ) - ) + # print("> Delete unused PySide2 stuff to save space") + # for dirname in ["examples", "qml"]: + # shutil.rmtree(os.path.join(build_path, "lib", "PySide2", dirname)) + # for filename in [ + # "lconvert.exe", + # "linguist.exe", + # "lrelease.exe", + # "lupdate.exe", + # "plugins/assetimporters/assimp.dll", + # "plugins/assetimporters/uip.dll", + # "plugins/audio/qtaudio_wasapi.dll", + # "plugins/audio/qtaudio_windows.dll", + # "plugins/bearer/qgenericbearer.dll", + # "plugins/canbus/qtpassthrucanbus.dll", + # "plugins/canbus/qtpeakcanbus.dll", + # "plugins/canbus/qtsysteccanbus.dll", + # "plugins/canbus/qttinycanbus.dll", + # "plugins/canbus/qtvectorcanbus.dll", + # "plugins/canbus/qtvirtualcanbus.dll", + # "plugins/gamepads/xinputgamepad.dll", + # "plugins/generic/qtuiotouchplugin.dll", + # "plugins/geometryloaders/defaultgeometryloader.dll", + # "plugins/geometryloaders/gltfgeometryloader.dll", + # "plugins/geoservices/qtgeoservices_esri.dll", + # "plugins/geoservices/qtgeoservices_itemsoverlay.dll", + # "plugins/geoservices/qtgeoservices_mapbox.dll", + # "plugins/geoservices/qtgeoservices_nokia.dll", + # "plugins/geoservices/qtgeoservices_osm.dll", + # "plugins/mediaservice/dsengine.dll", + # "plugins/mediaservice/qtmedia_audioengine.dll", + # "plugins/mediaservice/wmfengine.dll", + # "plugins/platforminputcontexts/qtvirtualkeyboardplugin.dll", + # "plugins/platforms/qdirect2d.dll", + # "plugins/platforms/qoffscreen.dll", + # "plugins/platforms/qwebgl.dll", + # "plugins/platformthemes/qxdgdesktopportal.dll", + # "plugins/playlistformats/qtmultimedia_m3u.dll", + # "plugins/position/qtposition_positionpoll.dll", + # "plugins/position/qtposition_serialnmea.dll", + # "plugins/position/qtposition_winrt.dll", + # "plugins/printsupport/windowsprintersupport.dll", + # "plugins/qmltooling/qmldbg_debugger.dll", + # "plugins/qmltooling/qmldbg_inspector.dll", + # "plugins/qmltooling/qmldbg_local.dll", + # "plugins/qmltooling/qmldbg_messages.dll", + # "plugins/qmltooling/qmldbg_native.dll", + # "plugins/qmltooling/qmldbg_nativedebugger.dll", + # "plugins/qmltooling/qmldbg_preview.dll", + # "plugins/qmltooling/qmldbg_profiler.dll", + # "plugins/qmltooling/qmldbg_quickprofiler.dll", + # "plugins/qmltooling/qmldbg_server.dll", + # "plugins/qmltooling/qmldbg_tcp.dll", + # "plugins/renderers/openglrenderer.dll", + # "plugins/renderplugins/scene2d.dll", + # "plugins/scenegraph/qsgd3d12backend.dll", + # "plugins/sceneparsers/gltfsceneexport.dll", + # "plugins/sceneparsers/gltfsceneimport.dll", + # "plugins/sensorgestures/qtsensorgestures_plugin.dll", + # "plugins/sensorgestures/qtsensorgestures_shakeplugin.dll", + # "plugins/sensors/qtsensors_generic.dll", + # "plugins/sqldrivers/qsqlite.dll", + # "plugins/sqldrivers/qsqlodbc.dll", + # "plugins/sqldrivers/qsqlpsql.dll", + # "plugins/styles/qwindowsvistastyle.dll", + # "plugins/texttospeech/qtexttospeech_sapi.dll", + # "plugins/virtualkeyboard/qtvirtualkeyboard_hangul.dll", + # "plugins/virtualkeyboard/qtvirtualkeyboard_openwnn.dll", + # "plugins/virtualkeyboard/qtvirtualkeyboard_pinyin.dll", + # "plugins/virtualkeyboard/qtvirtualkeyboard_tcime.dll", + # "plugins/virtualkeyboard/qtvirtualkeyboard_thai.dll", + # "plugins/webview/qtwebview_webengine.dll", + # "pyside2-lupdate.exe", + # "Qt3DAnimation.pyd", + # "Qt3DAnimation.pyi", + # "Qt3DCore.pyd", + # "Qt3DCore.pyi", + # "Qt3DExtras.pyd", + # "Qt3DExtras.pyi", + # "Qt3DInput.pyd", + # "Qt3DInput.pyi", + # "Qt3DLogic.pyd", + # "Qt3DLogic.pyi", + # "Qt3DRender.pyd", + # "Qt3DRender.pyi", + # "Qt53DAnimation.dll", + # "Qt53DCore.dll", + # "Qt53DExtras.dll", + # "Qt53DInput.dll", + # "Qt53DLogic.dll", + # "Qt53DQuick.dll", + # "Qt53DQuickAnimation.dll", + # "Qt53DQuickExtras.dll", + # "Qt53DQuickInput.dll", + # "Qt53DQuickRender.dll", + # "Qt53DQuickScene2D.dll", + # "Qt53DRender.dll", + # "Qt5Bluetooth.dll", + # "Qt5Bodymovin.dll", + # "Qt5Charts.dll", + # "Qt5Concurrent.dll", + # "Qt5DataVisualization.dll", + # "Qt5DBus.dll", + # "Qt5Designer.dll", + # "Qt5DesignerComponents.dll", + # "Qt5Gamepad.dll", + # "Qt5Help.dll", + # "Qt5Location.dll", + # "Qt5Multimedia.dll", + # "Qt5MultimediaQuick.dll", + # "Qt5MultimediaWidgets.dll", + # "Qt5Nfc.dll", + # "Qt5OpenGL.dll", + # "Qt5Pdf.dll", + # "Qt5PdfWidgets.dll", + # "Qt5Positioning.dll", + # "Qt5PositioningQuick.dll", + # "Qt5PrintSupport.dll", + # "Qt5Purchasing.dll", + # "Qt5Quick.dll", + # "Qt5Quick3D.dll", + # "Qt5Quick3DAssetImport.dll", + # "Qt5Quick3DRender.dll", + # "Qt5Quick3DRuntimeRender.dll", + # "Qt5Quick3DUtils.dll", + # "Qt5QuickControls2.dll", + # "Qt5QuickParticles.dll", + # "Qt5QuickShapes.dll", + # "Qt5QuickTemplates2.dll", + # "Qt5QuickTest.dll", + # "Qt5QuickWidgets.dll", + # "Qt5RemoteObjects.dll", + # "Qt5Script.dll", + # "Qt5ScriptTools.dll", + # "Qt5Scxml.dll", + # "Qt5Sensors.dll", + # "Qt5SerialBus.dll", + # "Qt5SerialPort.dll", + # "Qt5Sql.dll", + # "Qt5Svg.dll", + # "Qt5Test.dll", + # "Qt5TextToSpeech.dll", + # "Qt5VirtualKeyboard.dll", + # "Qt5WebChannel.dll", + # "Qt5WebEngine.dll", + # "Qt5WebEngineCore.dll", + # "Qt5WebEngineWidgets.dll", + # "Qt5WebSockets.dll", + # "Qt5WebView.dll", + # "Qt5Xml.dll", + # "Qt5XmlPatterns.dll", + # "QtAxContainer.pyd", + # "QtAxContainer.pyi", + # "QtCharts.pyd", + # "QtCharts.pyi", + # "QtConcurrent.pyd", + # "QtConcurrent.pyi", + # "QtDataVisualization.pyd", + # "QtDataVisualization.pyi", + # "qtdiag.exe", + # "QtHelp.pyd", + # "QtHelp.pyi", + # "QtLocation.pyd", + # "QtLocation.pyi", + # "QtMultimedia.pyd", + # "QtMultimedia.pyi", + # "QtMultimediaWidgets.pyd", + # "QtMultimediaWidgets.pyi", + # "QtNetwork.pyd", + # "QtNetwork.pyi", + # "QtOpenGL.pyd", + # "QtOpenGL.pyi", + # "QtOpenGLFunctions.pyd", + # "QtOpenGLFunctions.pyi", + # "QtPositioning.pyd", + # "QtPositioning.pyi", + # "QtPrintSupport.pyd", + # "QtPrintSupport.pyi", + # "QtQml.pyd", + # "QtQml.pyi", + # "QtQuick.pyd", + # "QtQuick.pyi", + # "QtQuickControls2.pyd", + # "QtQuickControls2.pyi", + # "QtQuickWidgets.pyd", + # "QtQuickWidgets.pyi", + # "QtRemoteObjects.pyd", + # "QtRemoteObjects.pyi", + # "QtScript.pyd", + # "QtScript.pyi", + # "QtScriptTools.pyd", + # "QtScriptTools.pyi", + # "QtScxml.pyd", + # "QtScxml.pyi", + # "QtSensors.pyd", + # "QtSensors.pyi", + # "QtSerialPort.pyd", + # "QtSerialPort.pyi", + # "QtSql.pyd", + # "QtSql.pyi", + # "QtSvg.pyd", + # "QtSvg.pyi", + # "QtTest.pyd", + # "QtTest.pyi", + # "QtTextToSpeech.pyd", + # "QtTextToSpeech.pyi", + # "QtUiTools.pyd", + # "QtUiTools.pyi", + # "QtWebChannel.pyd", + # "QtWebChannel.pyi", + # "QtWebEngine.pyd", + # "QtWebEngine.pyi", + # "QtWebEngineCore.pyd", + # "QtWebEngineCore.pyi", + # "QtWebEngineProcess.exe", + # "QtWebEngineWidgets.pyd", + # "QtWebEngineWidgets.pyi", + # "QtWebSockets.pyd", + # "QtWebSockets.pyi", + # "QtWinExtras.pyd", + # "QtWinExtras.pyi", + # "QtXml.pyd", + # "QtXml.pyi", + # "QtXmlPatterns.pyd", + # "QtXmlPatterns.pyi", + # "rcc.exe", + # "uic.exe", + # ]: + # os.remove( + # os.path.join( + # build_path, + # "lib", + # "PySide2", + # filename.replace("/", "\\"), + # ) + # ) - after_size = get_size(build_path) - freed_bytes = before_size - after_size - freed_mb = int(freed_bytes / 1024 / 1024) - print(f"> Freed {freed_mb} mb") + # after_size = get_size(build_path) + # freed_bytes = before_size - after_size + # freed_mb = int(freed_bytes / 1024 / 1024) + # print(f"> Freed {freed_mb} mb") if ci_build: print("Doing a CI build, skipping code signing and msi packaging") From 638d35a465e6aab885189f05dddee1b3aa69abad Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 30 Mar 2022 12:50:28 -0700 Subject: [PATCH 009/154] Try installing 32-bit python and implementing build-win32 --- .circleci/config.yml | 46 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 330b4b53..a9a1ea93 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,10 @@ workflows: requires: - test-cli # - test-gui + - build-win32: + requires: + - test-cli + # - test-gui jobs: test-cli: @@ -108,3 +112,45 @@ jobs: command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win-amd64-3.9 -DestinationPath ~\onionshare-win64.zip - store_artifacts: path: ~\onionshare-win64.zip + + build-win32: + executor: + name: win/default + shell: powershell.exe + + steps: + - checkout + - run: + name: Install chocolately dependencies + command: | + choco install python3 --params "/InstallDir32:C:\Python3.9-32bit" --version=3.9.12 + choco install 7zip + choco install go + - run: + name: Install poetry + command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | C:\Python3.9-32bit\python - + - run: + name: Install poetry dependencies + command: | + cd ~\project\desktop + poetry install + - run: + name: Get tor + command: | + cd ~\project\desktop + poetry run python .\scripts\get-tor-windows.py + - run: + name: Build meek + command: | + cd ~\project\desktop + python .\scripts\build-meek-client.py + - run: + name: Build + command: | + cd ~\project\desktop + poetry run python .\package\build-windows.py --ci-build + - run: + name: Compress + command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win32-3.9 -DestinationPath ~\onionshare-win32.zip + - store_artifacts: + path: ~\onionshare-win32.zip From defcd6ed2c78945404e66549ff61df9d2ec0507b Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 30 Mar 2022 13:49:45 -0700 Subject: [PATCH 010/154] Try installing 32-bit golang and poetry --- .circleci/config.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a9a1ea93..4a3da5f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -123,12 +123,16 @@ jobs: - run: name: Install chocolately dependencies command: | - choco install python3 --params "/InstallDir32:C:\Python3.9-32bit" --version=3.9.12 + choco install python3 --params "/InstallDir32:C:\Python39-32bit" --version=3.9.12 choco install 7zip - choco install go + - run: + name: Install 32-bit golang + command: | + Invoke-WebRequest -Uri https://go.dev/dl/go1.18.windows-386.msi -OutFile ~\Downloads\go1.18.windows-386.msi + Invoke-Command -ComputerName LabMachine2k16 -ScriptBlock{ Start-Process ~\Downloads\go1.18.windows-386.msi -ArgumentList "/quiet" } - run: name: Install poetry - command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | C:\Python3.9-32bit\python - + command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | C:\Python39-32bit\python - - run: name: Install poetry dependencies command: | From 27dc8082ec83a7ae20ab29f691b5b878ab415ca4 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 30 Mar 2022 13:56:18 -0700 Subject: [PATCH 011/154] Actually try installing 32-bit golang and poetry --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a3da5f3..921960d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -126,13 +126,13 @@ jobs: choco install python3 --params "/InstallDir32:C:\Python39-32bit" --version=3.9.12 choco install 7zip - run: - name: Install 32-bit golang + name: Install golang (32-bit) command: | Invoke-WebRequest -Uri https://go.dev/dl/go1.18.windows-386.msi -OutFile ~\Downloads\go1.18.windows-386.msi - Invoke-Command -ComputerName LabMachine2k16 -ScriptBlock{ Start-Process ~\Downloads\go1.18.windows-386.msi -ArgumentList "/quiet" } + msiexec.exe /i "~\Downloads\go1.18.windows-386.msi" /qn - run: - name: Install poetry - command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | C:\Python39-32bit\python - + name: Install poetry (32-bit) + command: C:\Python39-32bit\python -m pip install poetry - run: name: Install poetry dependencies command: | From 1d9ce020e5186e942ab2e4a3ebc807f866ea7a43 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 30 Mar 2022 14:30:49 -0700 Subject: [PATCH 012/154] In 32-bit build, use 32-bit python --- .circleci/config.yml | 10 +++++----- desktop/package/build-windows.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 921960d0..ea3c3c0f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,27 +132,27 @@ jobs: msiexec.exe /i "~\Downloads\go1.18.windows-386.msi" /qn - run: name: Install poetry (32-bit) - command: C:\Python39-32bit\python -m pip install poetry + command: C:\Python39-32bit\Scripts\pip install poetry - run: name: Install poetry dependencies command: | cd ~\project\desktop - poetry install + C:\Python39-32bit\Scripts\poetry install - run: name: Get tor command: | cd ~\project\desktop - poetry run python .\scripts\get-tor-windows.py + C:\Python39-32bit\Scripts\poetry run python .\scripts\get-tor-windows.py - run: name: Build meek command: | cd ~\project\desktop - python .\scripts\build-meek-client.py + C:\Python39-32bit\python .\scripts\build-meek-client.py - run: name: Build command: | cd ~\project\desktop - poetry run python .\package\build-windows.py --ci-build + C:\Python39-32bit\Scripts\poetry run python .\package\build-windows.py --ci-build - run: name: Compress command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win32-3.9 -DestinationPath ~\onionshare-win32.zip diff --git a/desktop/package/build-windows.py b/desktop/package/build-windows.py index 61ad692e..d9e78f95 100644 --- a/desktop/package/build-windows.py +++ b/desktop/package/build-windows.py @@ -185,7 +185,7 @@ def main(): print("> Building binaries") run( [ - shutil.which("python"), + "python", "setup-freeze.py", "build", ], From e77cd6939e02f602bc60a3b5d4e62bc6f6d073d7 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 30 Mar 2022 15:52:31 -0700 Subject: [PATCH 013/154] Try fixing installing 32-bit go, and try fixing 32-bit paths in scripts --- .circleci/config.yml | 18 +++++++++--------- desktop/package/build-windows.py | 18 +++++++++++++----- desktop/scripts/build-meek-client.py | 14 +++++++++++++- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea3c3c0f..f6114200 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,7 +83,6 @@ jobs: command: | choco install python --version=3.9.12 choco install 7zip - choco install go - run: name: Install poetry command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - @@ -123,36 +122,37 @@ jobs: - run: name: Install chocolately dependencies command: | - choco install python3 --params "/InstallDir32:C:\Python39-32bit" --version=3.9.12 + choco install python3 --params "/InstallDir32:C:\Python-32bit" --version=3.9.12 choco install 7zip - run: name: Install golang (32-bit) command: | - Invoke-WebRequest -Uri https://go.dev/dl/go1.18.windows-386.msi -OutFile ~\Downloads\go1.18.windows-386.msi - msiexec.exe /i "~\Downloads\go1.18.windows-386.msi" /qn + cd ~\Downloads + Invoke-WebRequest -Uri https://go.dev/dl/go1.18.windows-386.msi -OutFile go1.18.windows-386.msi + msiexec.exe /i go1.18.windows-386.msi /quiet /L*V go-install.log - run: name: Install poetry (32-bit) - command: C:\Python39-32bit\Scripts\pip install poetry + command: C:\Python-32bit\Scripts\pip install poetry - run: name: Install poetry dependencies command: | cd ~\project\desktop - C:\Python39-32bit\Scripts\poetry install + C:\Python-32bit\Scripts\poetry install - run: name: Get tor command: | cd ~\project\desktop - C:\Python39-32bit\Scripts\poetry run python .\scripts\get-tor-windows.py + C:\Python-32bit\Scripts\poetry run python .\scripts\get-tor-windows.py - run: name: Build meek command: | cd ~\project\desktop - C:\Python39-32bit\python .\scripts\build-meek-client.py + C:\Python-32bit\python .\scripts\build-meek-client.py - run: name: Build command: | cd ~\project\desktop - C:\Python39-32bit\Scripts\poetry run python .\package\build-windows.py --ci-build + C:\Python-32bit\Scripts\poetry run python .\package\build-windows.py --ci-build - run: name: Compress command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win32-3.9 -DestinationPath ~\onionshare-win32.zip diff --git a/desktop/package/build-windows.py b/desktop/package/build-windows.py index d9e78f95..1cce529f 100644 --- a/desktop/package/build-windows.py +++ b/desktop/package/build-windows.py @@ -162,6 +162,18 @@ def wix_build_components_xml(root, data): def main(): + # Figure out the architecture and python path + if "64 bit" in sys.version: + python_arch = "win-amd64" + else: + python_arch = "win32" + + if os.getlogin() == "circleci" and python_arch == "win32": + python_path = "C:\\Python-32bit\\python" + else: + python_path = shutil.which("python") + + # Arguments parser = argparse.ArgumentParser( formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=48) ) @@ -185,17 +197,13 @@ def main(): print("> Building binaries") run( [ - "python", + python_path, "setup-freeze.py", "build", ], desktop_dir, ) - if "64 bit" in sys.version: - python_arch = "win-amd64" - else: - python_arch = "win32" build_path = os.path.join(desktop_dir, "build", f"exe.{python_arch}-3.9") # before_size = get_size(build_path) diff --git a/desktop/scripts/build-meek-client.py b/desktop/scripts/build-meek-client.py index 3d81d62f..a33e314b 100755 --- a/desktop/scripts/build-meek-client.py +++ b/desktop/scripts/build-meek-client.py @@ -26,13 +26,25 @@ and hard-code the sha256 hash. """ import shutil import os +import sys import subprocess import inspect import platform def main(): - if shutil.which("go") is None: + # Figure out the architecture and python path + if "64 bit" in sys.version: + python_arch = "win-amd64" + else: + python_arch = "win32" + + if os.getlogin() == "circleci" and python_arch == "win32": + go_path = "C:\\Program Files (x86)\\Go\\bin\\go" + else: + go_path = shutil.which("go") + + if go_path is None: print("Install go: https://golang.org/doc/install") return From aff969ee09948530b5ae36a336102abfd01d46db Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 30 Mar 2022 16:09:31 -0700 Subject: [PATCH 014/154] Run setup-freeze.py directly instead of using build-windows.py, and fix go path in build meek script --- .circleci/config.yml | 2 +- desktop/scripts/build-meek-client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f6114200..b92236e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -152,7 +152,7 @@ jobs: name: Build command: | cd ~\project\desktop - C:\Python-32bit\Scripts\poetry run python .\package\build-windows.py --ci-build + C:\Python-32bit\Scripts\poetry run python .\setup-freeze.py build - run: name: Compress command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win32-3.9 -DestinationPath ~\onionshare-win32.zip diff --git a/desktop/scripts/build-meek-client.py b/desktop/scripts/build-meek-client.py index a33e314b..6e5d7974 100755 --- a/desktop/scripts/build-meek-client.py +++ b/desktop/scripts/build-meek-client.py @@ -50,7 +50,7 @@ def main(): subprocess.run( [ - "go", + go_path, "install", "git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0", ] From 9079dee4e31015f093bab9ab63e1c2f8646eabfe Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 11:57:15 -0700 Subject: [PATCH 015/154] Explicitly exclude top-level folders from cx_Freeze build --- desktop/setup-freeze.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/desktop/setup-freeze.py b/desktop/setup-freeze.py index 76ff6061..3bedb1a3 100644 --- a/desktop/setup-freeze.py +++ b/desktop/setup-freeze.py @@ -166,6 +166,9 @@ setup( "PySide2.QtWidgets", ], "excludes": [ + "package", + "screenshots", + "scripts", "test", "tkinter", "PySide2.Qt3DAnimation", From 686ea63580fc617b972689becebeea2f8ce3031c Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 12:31:48 -0700 Subject: [PATCH 016/154] Try a different way of explicitly defining what packages to include and exclude --- desktop/setup-freeze.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/desktop/setup-freeze.py b/desktop/setup-freeze.py index 3bedb1a3..ccb5a541 100644 --- a/desktop/setup-freeze.py +++ b/desktop/setup-freeze.py @@ -24,6 +24,7 @@ import platform import shutil import cx_Freeze from cx_Freeze import setup, Executable +from setuptools import find_packages # There's an obscure cx_Freeze bug that I'm hitting that's preventing the macOS # package from getting built. This is some monkeypatching to fix it. @@ -148,6 +149,11 @@ setup( name="onionshare", version=version, description="Securely and anonymously share files, host websites, and chat with friends using the Tor network", + packages=find_packages( + where=".", + include=["onionshare"], + exclude=["package", "screenshots", "scripts", "tests"], + ), options={ # build_exe, for Windows and macOS "build_exe": { @@ -166,9 +172,6 @@ setup( "PySide2.QtWidgets", ], "excludes": [ - "package", - "screenshots", - "scripts", "test", "tkinter", "PySide2.Qt3DAnimation", @@ -217,7 +220,7 @@ setup( # bdist_mac, making the macOS app bundle "bdist_mac": { "iconfile": os.path.join("onionshare", "resources", "onionshare.icns"), - "bundle_name": "OnionShare" + "bundle_name": "OnionShare", }, }, executables=[ From 4297a5a6dc3dc0f3e3f21c1056f8ca86598d8939 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 18:34:41 -0700 Subject: [PATCH 017/154] Create new windows.py script that combines various Windows build tasks, and update CI to use it --- .circleci/config.yml | 8 +- RELEASE.md | 7 +- desktop/package/build-windows.py | 792 +++++++++++++++---------------- desktop/package/windows.py | 605 +++++++++++++++++++++++ desktop/poetry.lock | 128 +++-- desktop/pyproject.toml | 1 + 6 files changed, 1064 insertions(+), 477 deletions(-) create mode 100644 desktop/package/windows.py diff --git a/.circleci/config.yml b/.circleci/config.yml index b92236e3..4673767e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,10 +102,11 @@ jobs: cd ~\project\desktop python .\scripts\build-meek-client.py - run: - name: Build + name: Build OnionShare command: | cd ~\project\desktop - poetry run python .\package\build-windows.py --ci-build + poetry run python .\setup-freeze.py build + poetry run python .\package\windows.py cleanup-build - run: name: Compress command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win-amd64-3.9 -DestinationPath ~\onionshare-win64.zip @@ -149,10 +150,11 @@ jobs: cd ~\project\desktop C:\Python-32bit\python .\scripts\build-meek-client.py - run: - name: Build + name: Build OnionShare command: | cd ~\project\desktop C:\Python-32bit\Scripts\poetry run python .\setup-freeze.py build + C:\Python-32bit\Scripts\poetry run python .\package\windows.py cleanup-build - run: name: Compress command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win32-3.9 -DestinationPath ~\onionshare-win32.zip diff --git a/RELEASE.md b/RELEASE.md index 76c33a4b..958275cc 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -106,10 +106,13 @@ Set up the development environment described in desktop `README.md`. - Go to https://dotnet.microsoft.com/download/dotnet-framework and download and install .NET Framework 3.5 SP1 Runtime. I downloaded `dotnetfx35.exe`. - Go to https://wixtoolset.org/releases/ and download and install WiX toolset. I downloaded `wix311.exe`. Add `C:\Program Files (x86)\WiX Toolset v3.11\bin` to the path. -Run the Windows build script: +Build the Windows binaries, delete extra files, codesign, and create an MSI package: ``` -poetry run python .\package\build-windows.py +poetry run python .\setup-freeze.py build +poetry run python .\package\windows.py cleanup-build +poetry run python .\package\windows.py codesign +poetry run python .\package\windows.py package ``` This will create `desktop/dist/OnionShare-$VERSION.msi`, signed. diff --git a/desktop/package/build-windows.py b/desktop/package/build-windows.py index 1cce529f..eb7d0218 100644 --- a/desktop/package/build-windows.py +++ b/desktop/package/build-windows.py @@ -5,7 +5,6 @@ import inspect import subprocess import shutil import uuid -import argparse import xml.etree.ElementTree as ET @@ -173,19 +172,6 @@ def main(): else: python_path = shutil.which("python") - # Arguments - parser = argparse.ArgumentParser( - formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=48) - ) - parser.add_argument( - "--ci-build", - action="store_true", - dest="ci_build", - help="Build in CI, don't code sign", - ) - args = parser.parse_args() - ci_build = bool(args.ci_build) - desktop_dir = os.path.join(root, "desktop") print("> Clean up from last build") @@ -206,408 +192,404 @@ def main(): build_path = os.path.join(desktop_dir, "build", f"exe.{python_arch}-3.9") - # before_size = get_size(build_path) + before_size = get_size(build_path) - # print("> Delete unused PySide2 stuff to save space") - # for dirname in ["examples", "qml"]: - # shutil.rmtree(os.path.join(build_path, "lib", "PySide2", dirname)) - # for filename in [ - # "lconvert.exe", - # "linguist.exe", - # "lrelease.exe", - # "lupdate.exe", - # "plugins/assetimporters/assimp.dll", - # "plugins/assetimporters/uip.dll", - # "plugins/audio/qtaudio_wasapi.dll", - # "plugins/audio/qtaudio_windows.dll", - # "plugins/bearer/qgenericbearer.dll", - # "plugins/canbus/qtpassthrucanbus.dll", - # "plugins/canbus/qtpeakcanbus.dll", - # "plugins/canbus/qtsysteccanbus.dll", - # "plugins/canbus/qttinycanbus.dll", - # "plugins/canbus/qtvectorcanbus.dll", - # "plugins/canbus/qtvirtualcanbus.dll", - # "plugins/gamepads/xinputgamepad.dll", - # "plugins/generic/qtuiotouchplugin.dll", - # "plugins/geometryloaders/defaultgeometryloader.dll", - # "plugins/geometryloaders/gltfgeometryloader.dll", - # "plugins/geoservices/qtgeoservices_esri.dll", - # "plugins/geoservices/qtgeoservices_itemsoverlay.dll", - # "plugins/geoservices/qtgeoservices_mapbox.dll", - # "plugins/geoservices/qtgeoservices_nokia.dll", - # "plugins/geoservices/qtgeoservices_osm.dll", - # "plugins/mediaservice/dsengine.dll", - # "plugins/mediaservice/qtmedia_audioengine.dll", - # "plugins/mediaservice/wmfengine.dll", - # "plugins/platforminputcontexts/qtvirtualkeyboardplugin.dll", - # "plugins/platforms/qdirect2d.dll", - # "plugins/platforms/qoffscreen.dll", - # "plugins/platforms/qwebgl.dll", - # "plugins/platformthemes/qxdgdesktopportal.dll", - # "plugins/playlistformats/qtmultimedia_m3u.dll", - # "plugins/position/qtposition_positionpoll.dll", - # "plugins/position/qtposition_serialnmea.dll", - # "plugins/position/qtposition_winrt.dll", - # "plugins/printsupport/windowsprintersupport.dll", - # "plugins/qmltooling/qmldbg_debugger.dll", - # "plugins/qmltooling/qmldbg_inspector.dll", - # "plugins/qmltooling/qmldbg_local.dll", - # "plugins/qmltooling/qmldbg_messages.dll", - # "plugins/qmltooling/qmldbg_native.dll", - # "plugins/qmltooling/qmldbg_nativedebugger.dll", - # "plugins/qmltooling/qmldbg_preview.dll", - # "plugins/qmltooling/qmldbg_profiler.dll", - # "plugins/qmltooling/qmldbg_quickprofiler.dll", - # "plugins/qmltooling/qmldbg_server.dll", - # "plugins/qmltooling/qmldbg_tcp.dll", - # "plugins/renderers/openglrenderer.dll", - # "plugins/renderplugins/scene2d.dll", - # "plugins/scenegraph/qsgd3d12backend.dll", - # "plugins/sceneparsers/gltfsceneexport.dll", - # "plugins/sceneparsers/gltfsceneimport.dll", - # "plugins/sensorgestures/qtsensorgestures_plugin.dll", - # "plugins/sensorgestures/qtsensorgestures_shakeplugin.dll", - # "plugins/sensors/qtsensors_generic.dll", - # "plugins/sqldrivers/qsqlite.dll", - # "plugins/sqldrivers/qsqlodbc.dll", - # "plugins/sqldrivers/qsqlpsql.dll", - # "plugins/styles/qwindowsvistastyle.dll", - # "plugins/texttospeech/qtexttospeech_sapi.dll", - # "plugins/virtualkeyboard/qtvirtualkeyboard_hangul.dll", - # "plugins/virtualkeyboard/qtvirtualkeyboard_openwnn.dll", - # "plugins/virtualkeyboard/qtvirtualkeyboard_pinyin.dll", - # "plugins/virtualkeyboard/qtvirtualkeyboard_tcime.dll", - # "plugins/virtualkeyboard/qtvirtualkeyboard_thai.dll", - # "plugins/webview/qtwebview_webengine.dll", - # "pyside2-lupdate.exe", - # "Qt3DAnimation.pyd", - # "Qt3DAnimation.pyi", - # "Qt3DCore.pyd", - # "Qt3DCore.pyi", - # "Qt3DExtras.pyd", - # "Qt3DExtras.pyi", - # "Qt3DInput.pyd", - # "Qt3DInput.pyi", - # "Qt3DLogic.pyd", - # "Qt3DLogic.pyi", - # "Qt3DRender.pyd", - # "Qt3DRender.pyi", - # "Qt53DAnimation.dll", - # "Qt53DCore.dll", - # "Qt53DExtras.dll", - # "Qt53DInput.dll", - # "Qt53DLogic.dll", - # "Qt53DQuick.dll", - # "Qt53DQuickAnimation.dll", - # "Qt53DQuickExtras.dll", - # "Qt53DQuickInput.dll", - # "Qt53DQuickRender.dll", - # "Qt53DQuickScene2D.dll", - # "Qt53DRender.dll", - # "Qt5Bluetooth.dll", - # "Qt5Bodymovin.dll", - # "Qt5Charts.dll", - # "Qt5Concurrent.dll", - # "Qt5DataVisualization.dll", - # "Qt5DBus.dll", - # "Qt5Designer.dll", - # "Qt5DesignerComponents.dll", - # "Qt5Gamepad.dll", - # "Qt5Help.dll", - # "Qt5Location.dll", - # "Qt5Multimedia.dll", - # "Qt5MultimediaQuick.dll", - # "Qt5MultimediaWidgets.dll", - # "Qt5Nfc.dll", - # "Qt5OpenGL.dll", - # "Qt5Pdf.dll", - # "Qt5PdfWidgets.dll", - # "Qt5Positioning.dll", - # "Qt5PositioningQuick.dll", - # "Qt5PrintSupport.dll", - # "Qt5Purchasing.dll", - # "Qt5Quick.dll", - # "Qt5Quick3D.dll", - # "Qt5Quick3DAssetImport.dll", - # "Qt5Quick3DRender.dll", - # "Qt5Quick3DRuntimeRender.dll", - # "Qt5Quick3DUtils.dll", - # "Qt5QuickControls2.dll", - # "Qt5QuickParticles.dll", - # "Qt5QuickShapes.dll", - # "Qt5QuickTemplates2.dll", - # "Qt5QuickTest.dll", - # "Qt5QuickWidgets.dll", - # "Qt5RemoteObjects.dll", - # "Qt5Script.dll", - # "Qt5ScriptTools.dll", - # "Qt5Scxml.dll", - # "Qt5Sensors.dll", - # "Qt5SerialBus.dll", - # "Qt5SerialPort.dll", - # "Qt5Sql.dll", - # "Qt5Svg.dll", - # "Qt5Test.dll", - # "Qt5TextToSpeech.dll", - # "Qt5VirtualKeyboard.dll", - # "Qt5WebChannel.dll", - # "Qt5WebEngine.dll", - # "Qt5WebEngineCore.dll", - # "Qt5WebEngineWidgets.dll", - # "Qt5WebSockets.dll", - # "Qt5WebView.dll", - # "Qt5Xml.dll", - # "Qt5XmlPatterns.dll", - # "QtAxContainer.pyd", - # "QtAxContainer.pyi", - # "QtCharts.pyd", - # "QtCharts.pyi", - # "QtConcurrent.pyd", - # "QtConcurrent.pyi", - # "QtDataVisualization.pyd", - # "QtDataVisualization.pyi", - # "qtdiag.exe", - # "QtHelp.pyd", - # "QtHelp.pyi", - # "QtLocation.pyd", - # "QtLocation.pyi", - # "QtMultimedia.pyd", - # "QtMultimedia.pyi", - # "QtMultimediaWidgets.pyd", - # "QtMultimediaWidgets.pyi", - # "QtNetwork.pyd", - # "QtNetwork.pyi", - # "QtOpenGL.pyd", - # "QtOpenGL.pyi", - # "QtOpenGLFunctions.pyd", - # "QtOpenGLFunctions.pyi", - # "QtPositioning.pyd", - # "QtPositioning.pyi", - # "QtPrintSupport.pyd", - # "QtPrintSupport.pyi", - # "QtQml.pyd", - # "QtQml.pyi", - # "QtQuick.pyd", - # "QtQuick.pyi", - # "QtQuickControls2.pyd", - # "QtQuickControls2.pyi", - # "QtQuickWidgets.pyd", - # "QtQuickWidgets.pyi", - # "QtRemoteObjects.pyd", - # "QtRemoteObjects.pyi", - # "QtScript.pyd", - # "QtScript.pyi", - # "QtScriptTools.pyd", - # "QtScriptTools.pyi", - # "QtScxml.pyd", - # "QtScxml.pyi", - # "QtSensors.pyd", - # "QtSensors.pyi", - # "QtSerialPort.pyd", - # "QtSerialPort.pyi", - # "QtSql.pyd", - # "QtSql.pyi", - # "QtSvg.pyd", - # "QtSvg.pyi", - # "QtTest.pyd", - # "QtTest.pyi", - # "QtTextToSpeech.pyd", - # "QtTextToSpeech.pyi", - # "QtUiTools.pyd", - # "QtUiTools.pyi", - # "QtWebChannel.pyd", - # "QtWebChannel.pyi", - # "QtWebEngine.pyd", - # "QtWebEngine.pyi", - # "QtWebEngineCore.pyd", - # "QtWebEngineCore.pyi", - # "QtWebEngineProcess.exe", - # "QtWebEngineWidgets.pyd", - # "QtWebEngineWidgets.pyi", - # "QtWebSockets.pyd", - # "QtWebSockets.pyi", - # "QtWinExtras.pyd", - # "QtWinExtras.pyi", - # "QtXml.pyd", - # "QtXml.pyi", - # "QtXmlPatterns.pyd", - # "QtXmlPatterns.pyi", - # "rcc.exe", - # "uic.exe", - # ]: - # os.remove( - # os.path.join( - # build_path, - # "lib", - # "PySide2", - # filename.replace("/", "\\"), - # ) - # ) - - # after_size = get_size(build_path) - # freed_bytes = before_size - after_size - # freed_mb = int(freed_bytes / 1024 / 1024) - # print(f"> Freed {freed_mb} mb") - - if ci_build: - print("Doing a CI build, skipping code signing and msi packaging") - - else: - print(f"> Signing onionshare.exe") - sign(os.path.join(build_path, "onionshare.exe"), desktop_dir) - - print(f"> Signing onionshare-cli.exe") - sign(os.path.join(build_path, "onionshare-cli.exe"), desktop_dir) - - print(f"> Build the WiX file") - version_filename = os.path.join( - root, "cli", "onionshare_cli", "resources", "version.txt" - ) - with open(version_filename) as f: - version = f.read().strip() - - dist_dir = os.path.join(root, build_path) - - data = { - "id": "TARGETDIR", - "name": "SourceDir", - "dirs": [ - { - "id": "ProgramFilesFolder", - "dirs": [], - }, - { - "id": "ProgramMenuFolder", - "dirs": [], - }, - ], - } - - data["dirs"][0]["dirs"].append( - wix_build_data( + print("> Delete unused PySide2 stuff to save space") + for dirname in ["examples", "qml"]: + shutil.rmtree(os.path.join(build_path, "lib", "PySide2", dirname)) + for filename in [ + "lconvert.exe", + "linguist.exe", + "lrelease.exe", + "lupdate.exe", + "plugins/assetimporters/assimp.dll", + "plugins/assetimporters/uip.dll", + "plugins/audio/qtaudio_wasapi.dll", + "plugins/audio/qtaudio_windows.dll", + "plugins/bearer/qgenericbearer.dll", + "plugins/canbus/qtpassthrucanbus.dll", + "plugins/canbus/qtpeakcanbus.dll", + "plugins/canbus/qtsysteccanbus.dll", + "plugins/canbus/qttinycanbus.dll", + "plugins/canbus/qtvectorcanbus.dll", + "plugins/canbus/qtvirtualcanbus.dll", + "plugins/gamepads/xinputgamepad.dll", + "plugins/generic/qtuiotouchplugin.dll", + "plugins/geometryloaders/defaultgeometryloader.dll", + "plugins/geometryloaders/gltfgeometryloader.dll", + "plugins/geoservices/qtgeoservices_esri.dll", + "plugins/geoservices/qtgeoservices_itemsoverlay.dll", + "plugins/geoservices/qtgeoservices_mapbox.dll", + "plugins/geoservices/qtgeoservices_nokia.dll", + "plugins/geoservices/qtgeoservices_osm.dll", + "plugins/mediaservice/dsengine.dll", + "plugins/mediaservice/qtmedia_audioengine.dll", + "plugins/mediaservice/wmfengine.dll", + "plugins/platforminputcontexts/qtvirtualkeyboardplugin.dll", + "plugins/platforms/qdirect2d.dll", + "plugins/platforms/qoffscreen.dll", + "plugins/platforms/qwebgl.dll", + "plugins/platformthemes/qxdgdesktopportal.dll", + "plugins/playlistformats/qtmultimedia_m3u.dll", + "plugins/position/qtposition_positionpoll.dll", + "plugins/position/qtposition_serialnmea.dll", + "plugins/position/qtposition_winrt.dll", + "plugins/printsupport/windowsprintersupport.dll", + "plugins/qmltooling/qmldbg_debugger.dll", + "plugins/qmltooling/qmldbg_inspector.dll", + "plugins/qmltooling/qmldbg_local.dll", + "plugins/qmltooling/qmldbg_messages.dll", + "plugins/qmltooling/qmldbg_native.dll", + "plugins/qmltooling/qmldbg_nativedebugger.dll", + "plugins/qmltooling/qmldbg_preview.dll", + "plugins/qmltooling/qmldbg_profiler.dll", + "plugins/qmltooling/qmldbg_quickprofiler.dll", + "plugins/qmltooling/qmldbg_server.dll", + "plugins/qmltooling/qmldbg_tcp.dll", + "plugins/renderers/openglrenderer.dll", + "plugins/renderplugins/scene2d.dll", + "plugins/scenegraph/qsgd3d12backend.dll", + "plugins/sceneparsers/gltfsceneexport.dll", + "plugins/sceneparsers/gltfsceneimport.dll", + "plugins/sensorgestures/qtsensorgestures_plugin.dll", + "plugins/sensorgestures/qtsensorgestures_shakeplugin.dll", + "plugins/sensors/qtsensors_generic.dll", + "plugins/sqldrivers/qsqlite.dll", + "plugins/sqldrivers/qsqlodbc.dll", + "plugins/sqldrivers/qsqlpsql.dll", + "plugins/styles/qwindowsvistastyle.dll", + "plugins/texttospeech/qtexttospeech_sapi.dll", + "plugins/virtualkeyboard/qtvirtualkeyboard_hangul.dll", + "plugins/virtualkeyboard/qtvirtualkeyboard_openwnn.dll", + "plugins/virtualkeyboard/qtvirtualkeyboard_pinyin.dll", + "plugins/virtualkeyboard/qtvirtualkeyboard_tcime.dll", + "plugins/virtualkeyboard/qtvirtualkeyboard_thai.dll", + "plugins/webview/qtwebview_webengine.dll", + "pyside2-lupdate.exe", + "Qt3DAnimation.pyd", + "Qt3DAnimation.pyi", + "Qt3DCore.pyd", + "Qt3DCore.pyi", + "Qt3DExtras.pyd", + "Qt3DExtras.pyi", + "Qt3DInput.pyd", + "Qt3DInput.pyi", + "Qt3DLogic.pyd", + "Qt3DLogic.pyi", + "Qt3DRender.pyd", + "Qt3DRender.pyi", + "Qt53DAnimation.dll", + "Qt53DCore.dll", + "Qt53DExtras.dll", + "Qt53DInput.dll", + "Qt53DLogic.dll", + "Qt53DQuick.dll", + "Qt53DQuickAnimation.dll", + "Qt53DQuickExtras.dll", + "Qt53DQuickInput.dll", + "Qt53DQuickRender.dll", + "Qt53DQuickScene2D.dll", + "Qt53DRender.dll", + "Qt5Bluetooth.dll", + "Qt5Bodymovin.dll", + "Qt5Charts.dll", + "Qt5Concurrent.dll", + "Qt5DataVisualization.dll", + "Qt5DBus.dll", + "Qt5Designer.dll", + "Qt5DesignerComponents.dll", + "Qt5Gamepad.dll", + "Qt5Help.dll", + "Qt5Location.dll", + "Qt5Multimedia.dll", + "Qt5MultimediaQuick.dll", + "Qt5MultimediaWidgets.dll", + "Qt5Nfc.dll", + "Qt5OpenGL.dll", + "Qt5Pdf.dll", + "Qt5PdfWidgets.dll", + "Qt5Positioning.dll", + "Qt5PositioningQuick.dll", + "Qt5PrintSupport.dll", + "Qt5Purchasing.dll", + "Qt5Quick.dll", + "Qt5Quick3D.dll", + "Qt5Quick3DAssetImport.dll", + "Qt5Quick3DRender.dll", + "Qt5Quick3DRuntimeRender.dll", + "Qt5Quick3DUtils.dll", + "Qt5QuickControls2.dll", + "Qt5QuickParticles.dll", + "Qt5QuickShapes.dll", + "Qt5QuickTemplates2.dll", + "Qt5QuickTest.dll", + "Qt5QuickWidgets.dll", + "Qt5RemoteObjects.dll", + "Qt5Script.dll", + "Qt5ScriptTools.dll", + "Qt5Scxml.dll", + "Qt5Sensors.dll", + "Qt5SerialBus.dll", + "Qt5SerialPort.dll", + "Qt5Sql.dll", + "Qt5Svg.dll", + "Qt5Test.dll", + "Qt5TextToSpeech.dll", + "Qt5VirtualKeyboard.dll", + "Qt5WebChannel.dll", + "Qt5WebEngine.dll", + "Qt5WebEngineCore.dll", + "Qt5WebEngineWidgets.dll", + "Qt5WebSockets.dll", + "Qt5WebView.dll", + "Qt5Xml.dll", + "Qt5XmlPatterns.dll", + "QtAxContainer.pyd", + "QtAxContainer.pyi", + "QtCharts.pyd", + "QtCharts.pyi", + "QtConcurrent.pyd", + "QtConcurrent.pyi", + "QtDataVisualization.pyd", + "QtDataVisualization.pyi", + "qtdiag.exe", + "QtHelp.pyd", + "QtHelp.pyi", + "QtLocation.pyd", + "QtLocation.pyi", + "QtMultimedia.pyd", + "QtMultimedia.pyi", + "QtMultimediaWidgets.pyd", + "QtMultimediaWidgets.pyi", + "QtNetwork.pyd", + "QtNetwork.pyi", + "QtOpenGL.pyd", + "QtOpenGL.pyi", + "QtOpenGLFunctions.pyd", + "QtOpenGLFunctions.pyi", + "QtPositioning.pyd", + "QtPositioning.pyi", + "QtPrintSupport.pyd", + "QtPrintSupport.pyi", + "QtQml.pyd", + "QtQml.pyi", + "QtQuick.pyd", + "QtQuick.pyi", + "QtQuickControls2.pyd", + "QtQuickControls2.pyi", + "QtQuickWidgets.pyd", + "QtQuickWidgets.pyi", + "QtRemoteObjects.pyd", + "QtRemoteObjects.pyi", + "QtScript.pyd", + "QtScript.pyi", + "QtScriptTools.pyd", + "QtScriptTools.pyi", + "QtScxml.pyd", + "QtScxml.pyi", + "QtSensors.pyd", + "QtSensors.pyi", + "QtSerialPort.pyd", + "QtSerialPort.pyi", + "QtSql.pyd", + "QtSql.pyi", + "QtSvg.pyd", + "QtSvg.pyi", + "QtTest.pyd", + "QtTest.pyi", + "QtTextToSpeech.pyd", + "QtTextToSpeech.pyi", + "QtUiTools.pyd", + "QtUiTools.pyi", + "QtWebChannel.pyd", + "QtWebChannel.pyi", + "QtWebEngine.pyd", + "QtWebEngine.pyi", + "QtWebEngineCore.pyd", + "QtWebEngineCore.pyi", + "QtWebEngineProcess.exe", + "QtWebEngineWidgets.pyd", + "QtWebEngineWidgets.pyi", + "QtWebSockets.pyd", + "QtWebSockets.pyi", + "QtWinExtras.pyd", + "QtWinExtras.pyi", + "QtXml.pyd", + "QtXml.pyi", + "QtXmlPatterns.pyd", + "QtXmlPatterns.pyi", + "rcc.exe", + "uic.exe", + ]: + os.remove( + os.path.join( build_path, - "INSTALLDIR", - "OnionShare", + "lib", + "PySide2", + filename.replace("/", "\\"), ) ) - root_el = ET.Element("Wix", xmlns="http://schemas.microsoft.com/wix/2006/wi") - product_el = ET.SubElement( - root_el, - "Product", - Name="OnionShare", - Manufacturer="Micah Lee, et al.", - Id="*", - UpgradeCode="$(var.ProductUpgradeCode)", - Language="1033", - Codepage="1252", - Version="$(var.ProductVersion)", + after_size = get_size(build_path) + freed_bytes = before_size - after_size + freed_mb = int(freed_bytes / 1024 / 1024) + print(f"> Freed {freed_mb} mb") + + print(f"> Signing onionshare.exe") + sign(os.path.join(build_path, "onionshare.exe"), desktop_dir) + + print(f"> Signing onionshare-cli.exe") + sign(os.path.join(build_path, "onionshare-cli.exe"), desktop_dir) + + print(f"> Build the WiX file") + version_filename = os.path.join( + root, "cli", "onionshare_cli", "resources", "version.txt" + ) + with open(version_filename) as f: + version = f.read().strip() + + dist_dir = os.path.join(root, build_path) + + data = { + "id": "TARGETDIR", + "name": "SourceDir", + "dirs": [ + { + "id": "ProgramFilesFolder", + "dirs": [], + }, + { + "id": "ProgramMenuFolder", + "dirs": [], + }, + ], + } + + data["dirs"][0]["dirs"].append( + wix_build_data( + build_path, + "INSTALLDIR", + "OnionShare", ) - ET.SubElement( - product_el, - "Package", - Id="*", - Keywords="Installer", - Description="OnionShare $(var.ProductVersion) Installer", - Manufacturer="Micah Lee, et al.", - InstallerVersion="100", - Languages="1033", - Compressed="yes", - SummaryCodepage="1252", - ) - ET.SubElement( - product_el, "Media", Id="1", Cabinet="product.cab", EmbedCab="yes" - ) - ET.SubElement( - product_el, - "Icon", - Id="ProductIcon", - SourceFile="..\\onionshare\\resources\\onionshare.ico", - ) - ET.SubElement(product_el, "Property", Id="ARPPRODUCTICON", Value="ProductIcon") - ET.SubElement( - product_el, - "Property", - Id="ARPHELPLINK", - Value="https://docs.onionshare.org", - ) - ET.SubElement( - product_el, - "Property", - Id="ARPURLINFOABOUT", - Value="https://onionshare.org", - ) - ET.SubElement(product_el, "UIRef", Id="WixUI_Minimal") - ET.SubElement(product_el, "UIRef", Id="WixUI_ErrorProgressText") - ET.SubElement( - product_el, - "WixVariable", - Id="WixUILicenseRtf", - Value="..\\package\\license.rtf", - ) - ET.SubElement( - product_el, - "WixVariable", - Id="WixUIDialogBmp", - Value="..\\package\\dialog.bmp", - ) - ET.SubElement( - product_el, - "MajorUpgrade", - AllowSameVersionUpgrades="yes", - DowngradeErrorMessage="A newer version of [ProductName] is already installed. If you are sure you want to downgrade, remove the existing installation via Programs and Features.", + ) + + root_el = ET.Element("Wix", xmlns="http://schemas.microsoft.com/wix/2006/wi") + product_el = ET.SubElement( + root_el, + "Product", + Name="OnionShare", + Manufacturer="Micah Lee, et al.", + Id="*", + UpgradeCode="$(var.ProductUpgradeCode)", + Language="1033", + Codepage="1252", + Version="$(var.ProductVersion)", + ) + ET.SubElement( + product_el, + "Package", + Id="*", + Keywords="Installer", + Description="OnionShare $(var.ProductVersion) Installer", + Manufacturer="Micah Lee, et al.", + InstallerVersion="100", + Languages="1033", + Compressed="yes", + SummaryCodepage="1252", + ) + ET.SubElement( + product_el, "Media", Id="1", Cabinet="product.cab", EmbedCab="yes" + ) + ET.SubElement( + product_el, + "Icon", + Id="ProductIcon", + SourceFile="..\\onionshare\\resources\\onionshare.ico", + ) + ET.SubElement(product_el, "Property", Id="ARPPRODUCTICON", Value="ProductIcon") + ET.SubElement( + product_el, + "Property", + Id="ARPHELPLINK", + Value="https://docs.onionshare.org", + ) + ET.SubElement( + product_el, + "Property", + Id="ARPURLINFOABOUT", + Value="https://onionshare.org", + ) + ET.SubElement(product_el, "UIRef", Id="WixUI_Minimal") + ET.SubElement(product_el, "UIRef", Id="WixUI_ErrorProgressText") + ET.SubElement( + product_el, + "WixVariable", + Id="WixUILicenseRtf", + Value="..\\package\\license.rtf", + ) + ET.SubElement( + product_el, + "WixVariable", + Id="WixUIDialogBmp", + Value="..\\package\\dialog.bmp", + ) + ET.SubElement( + product_el, + "MajorUpgrade", + AllowSameVersionUpgrades="yes", + DowngradeErrorMessage="A newer version of [ProductName] is already installed. If you are sure you want to downgrade, remove the existing installation via Programs and Features.", + ) + + wix_build_dir_xml(product_el, data) + component_ids = wix_build_components_xml(product_el, data) + + feature_el = ET.SubElement( + product_el, "Feature", Id="DefaultFeature", Level="1" + ) + for component_id in component_ids: + ET.SubElement(feature_el, "ComponentRef", Id=component_id) + ET.SubElement(feature_el, "ComponentRef", Id="ApplicationShortcuts") + + with open(os.path.join(root, "desktop", "build", "OnionShare.wxs"), "w") as f: + f.write('\n') + f.write(f'\n') + f.write( + '\n' ) - wix_build_dir_xml(product_el, data) - component_ids = wix_build_components_xml(product_el, data) + ET.indent(root_el) + f.write(ET.tostring(root_el).decode()) - feature_el = ET.SubElement( - product_el, "Feature", Id="DefaultFeature", Level="1" - ) - for component_id in component_ids: - ET.SubElement(feature_el, "ComponentRef", Id=component_id) - ET.SubElement(feature_el, "ComponentRef", Id="ApplicationShortcuts") + print(f"> Build the MSI") + run( + [shutil.which("candle.exe"), "OnionShare.wxs"], + os.path.join(desktop_dir, "build"), + ) + run( + [shutil.which("light.exe"), "-ext", "WixUIExtension", "OnionShare.wixobj"], + os.path.join(desktop_dir, "build"), + ) - with open(os.path.join(root, "desktop", "build", "OnionShare.wxs"), "w") as f: - f.write('\n') - f.write(f'\n') - f.write( - '\n' - ) - - ET.indent(root_el) - f.write(ET.tostring(root_el).decode()) - - print(f"> Build the MSI") - run( - [shutil.which("candle.exe"), "OnionShare.wxs"], - os.path.join(desktop_dir, "build"), - ) - run( - [shutil.which("light.exe"), "-ext", "WixUIExtension", "OnionShare.wixobj"], - os.path.join(desktop_dir, "build"), - ) - - print(f"> Prepare OnionShare.msi for signing") - run( - [ - shutil.which("insignia.exe"), - "-im", - os.path.join(desktop_dir, "build", "OnionShare.msi"), - ], - error_ok=True, - ) - sign(os.path.join(desktop_dir, "build", "OnionShare.msi")) - - final_msi_filename = os.path.join( - desktop_dir, "dist", f"OnionShare-{version}.msi" - ) - print(f"> Final MSI: {final_msi_filename}") - os.makedirs(os.path.join(desktop_dir, "dist"), exist_ok=True) - os.rename( + print(f"> Prepare OnionShare.msi for signing") + run( + [ + shutil.which("insignia.exe"), + "-im", os.path.join(desktop_dir, "build", "OnionShare.msi"), - final_msi_filename, - ) + ], + error_ok=True, + ) + sign(os.path.join(desktop_dir, "build", "OnionShare.msi")) + + final_msi_filename = os.path.join( + desktop_dir, "dist", f"OnionShare-{version}.msi" + ) + print(f"> Final MSI: {final_msi_filename}") + os.makedirs(os.path.join(desktop_dir, "dist"), exist_ok=True) + os.rename( + os.path.join(desktop_dir, "build", "OnionShare.msi"), + final_msi_filename, + ) if __name__ == "__main__": diff --git a/desktop/package/windows.py b/desktop/package/windows.py new file mode 100644 index 00000000..a0bb3db9 --- /dev/null +++ b/desktop/package/windows.py @@ -0,0 +1,605 @@ +#!/usr/bin/env python3 +from distutils.command.build import build +import sys +import os +import inspect +import click +import shutil +import subprocess +import uuid +import xml.etree.ElementTree as ET + +root = os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) + ) +) +desktop_dir = os.path.join(root, "desktop") + + +def get_build_path(): + if "64 bit" in sys.version: + python_arch = "win-amd64" + else: + python_arch = "win32" + + build_path = os.path.join(desktop_dir, "build", f"exe.{python_arch}-3.9") + return build_path + + +def get_size(dir): + size = 0 + for path, dirs, files in os.walk(dir): + for f in files: + fp = os.path.join(path, f) + size += os.path.getsize(fp) + return size + + +def run(cmd, cwd=None, error_ok=False): + print(cmd) + try: + subprocess.run(cmd, cwd=cwd, check=True) + except subprocess.CalledProcessError as e: + if not error_ok: + raise subprocess.CalledProcessError(e) + + +def sign(filename): + run( + [ + shutil.which("signtool"), + "sign", + "/v", + "/d", + "OnionShare", + "/sha1", + "bb1d265ab02272e8fc742f149dcf8751cac63f50", + "/fd", + "SHA256", + "/td", + "SHA256", + "/tr", + "http://timestamp.digicert.com", + filename, + ] + ) + + +def wix_build_data(dirname, dir_prefix, id_, name): + data = { + "id": id_, + "name": name, + "files": [], + "dirs": [], + } + + for basename in os.listdir(dirname): + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + data["files"].append(os.path.join(dir_prefix, basename)) + elif os.path.isdir(filename): + if id_ == "INSTALLDIR": + id_prefix = "Folder" + else: + id_prefix = id_ + + # Skip lib/Pyside2/Examples folder + if "\\lib\\PySide2\\examples" in dirname: + continue + + id_value = f"{id_prefix}{basename.capitalize().replace('-', '_')}" + data["dirs"].append( + wix_build_data( + os.path.join(dirname, basename), + os.path.join(dir_prefix, basename), + id_value, + basename, + ) + ) + + if len(data["files"]) > 0: + if id_ == "INSTALLDIR": + data["component_id"] = "ApplicationFiles" + else: + data["component_id"] = "FolderComponent" + id_[len("Folder") :] + data["component_guid"] = str(uuid.uuid4()) + + return data + + +def wix_build_dir_xml(root, data): + attrs = {} + if "id" in data: + attrs["Id"] = data["id"] + if "name" in data: + attrs["Name"] = data["name"] + el = ET.SubElement(root, "Directory", attrs) + for subdata in data["dirs"]: + wix_build_dir_xml(el, subdata) + + # If this is the ProgramMenuFolder, add the menu component + if "id" in data and data["id"] == "ProgramMenuFolder": + component_el = ET.SubElement( + el, + "Component", + Id="ApplicationShortcuts", + Guid="539e7de8-a124-4c09-aa55-0dd516aad7bc", + ) + ET.SubElement( + component_el, + "Shortcut", + Id="ApplicationShortcut1", + Name="OnionShare", + Description="OnionShare", + Target="[INSTALLDIR]onionshare.exe", + WorkingDirectory="INSTALLDIR", + ) + ET.SubElement( + component_el, + "RegistryValue", + Root="HKCU", + Key="Software\OnionShare", + Name="installed", + Type="integer", + Value="1", + KeyPath="yes", + ) + + +def wix_build_components_xml(root, data): + component_ids = [] + if "component_id" in data: + component_ids.append(data["component_id"]) + + for subdata in data["dirs"]: + if "component_guid" in subdata: + dir_ref_el = ET.SubElement(root, "DirectoryRef", Id=subdata["id"]) + component_el = ET.SubElement( + dir_ref_el, + "Component", + Id=subdata["component_id"], + Guid=subdata["component_guid"], + ) + for filename in subdata["files"]: + file_el = ET.SubElement( + component_el, "File", Source=filename, Id="file_" + uuid.uuid4().hex + ) + + component_ids += wix_build_components_xml(root, subdata) + + return component_ids + + +@click.group() +def main(): + """ + Windows build tasks + """ + + +@main.command() +def cleanup_build(): + """Delete unused PySide2 stuff to save space""" + build_path = get_build_path() + before_size = get_size(build_path) + + for dirname in ["examples", "qml"]: + shutil.rmtree(os.path.join(build_path, "lib", "PySide2", dirname)) + for filename in [ + "lconvert.exe", + "linguist.exe", + "lrelease.exe", + "lupdate.exe", + "plugins/assetimporters/assimp.dll", + "plugins/assetimporters/uip.dll", + "plugins/audio/qtaudio_wasapi.dll", + "plugins/audio/qtaudio_windows.dll", + "plugins/bearer/qgenericbearer.dll", + "plugins/canbus/qtpassthrucanbus.dll", + "plugins/canbus/qtpeakcanbus.dll", + "plugins/canbus/qtsysteccanbus.dll", + "plugins/canbus/qttinycanbus.dll", + "plugins/canbus/qtvectorcanbus.dll", + "plugins/canbus/qtvirtualcanbus.dll", + "plugins/gamepads/xinputgamepad.dll", + "plugins/generic/qtuiotouchplugin.dll", + "plugins/geometryloaders/defaultgeometryloader.dll", + "plugins/geometryloaders/gltfgeometryloader.dll", + "plugins/geoservices/qtgeoservices_esri.dll", + "plugins/geoservices/qtgeoservices_itemsoverlay.dll", + "plugins/geoservices/qtgeoservices_mapbox.dll", + "plugins/geoservices/qtgeoservices_nokia.dll", + "plugins/geoservices/qtgeoservices_osm.dll", + "plugins/mediaservice/dsengine.dll", + "plugins/mediaservice/qtmedia_audioengine.dll", + "plugins/mediaservice/wmfengine.dll", + "plugins/platforminputcontexts/qtvirtualkeyboardplugin.dll", + "plugins/platforms/qdirect2d.dll", + "plugins/platforms/qoffscreen.dll", + "plugins/platforms/qwebgl.dll", + "plugins/platformthemes/qxdgdesktopportal.dll", + "plugins/playlistformats/qtmultimedia_m3u.dll", + "plugins/position/qtposition_positionpoll.dll", + "plugins/position/qtposition_serialnmea.dll", + "plugins/position/qtposition_winrt.dll", + "plugins/printsupport/windowsprintersupport.dll", + "plugins/qmltooling/qmldbg_debugger.dll", + "plugins/qmltooling/qmldbg_inspector.dll", + "plugins/qmltooling/qmldbg_local.dll", + "plugins/qmltooling/qmldbg_messages.dll", + "plugins/qmltooling/qmldbg_native.dll", + "plugins/qmltooling/qmldbg_nativedebugger.dll", + "plugins/qmltooling/qmldbg_preview.dll", + "plugins/qmltooling/qmldbg_profiler.dll", + "plugins/qmltooling/qmldbg_quickprofiler.dll", + "plugins/qmltooling/qmldbg_server.dll", + "plugins/qmltooling/qmldbg_tcp.dll", + "plugins/renderers/openglrenderer.dll", + "plugins/renderplugins/scene2d.dll", + "plugins/scenegraph/qsgd3d12backend.dll", + "plugins/sceneparsers/gltfsceneexport.dll", + "plugins/sceneparsers/gltfsceneimport.dll", + "plugins/sensorgestures/qtsensorgestures_plugin.dll", + "plugins/sensorgestures/qtsensorgestures_shakeplugin.dll", + "plugins/sensors/qtsensors_generic.dll", + "plugins/sqldrivers/qsqlite.dll", + "plugins/sqldrivers/qsqlodbc.dll", + "plugins/sqldrivers/qsqlpsql.dll", + "plugins/styles/qwindowsvistastyle.dll", + "plugins/texttospeech/qtexttospeech_sapi.dll", + "plugins/virtualkeyboard/qtvirtualkeyboard_hangul.dll", + "plugins/virtualkeyboard/qtvirtualkeyboard_openwnn.dll", + "plugins/virtualkeyboard/qtvirtualkeyboard_pinyin.dll", + "plugins/virtualkeyboard/qtvirtualkeyboard_tcime.dll", + "plugins/virtualkeyboard/qtvirtualkeyboard_thai.dll", + "plugins/webview/qtwebview_webengine.dll", + "pyside2-lupdate.exe", + "Qt3DAnimation.pyd", + "Qt3DAnimation.pyi", + "Qt3DCore.pyd", + "Qt3DCore.pyi", + "Qt3DExtras.pyd", + "Qt3DExtras.pyi", + "Qt3DInput.pyd", + "Qt3DInput.pyi", + "Qt3DLogic.pyd", + "Qt3DLogic.pyi", + "Qt3DRender.pyd", + "Qt3DRender.pyi", + "Qt53DAnimation.dll", + "Qt53DCore.dll", + "Qt53DExtras.dll", + "Qt53DInput.dll", + "Qt53DLogic.dll", + "Qt53DQuick.dll", + "Qt53DQuickAnimation.dll", + "Qt53DQuickExtras.dll", + "Qt53DQuickInput.dll", + "Qt53DQuickRender.dll", + "Qt53DQuickScene2D.dll", + "Qt53DRender.dll", + "Qt5Bluetooth.dll", + "Qt5Bodymovin.dll", + "Qt5Charts.dll", + "Qt5Concurrent.dll", + "Qt5DataVisualization.dll", + "Qt5DBus.dll", + "Qt5Designer.dll", + "Qt5DesignerComponents.dll", + "Qt5Gamepad.dll", + "Qt5Help.dll", + "Qt5Location.dll", + "Qt5Multimedia.dll", + "Qt5MultimediaQuick.dll", + "Qt5MultimediaWidgets.dll", + "Qt5Nfc.dll", + "Qt5OpenGL.dll", + "Qt5Pdf.dll", + "Qt5PdfWidgets.dll", + "Qt5Positioning.dll", + "Qt5PositioningQuick.dll", + "Qt5PrintSupport.dll", + "Qt5Purchasing.dll", + "Qt5Quick.dll", + "Qt5Quick3D.dll", + "Qt5Quick3DAssetImport.dll", + "Qt5Quick3DRender.dll", + "Qt5Quick3DRuntimeRender.dll", + "Qt5Quick3DUtils.dll", + "Qt5QuickControls2.dll", + "Qt5QuickParticles.dll", + "Qt5QuickShapes.dll", + "Qt5QuickTemplates2.dll", + "Qt5QuickTest.dll", + "Qt5QuickWidgets.dll", + "Qt5RemoteObjects.dll", + "Qt5Script.dll", + "Qt5ScriptTools.dll", + "Qt5Scxml.dll", + "Qt5Sensors.dll", + "Qt5SerialBus.dll", + "Qt5SerialPort.dll", + "Qt5Sql.dll", + "Qt5Svg.dll", + "Qt5Test.dll", + "Qt5TextToSpeech.dll", + "Qt5VirtualKeyboard.dll", + "Qt5WebChannel.dll", + "Qt5WebEngine.dll", + "Qt5WebEngineCore.dll", + "Qt5WebEngineWidgets.dll", + "Qt5WebSockets.dll", + "Qt5WebView.dll", + "Qt5Xml.dll", + "Qt5XmlPatterns.dll", + "QtAxContainer.pyd", + "QtAxContainer.pyi", + "QtCharts.pyd", + "QtCharts.pyi", + "QtConcurrent.pyd", + "QtConcurrent.pyi", + "QtDataVisualization.pyd", + "QtDataVisualization.pyi", + "qtdiag.exe", + "QtHelp.pyd", + "QtHelp.pyi", + "QtLocation.pyd", + "QtLocation.pyi", + "QtMultimedia.pyd", + "QtMultimedia.pyi", + "QtMultimediaWidgets.pyd", + "QtMultimediaWidgets.pyi", + "QtNetwork.pyd", + "QtNetwork.pyi", + "QtOpenGL.pyd", + "QtOpenGL.pyi", + "QtOpenGLFunctions.pyd", + "QtOpenGLFunctions.pyi", + "QtPositioning.pyd", + "QtPositioning.pyi", + "QtPrintSupport.pyd", + "QtPrintSupport.pyi", + "QtQml.pyd", + "QtQml.pyi", + "QtQuick.pyd", + "QtQuick.pyi", + "QtQuickControls2.pyd", + "QtQuickControls2.pyi", + "QtQuickWidgets.pyd", + "QtQuickWidgets.pyi", + "QtRemoteObjects.pyd", + "QtRemoteObjects.pyi", + "QtScript.pyd", + "QtScript.pyi", + "QtScriptTools.pyd", + "QtScriptTools.pyi", + "QtScxml.pyd", + "QtScxml.pyi", + "QtSensors.pyd", + "QtSensors.pyi", + "QtSerialPort.pyd", + "QtSerialPort.pyi", + "QtSql.pyd", + "QtSql.pyi", + "QtSvg.pyd", + "QtSvg.pyi", + "QtTest.pyd", + "QtTest.pyi", + "QtTextToSpeech.pyd", + "QtTextToSpeech.pyi", + "QtUiTools.pyd", + "QtUiTools.pyi", + "QtWebChannel.pyd", + "QtWebChannel.pyi", + "QtWebEngine.pyd", + "QtWebEngine.pyi", + "QtWebEngineCore.pyd", + "QtWebEngineCore.pyi", + "QtWebEngineProcess.exe", + "QtWebEngineWidgets.pyd", + "QtWebEngineWidgets.pyi", + "QtWebSockets.pyd", + "QtWebSockets.pyi", + "QtWinExtras.pyd", + "QtWinExtras.pyi", + "QtXml.pyd", + "QtXml.pyi", + "QtXmlPatterns.pyd", + "QtXmlPatterns.pyi", + "rcc.exe", + "uic.exe", + ]: + os.remove( + os.path.join( + build_path, + "lib", + "PySide2", + filename.replace("/", "\\"), + ) + ) + + after_size = get_size(build_path) + freed_bytes = before_size - after_size + freed_mb = int(freed_bytes / 1024 / 1024) + print(f"Freed {freed_mb} mb") + + +@main.command() +@click.argument("build_path") +def codesign(build_path): + """Sign Windows binaries before packaging""" + if not os.path.isdir(build_path): + click.echo("Invalid build path") + return + + click.echo("> Signing onionshare.exe") + sign(os.path.join(build_path, "onionshare.exe")) + + click.echo("> Signing onionshare-cli.exe") + sign(os.path.join(build_path, "onionshare-cli.exe")) + + click.echo("> Signing meek-client.exe") + sign( + os.path.join( + build_path, + "lib", + "onionshare", + "resources", + "tor", + "Tor", + "meek-client.exe", + ) + ) + + +@main.command() +@click.argument("build_path") +def package(build_path): + """Build the MSI package""" + + print(f"> Build the WiX file") + version_filename = os.path.join( + root, "cli", "onionshare_cli", "resources", "version.txt" + ) + with open(version_filename) as f: + version = f.read().strip() + + data = { + "id": "TARGETDIR", + "name": "SourceDir", + "dirs": [ + { + "id": "ProgramFilesFolder", + "dirs": [], + }, + { + "id": "ProgramMenuFolder", + "dirs": [], + }, + ], + } + + data["dirs"][0]["dirs"].append( + wix_build_data( + build_path, + "INSTALLDIR", + "OnionShare", + ) + ) + + root_el = ET.Element("Wix", xmlns="http://schemas.microsoft.com/wix/2006/wi") + product_el = ET.SubElement( + root_el, + "Product", + Name="OnionShare", + Manufacturer="Micah Lee, et al.", + Id="*", + UpgradeCode="$(var.ProductUpgradeCode)", + Language="1033", + Codepage="1252", + Version="$(var.ProductVersion)", + ) + ET.SubElement( + product_el, + "Package", + Id="*", + Keywords="Installer", + Description="OnionShare $(var.ProductVersion) Installer", + Manufacturer="Micah Lee, et al.", + InstallerVersion="100", + Languages="1033", + Compressed="yes", + SummaryCodepage="1252", + ) + ET.SubElement(product_el, "Media", Id="1", Cabinet="product.cab", EmbedCab="yes") + ET.SubElement( + product_el, + "Icon", + Id="ProductIcon", + SourceFile="..\\onionshare\\resources\\onionshare.ico", + ) + ET.SubElement(product_el, "Property", Id="ARPPRODUCTICON", Value="ProductIcon") + ET.SubElement( + product_el, + "Property", + Id="ARPHELPLINK", + Value="https://docs.onionshare.org", + ) + ET.SubElement( + product_el, + "Property", + Id="ARPURLINFOABOUT", + Value="https://onionshare.org", + ) + ET.SubElement(product_el, "UIRef", Id="WixUI_Minimal") + ET.SubElement(product_el, "UIRef", Id="WixUI_ErrorProgressText") + ET.SubElement( + product_el, + "WixVariable", + Id="WixUILicenseRtf", + Value="..\\package\\license.rtf", + ) + ET.SubElement( + product_el, + "WixVariable", + Id="WixUIDialogBmp", + Value="..\\package\\dialog.bmp", + ) + ET.SubElement( + product_el, + "MajorUpgrade", + AllowSameVersionUpgrades="yes", + DowngradeErrorMessage="A newer version of [ProductName] is already installed. If you are sure you want to downgrade, remove the existing installation via Programs and Features.", + ) + + wix_build_dir_xml(product_el, data) + component_ids = wix_build_components_xml(product_el, data) + + feature_el = ET.SubElement(product_el, "Feature", Id="DefaultFeature", Level="1") + for component_id in component_ids: + ET.SubElement(feature_el, "ComponentRef", Id=component_id) + ET.SubElement(feature_el, "ComponentRef", Id="ApplicationShortcuts") + + with open(os.path.join(root, "desktop", "build", "OnionShare.wxs"), "w") as f: + f.write('\n') + f.write(f'\n') + f.write( + '\n' + ) + + ET.indent(root_el) + f.write(ET.tostring(root_el).decode()) + + print(f"> Build the MSI") + run( + [shutil.which("candle.exe"), "OnionShare.wxs"], + os.path.join(desktop_dir, "build"), + ) + run( + [shutil.which("light.exe"), "-ext", "WixUIExtension", "OnionShare.wixobj"], + os.path.join(desktop_dir, "build"), + ) + + print(f"> Prepare OnionShare.msi for signing") + run( + [ + shutil.which("insignia.exe"), + "-im", + os.path.join(desktop_dir, "build", "OnionShare.msi"), + ], + error_ok=True, + ) + sign(os.path.join(desktop_dir, "build", "OnionShare.msi")) + + final_msi_filename = os.path.join(desktop_dir, "dist", f"OnionShare-{version}.msi") + print(f"> Final MSI: {final_msi_filename}") + os.makedirs(os.path.join(desktop_dir, "dist"), exist_ok=True) + os.rename( + os.path.join(desktop_dir, "build", "OnionShare.msi"), + final_msi_filename, + ) + + +if __name__ == "__main__": + main() diff --git a/desktop/poetry.lock b/desktop/poetry.lock index aca454fe..a5d335ae 100644 --- a/desktop/poetry.lock +++ b/desktop/poetry.lock @@ -30,7 +30,7 @@ python-versions = ">=3.6" [[package]] name = "black" -version = "22.1.0" +version = "22.3.0" description = "The uncompromising code formatter." category = "dev" optional = false @@ -42,7 +42,7 @@ dataclasses = {version = ">=0.6", markers = "python_version < \"3.7\""} mypy-extensions = ">=0.4.3" pathspec = ">=0.9.0" platformdirs = ">=2" -tomli = ">=1.1.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} @@ -142,7 +142,7 @@ python-versions = ">=3.6, <3.7" [[package]] name = "dnspython" -version = "2.2.0" +version = "2.2.1" description = "DNS toolkit" category = "main" optional = false @@ -299,7 +299,7 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "lief" -version = "0.11.5" +version = "0.12.0" description = "Library to instrument executable formats" category = "main" optional = false @@ -633,7 +633,7 @@ python-versions = ">=3.6" [[package]] name = "unidecode" -version = "1.3.3" +version = "1.3.4" description = "ASCII transliterations of Unicode text" category = "main" optional = false @@ -641,14 +641,14 @@ python-versions = ">=3.5" [[package]] name = "urllib3" -version = "1.26.8" +version = "1.26.9" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] -brotli = ["brotlipy (>=0.6.0)"] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] @@ -706,7 +706,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "1.1" python-versions = ">=3.6.2,<3.10" -content-hash = "29504ac7877359020e54f814cd2e8cc5b79d7177fb9711d398b213a1c64dc24c" +content-hash = "3d77cb0fc1b57267e0997daee19b535d3feda80a36a795013e322dbd130d514d" [metadata.files] atomicwrites = [ @@ -722,29 +722,29 @@ bidict = [ {file = "bidict-0.21.4.tar.gz", hash = "sha256:42c84ffbe6f8de898af6073b4be9ea7ccedcd78d3474aa844c54e49d5a079f6f"}, ] black = [ - {file = "black-22.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1297c63b9e1b96a3d0da2d85d11cd9bf8664251fd69ddac068b98dc4f34f73b6"}, - {file = "black-22.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2ff96450d3ad9ea499fc4c60e425a1439c2120cbbc1ab959ff20f7c76ec7e866"}, - {file = "black-22.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e21e1f1efa65a50e3960edd068b6ae6d64ad6235bd8bfea116a03b21836af71"}, - {file = "black-22.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2f69158a7d120fd641d1fa9a921d898e20d52e44a74a6fbbcc570a62a6bc8ab"}, - {file = "black-22.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:228b5ae2c8e3d6227e4bde5920d2fc66cc3400fde7bcc74f480cb07ef0b570d5"}, - {file = "black-22.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b1a5ed73ab4c482208d20434f700d514f66ffe2840f63a6252ecc43a9bc77e8a"}, - {file = "black-22.1.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35944b7100af4a985abfcaa860b06af15590deb1f392f06c8683b4381e8eeaf0"}, - {file = "black-22.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7835fee5238fc0a0baf6c9268fb816b5f5cd9b8793423a75e8cd663c48d073ba"}, - {file = "black-22.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dae63f2dbf82882fa3b2a3c49c32bffe144970a573cd68d247af6560fc493ae1"}, - {file = "black-22.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa1db02410b1924b6749c245ab38d30621564e658297484952f3d8a39fce7e8"}, - {file = "black-22.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c8226f50b8c34a14608b848dc23a46e5d08397d009446353dad45e04af0c8e28"}, - {file = "black-22.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2d6f331c02f0f40aa51a22e479c8209d37fcd520c77721c034517d44eecf5912"}, - {file = "black-22.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:742ce9af3086e5bd07e58c8feb09dbb2b047b7f566eb5f5bc63fd455814979f3"}, - {file = "black-22.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fdb8754b453fb15fad3f72cd9cad3e16776f0964d67cf30ebcbf10327a3777a3"}, - {file = "black-22.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5660feab44c2e3cb24b2419b998846cbb01c23c7fe645fee45087efa3da2d61"}, - {file = "black-22.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:6f2f01381f91c1efb1451998bd65a129b3ed6f64f79663a55fe0e9b74a5f81fd"}, - {file = "black-22.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:efbadd9b52c060a8fc3b9658744091cb33c31f830b3f074422ed27bad2b18e8f"}, - {file = "black-22.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8871fcb4b447206904932b54b567923e5be802b9b19b744fdff092bd2f3118d0"}, - {file = "black-22.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ccad888050f5393f0d6029deea2a33e5ae371fd182a697313bdbd835d3edaf9c"}, - {file = "black-22.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07e5c049442d7ca1a2fc273c79d1aecbbf1bc858f62e8184abe1ad175c4f7cc2"}, - {file = "black-22.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:373922fc66676133ddc3e754e4509196a8c392fec3f5ca4486673e685a421321"}, - {file = "black-22.1.0-py3-none-any.whl", hash = "sha256:3524739d76b6b3ed1132422bf9d82123cd1705086723bc3e235ca39fd21c667d"}, - {file = "black-22.1.0.tar.gz", hash = "sha256:a7c0192d35635f6fc1174be575cb7915e92e5dd629ee79fdaf0dcfa41a80afb5"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, + {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, + {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, + {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, + {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, + {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, + {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, + {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, + {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, + {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, + {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, + {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, + {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, + {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, + {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, + {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, + {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, + {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, ] cepa = [ {file = "cepa-1.8.3.tar.gz", hash = "sha256:1dc6f0b324d37a2ed2ca274648ece8fd2c96a1d2f440f58c0ca17afd4b5ede7a"}, @@ -850,8 +850,8 @@ dataclasses = [ {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, ] dnspython = [ - {file = "dnspython-2.2.0-py3-none-any.whl", hash = "sha256:081649da27ced5e75709a1ee542136eaba9842a0fe4c03da4fb0a3d3ed1f3c44"}, - {file = "dnspython-2.2.0.tar.gz", hash = "sha256:e79351e032d0b606b98d38a4b0e6e2275b31a5b85c873e587cc11b73aca026d6"}, + {file = "dnspython-2.2.1-py3-none-any.whl", hash = "sha256:a851e51367fb93e9e1361732c1d60dab63eff98712e503ea7d92e6eccb109b4f"}, + {file = "dnspython-2.2.1.tar.gz", hash = "sha256:0f7569a4a6ff151958b64304071d370daa3243d15941a7beedf0c9fe5105603e"}, ] eventlet = [ {file = "eventlet-0.33.0-py2.py3-none-any.whl", hash = "sha256:d10a8fcc9e33381905d9873303fde96ebe3541c03fb795055d2c7347dce0639c"}, @@ -982,36 +982,30 @@ jinja2 = [ {file = "Jinja2-3.0.3.tar.gz", hash = "sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7"}, ] lief = [ - {file = "lief-0.11.5-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:be2b5420a8c48968c37621aa9ff0b34bc56115687f095da419ff97e4542cd27a"}, - {file = "lief-0.11.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f7f33e44ea73aebdfa4f511866a919a7a8cb5cbb6a3ca573549f8137232ec7b"}, - {file = "lief-0.11.5-cp310-cp310-win32.whl", hash = "sha256:c30be8afd44f2b2441427f71b06a0cae190d535268304ba92f13b0eec9436221"}, - {file = "lief-0.11.5-cp310-cp310-win_amd64.whl", hash = "sha256:aa41e6fbd8feb7e42a1d67b99604f4c32b31e5dd1e36c99a90af73f19dc377f8"}, - {file = "lief-0.11.5-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:1cca100e77382f4137a3b1283769efa0e68a965fa4f3e21e64e3f67b6e22fdc8"}, - {file = "lief-0.11.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:621ad19f77884a008d61e05b92aed8309a8460e93916f4722439beaa529ca37d"}, - {file = "lief-0.11.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c672dcd78dbbe2c0746721cdb1593b237a8b983d039e73713b055449e4a58207"}, - {file = "lief-0.11.5-cp36-cp36m-win32.whl", hash = "sha256:5a0da170943aaf7019b27b9a7199b860298426c0455f88add392f472605c39ee"}, - {file = "lief-0.11.5-cp36-cp36m-win_amd64.whl", hash = "sha256:5f5fb42461b5d5d5b2ccf7fe17e8f26bd632afcbaedf29a9d30819eeea5dab29"}, - {file = "lief-0.11.5-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:710112ebc642bf5287a7b25c54c8a4e1079cbb403d4e844a364e1c3cbed52486"}, - {file = "lief-0.11.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bfc0246af63361e22a952f8babd542477d64288d993c5a053a72f9e3f59da795"}, - {file = "lief-0.11.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:8b219ce4a41b0734fe9a7fbfde7d23a92bc005c8684882662808fc438568c1b5"}, - {file = "lief-0.11.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f510836d19cee407015ee565ea566e444471f0ecb3028a5c5e2219a7583f3c4"}, - {file = "lief-0.11.5-cp37-cp37m-win32.whl", hash = "sha256:9c6cc9da3e3a56ad29fc4e77e7109e960bd0cae3e3ba5307e3ae5c65d85fbdc4"}, - {file = "lief-0.11.5-cp37-cp37m-win_amd64.whl", hash = "sha256:a1f7792f1d811a898d3d676c32731d6b055573a2c3e67988ab1b32917db3de96"}, - {file = "lief-0.11.5-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:fd41077526e30bfcafa3d03bff8466a4a9ae4bbe21cadd6a09168a62ce18710c"}, - {file = "lief-0.11.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5122e4e70fecc32e7fdf2e9cd9b580ddd63fb4509eae373be78b3c11d67175b8"}, - {file = "lief-0.11.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:e6d9621c1db852ca4de37efe98151838edf0a976fe03cace471b3a761861f95e"}, - {file = "lief-0.11.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:17314177c0124ccd450554bbcb203b8cd2660c94e36bdc05a6eba04bb0af3954"}, - {file = "lief-0.11.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b275a542b5ef173ec9602d2f511a895c4228db63bbbc58699859da4afe8bfd58"}, - {file = "lief-0.11.5-cp38-cp38-win32.whl", hash = "sha256:208294f208354f57ded772efc4c3b2ea61fae35325a048d38c21571cb35e4bfc"}, - {file = "lief-0.11.5-cp38-cp38-win_amd64.whl", hash = "sha256:f4e8a878615a46ef4ae016261a59152b8c019a35adb865e26a37c8ef25200d7e"}, - {file = "lief-0.11.5-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:544b0f8a587bc5f6fd39cf47d9785af2714f982682efcd1dd3291604e7cb6351"}, - {file = "lief-0.11.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e743345290649f54efcf2c1ea530f3520a7b22583fb8b0772df48b1901ecb1ea"}, - {file = "lief-0.11.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:eb8c2ae617ff54c4ea73dbd055544681b3cfeafbdbf0fe4535fac494515ab65b"}, - {file = "lief-0.11.5-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a4bb649a2f5404b8e2e4b8beb3772466430e7382fc5f7f014f3f778137133987"}, - {file = "lief-0.11.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44bd7804a39837ff46cd543154f6e4a28e2d4fafa312752ca6deea1c849995ce"}, - {file = "lief-0.11.5-cp39-cp39-win32.whl", hash = "sha256:8fd1ecdb3001e8e19df7278b77df5d6394ad6071354e177d11ad08b0a727d390"}, - {file = "lief-0.11.5-cp39-cp39-win_amd64.whl", hash = "sha256:c773eaee900f398cc98a9c8501d9ab7465af9729979841bb78f4aaa8b821fd9a"}, - {file = "lief-0.11.5.zip", hash = "sha256:932ba495388fb52b4ba056a0b00abe0bda3567ad3ebc6d726be1e87b8be08b3f"}, + {file = "lief-0.12.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:fade4db6e7127586ba94a361e46066ce16d2406e8a4395b7986e90bb494f6c10"}, + {file = "lief-0.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:078bdda599882f3534bc4fe02005c3e0c1ca1c4532aeb48501ad94edc860a4f7"}, + {file = "lief-0.12.0-cp310-cp310-win32.whl", hash = "sha256:5638521125da9e66b8ff8403342bc6af54965b8b59ac86d114b886c40a49ed3c"}, + {file = "lief-0.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:211684f25d7b98a0d9e8a77113ff3f358119fa305acea1f372519b11c54aa0c4"}, + {file = "lief-0.12.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:8523945fb5333d8cac086cebb937a8cfe83b11c32e3d161e747c98d1e1a1ac37"}, + {file = "lief-0.12.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4826afd38e85acf3ac1dd6924d8e8aee7998faee4a91d846e7dfb96efc4690c"}, + {file = "lief-0.12.0-cp36-cp36m-win32.whl", hash = "sha256:156517a762f2588605c22b79e76dc825bc8983bf3045ae52477ba11f3b0a90ff"}, + {file = "lief-0.12.0-cp36-cp36m-win_amd64.whl", hash = "sha256:23098eb6a6f6d3720a21e24652f66d4674fb524abc6e39cc5373413f70fea86c"}, + {file = "lief-0.12.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1fee17492021f8733783b272d26a5c8aeceff4576baad991a8bb4ecb621b9229"}, + {file = "lief-0.12.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:767d1993dcc8953490357a2bfee73eb41b818c8235301ac5f5f3e6cec36dee21"}, + {file = "lief-0.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a540d58ac570b8a225ec220c896b665024e495d8a669d0f5396baa95c1d0f7ff"}, + {file = "lief-0.12.0-cp37-cp37m-win32.whl", hash = "sha256:510e4ee28f084d4368c9c7eecd0b3c32600adbe78f2afe99b50d1dccc587ef30"}, + {file = "lief-0.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:da6978d8e0014a5e4b0bfe37cc6fe72278c051a08e0b3392a65f5b6c4caf0187"}, + {file = "lief-0.12.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:d8a34b1e471ed8fcbe030f6ce7c93d618af51bd20abd170b8d0ae011f9d57f39"}, + {file = "lief-0.12.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:c240338700549f86768475c7c516d5c13afa8326ed1e317c880750c642d5af4c"}, + {file = "lief-0.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53b94c5608f35b8001400b208c86fed52fcd14dab1d4e600beb2a5829830377e"}, + {file = "lief-0.12.0-cp38-cp38-win32.whl", hash = "sha256:62b2a4738e74b5b6026d551bf17f1d28822ca6485b92dfbef58c622759c32d82"}, + {file = "lief-0.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:4fb81a3b20fcc9204a90815a9ab40c263e10323161e8fb24053a453032dde065"}, + {file = "lief-0.12.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:0088834190b2ab75d2f7072126739500866a2f86f89a7b4a528f882ce6317c31"}, + {file = "lief-0.12.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1e866ad8476d181fc8d719089a1ba71cfafb877b89f11079050e9bdf36b46199"}, + {file = "lief-0.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10a920f7a9d6b40040d2cae331ebfdc781fba151e758b2440a6849af391e4c5d"}, + {file = "lief-0.12.0-cp39-cp39-win32.whl", hash = "sha256:cfe8a024678be1e552841c4ef80d548683aa217430b715b1a7f7fbdb7171ec99"}, + {file = "lief-0.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:c0365acf4cd476ba9fba9165013f78165f3ced935097fa15d32bfab68ed61a2c"}, + {file = "lief-0.12.0.zip", hash = "sha256:f1dfe770ce7b80db9380af6d8b3a09e63b7d7ef7584f4e3716ff5ad94376afc1"}, ] markupsafe = [ {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"}, @@ -1258,12 +1252,12 @@ typing-extensions = [ {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, ] unidecode = [ - {file = "Unidecode-1.3.3-py3-none-any.whl", hash = "sha256:a5a8a4b6fb033724ffba8502af2e65ca5bfc3dd53762dedaafe4b0134ad42e3c"}, - {file = "Unidecode-1.3.3.tar.gz", hash = "sha256:8521f2853fd250891dc27d156a9d30e61c4e76319da963c4a1c27083a909ac30"}, + {file = "Unidecode-1.3.4-py3-none-any.whl", hash = "sha256:afa04efcdd818a93237574791be9b2817d7077c25a068b00f8cff7baa4e59257"}, + {file = "Unidecode-1.3.4.tar.gz", hash = "sha256:8e4352fb93d5a735c788110d2e7ac8e8031eb06ccbfe8d324ab71735015f9342"}, ] urllib3 = [ - {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, - {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, + {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, + {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, ] werkzeug = [ {file = "Werkzeug-2.0.3-py3-none-any.whl", hash = "sha256:1421ebfc7648a39a5c58c601b154165d05cf47a3cd0ccb70857cbdacf6c8f2b8"}, diff --git a/desktop/pyproject.toml b/desktop/pyproject.toml index 6c77e4cf..d6809d99 100644 --- a/desktop/pyproject.toml +++ b/desktop/pyproject.toml @@ -13,6 +13,7 @@ qrcode = "*" cx_freeze = "*" [tool.poetry.dev-dependencies] +click = "*" black = "*" pytest = "*" pytest-faulthandler = "*" From f9b79d667f4f06cfd3f5a3fd5e1d1492cc6397e9 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 18:40:36 -0700 Subject: [PATCH 018/154] Don't install 7zip with choco because it's already in the CircleCI windows orb, and zip contents of build folder instead of build folder itself --- .circleci/config.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4673767e..9848c57d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,10 +79,9 @@ jobs: steps: - checkout - run: - name: Install chocolately dependencies + name: Install Python 3.9.12 command: | choco install python --version=3.9.12 - choco install 7zip - run: name: Install poetry command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - @@ -109,7 +108,7 @@ jobs: poetry run python .\package\windows.py cleanup-build - run: name: Compress - command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win-amd64-3.9 -DestinationPath ~\onionshare-win64.zip + command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win-amd64-3.9\* -DestinationPath ~\onionshare-win64.zip - store_artifacts: path: ~\onionshare-win64.zip @@ -121,10 +120,9 @@ jobs: steps: - checkout - run: - name: Install chocolately dependencies + name: Install Python 3.9.12 (32-bit) command: | choco install python3 --params "/InstallDir32:C:\Python-32bit" --version=3.9.12 - choco install 7zip - run: name: Install golang (32-bit) command: | @@ -157,6 +155,6 @@ jobs: C:\Python-32bit\Scripts\poetry run python .\package\windows.py cleanup-build - run: name: Compress - command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win32-3.9 -DestinationPath ~\onionshare-win32.zip + command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win32-3.9\* -DestinationPath ~\onionshare-win32.zip - store_artifacts: path: ~\onionshare-win32.zip From fd052e6f1c9f91538738c4020f832bed378239db Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 18:54:32 -0700 Subject: [PATCH 019/154] Fix compressing Windows builds, add skeleton for making macOS build, and start snapcraft build --- .circleci/config.yml | 47 +++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9848c57d..7a11181c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,17 +17,22 @@ workflows: requires: - test-cli # - test-gui + - build-macos: + requires: + - test-cli + # - test-gui + - build-snapcraft: + requires: + - test-cli + # - test-gui jobs: test-cli: docker: - image: cimg/python:3.9 - working_directory: ~/repo - steps: - checkout - - run: name: Install dependencies command: | @@ -36,7 +41,6 @@ jobs: pip install poetry cd ~/repo/cli poetry install - - run: name: Run tests command: | @@ -50,12 +54,9 @@ jobs: test-gui: docker: - image: cimg/python:3.9 - working_directory: ~/repo - steps: - checkout - - run: name: Install dependencies command: | @@ -64,7 +65,6 @@ jobs: sudo apt-get install -y xvfb x11-utils libxkbcommon-x11-0 libxcb-randr0-dev libxcb-xtest0-dev libxcb-xinerama0-dev libxcb-shape0-dev libxcb-xkb-dev libxcb-render-util0 libxcb-icccm4 libxcb-keysyms1 libxcb-image0 cd ~/repo/desktop poetry install - - run: name: Run tests command: | @@ -75,7 +75,6 @@ jobs: executor: name: win/default shell: powershell.exe - steps: - checkout - run: @@ -108,7 +107,9 @@ jobs: poetry run python .\package\windows.py cleanup-build - run: name: Compress - command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win-amd64-3.9\* -DestinationPath ~\onionshare-win64.zip + command: | + mv ~\project\desktop\build\exe.win-amd64-3.9\ ~\onionshare-win64 + Compress-Archive -LiteralPath ~\onionshare-win64 -DestinationPath ~\onionshare-win64.zip - store_artifacts: path: ~\onionshare-win64.zip @@ -116,7 +117,6 @@ jobs: executor: name: win/default shell: powershell.exe - steps: - checkout - run: @@ -155,6 +155,29 @@ jobs: C:\Python-32bit\Scripts\poetry run python .\package\windows.py cleanup-build - run: name: Compress - command: Compress-Archive -LiteralPath ~\project\desktop\build\exe.win32-3.9\* -DestinationPath ~\onionshare-win32.zip + command: | + mv ~\project\desktop\build\exe.win32-3.9\ ~\onionshare-win32 + Compress-Archive -LiteralPath ~\onionshare-win32 -DestinationPath ~\onionshare-win32.zip - store_artifacts: path: ~\onionshare-win32.zip + + build-macos: + macos: + xcode: 12.5.1 + steps: + - checkout + + build-snapcraft: + docker: + - image: ubuntu/20.04 + working_directory: ~/repo + steps: + - checkout + - run: + name: Build the snap + command: | + cd ~/repo + snapcraft + mv onionshare-*_amd64.snap onionshare_amd64.snap + - store_artifacts: + path: ~/repo/onionshare_amd64.snap From f4a50b573d78888dffac77769c5513252d993f5a Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 18:59:38 -0700 Subject: [PATCH 020/154] For snapcraft, switch from docker to machine --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a11181c..91944c6c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -168,8 +168,8 @@ jobs: - checkout build-snapcraft: - docker: - - image: ubuntu/20.04 + machine: + image: ubuntu-2004:current working_directory: ~/repo steps: - checkout From 4e5ad4d0ca996f0dd651e8e2e52107927107d01d Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 19:35:26 -0700 Subject: [PATCH 021/154] Work on macOS build --- .circleci/config.yml | 41 +++++ RELEASE.md | 9 +- desktop/package/macos.py | 311 +++++++++++++++++++++++++++++++++++++ desktop/package/windows.py | 4 +- 4 files changed, 359 insertions(+), 6 deletions(-) create mode 100644 desktop/package/macos.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 91944c6c..1f7d763a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -166,6 +166,47 @@ jobs: xcode: 12.5.1 steps: - checkout + - run: + name: Install Homebrew dependencies + command: | + brew install wget + brew install go + - run: + name: Install Python 3.9.12 + command: | + wget https://www.python.org/ftp/python/3.9.12/python-3.9.12-macos11.pkg -O ~/Downloads/python.pkg + sudo installer -pkg ~/Downloads/python.pkg -target / + - run: + name: Install poetry + command: | + pip3 install poetry + ln -s /Library/Frameworks/Python.framework/Versions/3.9/bin/poetry /usr/local/bin + - run: + name: Install poetry dependencies + command: | + cd ~/project/desktop + poetry install + - run: + name: Get tor + command: | + cd ~/project/desktop + poetry run ./scripts/get-tor-osx.py + - run: + name: Build meek + command: | + cd ~/project/desktop + ./scripts/build-meek-client.py + - run: + name: Build OnionShare + command: | + cd ~/project/desktop + poetry run python ./setup-freeze.py bdist_mac + poetry run python ./package/macos.py cleanup-build + - run: + name: Compress + command: zip -r ~/onionshare-macos.zip ~/project/desktop/build/OnionShare.app + - store_artifacts: + path: ~/onionshare-macos.zip build-snapcraft: machine: diff --git a/RELEASE.md b/RELEASE.md index 958275cc..cab2909b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -111,8 +111,8 @@ Build the Windows binaries, delete extra files, codesign, and create an MSI pack ``` poetry run python .\setup-freeze.py build poetry run python .\package\windows.py cleanup-build -poetry run python .\package\windows.py codesign -poetry run python .\package\windows.py package +poetry run python .\package\windows.py codesign [build_dir] +poetry run python .\package\windows.py package [build_dir] ``` This will create `desktop/dist/OnionShare-$VERSION.msi`, signed. @@ -124,7 +124,10 @@ Set up the development environment described in `README.md`. Then build an executable, make it a macOS app bundle, and package it in a dmg: ```sh -poetry run ./package/build-mac.py +poetry run python ./setup-freeze.py bdist_mac +poetry run python ./package/macos.py cleanup-build +poetry run python ./package/macos.py codesign [app_path] +poetry run python ./package/macos.py package [app_path] ``` The will create `dist/OnionShare-$VERSION.dmg`. diff --git a/desktop/package/macos.py b/desktop/package/macos.py new file mode 100644 index 00000000..307e8ddc --- /dev/null +++ b/desktop/package/macos.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +import os +import inspect +import click +import subprocess +import shutil +import glob +import itertools + +root = os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) + ) +) +desktop_dir = os.path.join(root, "desktop") + +identity_name_application = "Developer ID Application: Micah Lee (N9B95FDWH4)" +entitlements_plist_path = f"{desktop_dir}/package/Entitlements.plist" + + +def get_app_path(): + return os.path.join(desktop_dir, "build", "OnionShare.app") + + +def run(cmd, cwd=None, error_ok=False): + print(f"{cmd} # cwd={cwd}") + subprocess.run(cmd, cwd=cwd, check=True) + + +def get_size(dir): + size = 0 + for path, dirs, files in os.walk(dir): + for f in files: + fp = os.path.join(path, f) + size += os.path.getsize(fp) + return size + + +def sign(path, entitlements, identity): + run( + [ + "codesign", + "--sign", + identity, + "--entitlements", + str(entitlements), + "--timestamp", + "--deep", + "--force", + "--options", + "runtime,library", + str(path), + ] + ) + + +@click.group() +def main(): + """ + macOS build tasks + """ + + +@main.command() +def cleanup_build(): + """Delete unused PySide2 stuff to save space""" + app_path = get_app_path() + before_size = get_size(app_path) + + print("> Delete unused Qt Frameworks") + for framework in [ + "Qt3DAnimation", + "Qt3DCore", + "Qt3DExtras", + "Qt3DInput", + "Qt3DLogic", + "Qt3DQuick", + "Qt3DQuickAnimation", + "Qt3DQuickExtras", + "Qt3DQuickInput", + "Qt3DQuickRender", + "Qt3DQuickScene2D", + "Qt3DRender", + "QtBluetooth", + "QtBodymovin", + "QtCharts", + "QtConcurrent", + "QtDataVisualization", + "QtDesigner", + "QtDesignerComponents", + "QtGamepad", + "QtHelp", + "QtLocation", + "QtMultimedia", + "QtMultimediaQuick", + "QtMultimediaWidgets", + "QtNetwork", + "QtNetworkAuth", + "QtNfc", + "QtOpenGL", + "QtPdf", + "QtPdfWidgets", + "QtPositioning", + "QtPositioningQuick", + "QtPrintSupport", + "QtPurchasing", + "QtQml", + "QtQmlModels", + "QtQmlWorkerScript", + "QtQuick", + "QtQuick3D", + "QtQuick3DAssetImport", + "QtQuick3DRender", + "QtQuick3DRuntimeRender", + "QtQuick3DUtils", + "QtQuickControls2", + "QtQuickParticles", + "QtQuickShapes", + "QtQuickTemplates2", + "QtQuickTest", + "QtQuickWidgets", + "QtRemoteObjects", + "QtRepParser", + "QtScript", + "QtScriptTools", + "QtScxml", + "QtSensors", + "QtSerialBus", + "QtSerialPort", + "QtSql", + "QtSvg", + "QtTest", + "QtTextToSpeech", + "QtUiPlugin", + "QtVirtualKeyboard", + "QtWebChannel", + "QtWebEngine", + "QtWebEngineCore", + "QtWebEngineWidgets", + "QtWebSockets", + "QtWebView", + "QtXml", + "QtXmlPatterns", + ]: + shutil.rmtree( + f"{app_path}/Contents/MacOS/lib/PySide2/Qt/lib/{framework}.framework" + ) + try: + os.remove(f"{app_path}/Contents/MacOS/lib/PySide2/{framework}.abi3.so") + os.remove(f"{app_path}/Contents/MacOS/lib/PySide2/{framework}.pyi") + except FileNotFoundError: + pass + + print("> Move files around so Apple will notarize") + # https://github.com/marcelotduarte/cx_Freeze/issues/594 + # https://gist.github.com/TechnicalPirate/259a9c24878fcad948452cb148af2a2c#file-custom_bdist_mac-py-L415 + + # Move lib from MacOS into Resources + os.rename( + f"{app_path}/Contents/MacOS/lib", + f"{app_path}/Contents/Resources/lib", + ) + run( + ["ln", "-s", "../Resources/lib"], + cwd=f"{app_path}/Contents/MacOS", + ) + + # Move frameworks from Resources/lib into Frameworks + os.makedirs(f"{app_path}/Contents/Frameworks", exist_ok=True) + for framework_filename in glob.glob( + f"{app_path}/Contents/Resources/lib/PySide2/Qt/lib/Qt*.framework" + ): + basename = os.path.basename(framework_filename) + + os.rename(framework_filename, f"{app_path}/Contents/Frameworks/{basename}") + run( + ["ln", "-s", f"../../../../../Frameworks/{basename}"], + cwd=f"{app_path}/Contents/Resources/lib/PySide2/Qt/lib", + ) + if os.path.exists(f"{app_path}/Contents/Frameworks/{basename}/Resources"): + os.rename( + f"{app_path}/Contents/Frameworks/{basename}/Resources", + f"{app_path}/Contents/Frameworks/{basename}/Versions/5/Resources", + ) + run( + ["ln", "-s", "Versions/5/Resources"], + cwd=f"{app_path}/Contents/Frameworks/{basename}", + ) + + run( + ["ln", "-s", "5", "Current"], + cwd=f"{app_path}/Contents/Frameworks/{basename}/Versions", + ) + + # Move Qt plugins + os.rename( + f"{app_path}/Contents/Resources/lib/PySide2/Qt/plugins", + f"{app_path}/Contents/Frameworks/plugins", + ) + run( + ["ln", "-s", "../../../../Frameworks/plugins"], + cwd=f"{app_path}/Contents/Resources/lib/PySide2/Qt", + ) + + print("> Delete more unused PySide2 stuff to save space") + for filename in [ + f"{app_path}/Contents/Resources/lib/PySide2/Designer.app", + f"{app_path}/Contents/Resources/lib/PySide2/examples", + f"{app_path}/Contents/Resources/lib/PySide2/glue", + f"{app_path}/Contents/Resources/lib/PySide2/include", + f"{app_path}/Contents/Resources/lib/PySide2/pyside2-lupdate", + f"{app_path}/Contents/Resources/lib/PySide2/Qt/qml", + f"{app_path}/Contents/Resources/lib/PySide2/libpyside2.abi3.5.15.dylib", + f"{app_path}/Contents/Resources/lib/PySide2/Qt/lib/QtRepParser.framework", + f"{app_path}/Contents/Resources/lib/PySide2/Qt/lib/QtUiPlugin.framework", + f"{app_path}/Contents/Resources/lib/PySide2/Qt/lib/QtWebEngineCore.framework/Helpers", + f"{app_path}/Contents/Resources/lib/shiboken2/libshiboken2.abi3.5.15.dylib", + f"{app_path}/Contents/Resources/lib/shiboken2/docs", + f"{app_path}/Contents/Resources/lib/PySide2/rcc", + f"{app_path}/Contents/Resources/lib/PySide2/uic", + ]: + if os.path.isdir(filename): + shutil.rmtree(filename) + elif os.path.isfile(filename): + os.remove(filename) + else: + print(f"Cannot delete, filename not found: {filename}") + + after_size = get_size(f"{app_path}") + freed_bytes = before_size - after_size + freed_mb = int(freed_bytes / 1024 / 1024) + print(f"> Freed {freed_mb} mb") + + +@main.command() +@click.argument("app_path") +def codesign(app_path): + """Sign macOS binaries before packaging""" + for path in itertools.chain( + glob.glob(f"{app_path}/Contents/Resources/lib/**/*.so", recursive=True), + glob.glob(f"{app_path}/Contents/Resources/lib/**/*.dylib", recursive=True), + [ + f"{app_path}/Contents/Frameworks/QtCore.framework/Versions/5/QtCore", + f"{app_path}/Contents/Frameworks/QtDBus.framework/Versions/5/QtDBus", + f"{app_path}/Contents/Frameworks/QtGui.framework/Versions/5/QtGui", + f"{app_path}/Contents/Frameworks/QtMacExtras.framework/Versions/5/QtMacExtras", + f"{app_path}/Contents/Frameworks/QtWidgets.framework/Versions/5/QtWidgets", + f"{app_path}/Contents/Resources/lib/Python", + f"{app_path}/Contents/Resources/lib/onionshare/resources/tor/meek-client", + f"{app_path}/Contents/Resources/lib/onionshare/resources/tor/obfs4proxy", + f"{app_path}/Contents/Resources/lib/onionshare/resources/tor/snowflake-client", + f"{app_path}/Contents/Resources/lib/onionshare/resources/tor/tor", + f"{app_path}/Contents/Resources/lib/onionshare/resources/tor/libevent-2.1.7.dylib", + f"{app_path}/Contents/MacOS/onionshare", + f"{app_path}/Contents/MacOS/onionshare-cli", + f"{app_path}", + ], + ): + codesign(path, entitlements_plist_path, identity_name_application) + + print(f"> Signed app bundle: {app_path}") + + +@main.command() +@click.argument("app_path") +def package(app_path): + """Build the DMG package""" + if not os.path.exists("/usr/local/bin/create-dmg"): + print("> Error: create-dmg is not installed") + return + + print("> Create DMG") + version_filename = f"{root}/cli/onionshare_cli/resources/version.txt" + with open(version_filename) as f: + version = f.read().strip() + + os.makedirs(f"{desktop_dir}/dist", exist_ok=True) + dmg_path = f"{desktop_dir}/dist/OnionShare-{version}.dmg" + run( + [ + "create-dmg", + "--volname", + "OnionShare", + "--volicon", + f"{desktop_dir}/onionshare/resources/onionshare.icns", + "--window-size", + "400", + "200", + "--icon-size", + "100", + "--icon", + "OnionShare.app", + "100", + "70", + "--hide-extension", + "OnionShare.app", + "--app-drop-link", + "300", + "70", + dmg_path, + app_path, + "--identity", + identity_name_application, + ] + ) + + print(f"> Finished building DMG: {dmg_path}") + + +if __name__ == "__main__": + main() diff --git a/desktop/package/windows.py b/desktop/package/windows.py index a0bb3db9..215496bd 100644 --- a/desktop/package/windows.py +++ b/desktop/package/windows.py @@ -22,9 +22,7 @@ def get_build_path(): python_arch = "win-amd64" else: python_arch = "win32" - - build_path = os.path.join(desktop_dir, "build", f"exe.{python_arch}-3.9") - return build_path + return os.path.join(desktop_dir, "build", f"exe.{python_arch}-3.9") def get_size(dir): From d54b51e7b1acf2335d6933d244fa3c62225ea36a Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 19:57:47 -0700 Subject: [PATCH 022/154] Install snapcraft before building the snap, and fix some macos cleanup --- .circleci/config.yml | 3 + desktop/package/build-mac.py | 308 ---------------- desktop/package/build-windows.py | 596 ------------------------------- desktop/package/macos.py | 3 - 4 files changed, 3 insertions(+), 907 deletions(-) delete mode 100755 desktop/package/build-mac.py delete mode 100644 desktop/package/build-windows.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f7d763a..fd83ed5e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -214,6 +214,9 @@ jobs: working_directory: ~/repo steps: - checkout + - run: + name: Install snapcraft + command: sudo snap install snapcraft --classic - run: name: Build the snap command: | diff --git a/desktop/package/build-mac.py b/desktop/package/build-mac.py deleted file mode 100755 index 9d2a0dfd..00000000 --- a/desktop/package/build-mac.py +++ /dev/null @@ -1,308 +0,0 @@ -#!/usr/bin/env python3 -import os -import inspect -import subprocess -import shutil -import itertools -import glob - -root = os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) - ) -) - - -def run(cmd, cwd=None, error_ok=False): - print(f"{cmd} # cwd={cwd}") - subprocess.run(cmd, cwd=cwd, check=True) - - -def get_size(dir): - size = 0 - for path, dirs, files in os.walk(dir): - for f in files: - fp = os.path.join(path, f) - size += os.path.getsize(fp) - return size - - -def codesign(path, entitlements, identity): - run( - [ - "codesign", - "--sign", - identity, - "--entitlements", - str(entitlements), - "--timestamp", - "--deep", - "--force", - "--options", - "runtime,library", - str(path), - ] - ) - - -def main(): - desktop_dir = f"{root}/desktop" - app_dir = f"{desktop_dir}/build/OnionShare.app" - - print("○ Clean up from last build") - if os.path.exists(f"{desktop_dir}/build"): - shutil.rmtree(f"{desktop_dir}/build") - if os.path.exists(f"{desktop_dir}/dist"): - shutil.rmtree(f"{desktop_dir}/dist") - - print("○ Building binaries") - run( - [ - shutil.which("python"), - "setup-freeze.py", - "bdist_mac", - ], - desktop_dir, - ) - before_size = get_size(f"{app_dir}") - - print("○ Delete unused Qt Frameworks") - for framework in [ - "Qt3DAnimation", - "Qt3DCore", - "Qt3DExtras", - "Qt3DInput", - "Qt3DLogic", - "Qt3DQuick", - "Qt3DQuickAnimation", - "Qt3DQuickExtras", - "Qt3DQuickInput", - "Qt3DQuickRender", - "Qt3DQuickScene2D", - "Qt3DRender", - "QtBluetooth", - "QtBodymovin", - "QtCharts", - "QtConcurrent", - "QtDataVisualization", - "QtDesigner", - "QtDesignerComponents", - "QtGamepad", - "QtHelp", - "QtLocation", - "QtMultimedia", - "QtMultimediaQuick", - "QtMultimediaWidgets", - "QtNetwork", - "QtNetworkAuth", - "QtNfc", - "QtOpenGL", - "QtPdf", - "QtPdfWidgets", - "QtPositioning", - "QtPositioningQuick", - "QtPrintSupport", - "QtPurchasing", - "QtQml", - "QtQmlModels", - "QtQmlWorkerScript", - "QtQuick", - "QtQuick3D", - "QtQuick3DAssetImport", - "QtQuick3DRender", - "QtQuick3DRuntimeRender", - "QtQuick3DUtils", - "QtQuickControls2", - "QtQuickParticles", - "QtQuickShapes", - "QtQuickTemplates2", - "QtQuickTest", - "QtQuickWidgets", - "QtRemoteObjects", - "QtRepParser", - "QtScript", - "QtScriptTools", - "QtScxml", - "QtSensors", - "QtSerialBus", - "QtSerialPort", - "QtSql", - "QtSvg", - "QtTest", - "QtTextToSpeech", - "QtUiPlugin", - "QtVirtualKeyboard", - "QtWebChannel", - "QtWebEngine", - "QtWebEngineCore", - "QtWebEngineWidgets", - "QtWebSockets", - "QtWebView", - "QtXml", - "QtXmlPatterns", - ]: - shutil.rmtree( - f"{app_dir}/Contents/MacOS/lib/PySide2/Qt/lib/{framework}.framework" - ) - try: - os.remove( - f"{app_dir}/Contents/MacOS/lib/PySide2/{framework}.abi3.so" - ) - os.remove( - f"{app_dir}/Contents/MacOS/lib/PySide2/{framework}.pyi" - ) - except FileNotFoundError: - pass - - print("○ Move files around so Apple will notarize") - # https://github.com/marcelotduarte/cx_Freeze/issues/594 - # https://gist.github.com/TechnicalPirate/259a9c24878fcad948452cb148af2a2c#file-custom_bdist_mac-py-L415 - - # Move lib from MacOS into Resources - os.rename( - f"{app_dir}/Contents/MacOS/lib", - f"{app_dir}/Contents/Resources/lib", - ) - run( - ["ln", "-s", "../Resources/lib"], - cwd=f"{app_dir}/Contents/MacOS", - ) - - # Move frameworks from Resources/lib into Frameworks - os.makedirs(f"{app_dir}/Contents/Frameworks", exist_ok=True) - for framework_filename in glob.glob( - f"{app_dir}/Contents/Resources/lib/PySide2/Qt/lib/Qt*.framework" - ): - basename = os.path.basename(framework_filename) - - os.rename(framework_filename, f"{app_dir}/Contents/Frameworks/{basename}") - run( - ["ln", "-s", f"../../../../../Frameworks/{basename}"], - cwd=f"{app_dir}/Contents/Resources/lib/PySide2/Qt/lib", - ) - if os.path.exists(f"{app_dir}/Contents/Frameworks/{basename}/Resources"): - os.rename( - f"{app_dir}/Contents/Frameworks/{basename}/Resources", - f"{app_dir}/Contents/Frameworks/{basename}/Versions/5/Resources" - ) - run( - ["ln", "-s", "Versions/5/Resources"], - cwd=f"{app_dir}/Contents/Frameworks/{basename}", - ) - - run( - ["ln", "-s", "5", "Current"], - cwd=f"{app_dir}/Contents/Frameworks/{basename}/Versions", - ) - - # Move Qt plugins - os.rename( - f"{app_dir}/Contents/Resources/lib/PySide2/Qt/plugins", - f"{app_dir}/Contents/Frameworks/plugins", - ) - run( - ["ln", "-s", "../../../../Frameworks/plugins"], - cwd=f"{app_dir}/Contents/Resources/lib/PySide2/Qt", - ) - - print("○ Delete more unused PySide2 stuff to save space") - for filename in [ - f"{app_dir}/Contents/Resources/lib/PySide2/Designer.app", - f"{app_dir}/Contents/Resources/lib/PySide2/examples", - f"{app_dir}/Contents/Resources/lib/PySide2/glue", - f"{app_dir}/Contents/Resources/lib/PySide2/include", - f"{app_dir}/Contents/Resources/lib/PySide2/pyside2-lupdate", - f"{app_dir}/Contents/Resources/lib/PySide2/Qt/qml", - f"{app_dir}/Contents/Resources/lib/PySide2/libpyside2.abi3.5.15.dylib", - f"{app_dir}/Contents/Resources/lib/PySide2/Qt/lib/QtRepParser.framework", - f"{app_dir}/Contents/Resources/lib/PySide2/Qt/lib/QtUiPlugin.framework", - f"{app_dir}/Contents/Resources/lib/PySide2/Qt/lib/QtWebEngineCore.framework/Helpers", - f"{app_dir}/Contents/Resources/lib/shiboken2/libshiboken2.abi3.5.15.dylib", - f"{app_dir}/Contents/Resources/lib/shiboken2/docs", - f"{app_dir}/Contents/Resources/lib/PySide2/rcc", - f"{app_dir}/Contents/Resources/lib/PySide2/uic", - ]: - if os.path.isdir(filename): - shutil.rmtree(filename) - elif os.path.isfile(filename): - os.remove(filename) - else: - print(f"Cannot delete, filename not found: {filename}") - - after_size = get_size(f"{app_dir}") - freed_bytes = before_size - after_size - freed_mb = int(freed_bytes / 1024 / 1024) - print(f"○ Freed {freed_mb} mb") - - print("○ Sign app bundle") - identity_name_application = "Developer ID Application: Micah Lee (N9B95FDWH4)" - entitlements_plist_path = f"{desktop_dir}/package/Entitlements.plist" - - for path in itertools.chain( - glob.glob(f"{app_dir}/Contents/Resources/lib/**/*.so", recursive=True), - glob.glob(f"{app_dir}/Contents/Resources/lib/**/*.dylib", recursive=True), - [ - f"{app_dir}/Contents/Frameworks/QtCore.framework/Versions/5/QtCore", - f"{app_dir}/Contents/Frameworks/QtDBus.framework/Versions/5/QtDBus", - f"{app_dir}/Contents/Frameworks/QtGui.framework/Versions/5/QtGui", - f"{app_dir}/Contents/Frameworks/QtMacExtras.framework/Versions/5/QtMacExtras", - f"{app_dir}/Contents/Frameworks/QtWidgets.framework/Versions/5/QtWidgets", - f"{app_dir}/Contents/Resources/lib/Python", - f"{app_dir}/Contents/Resources/lib/onionshare/resources/tor/meek-client", - f"{app_dir}/Contents/Resources/lib/onionshare/resources/tor/obfs4proxy", - f"{app_dir}/Contents/Resources/lib/onionshare/resources/tor/snowflake-client", - f"{app_dir}/Contents/Resources/lib/onionshare/resources/tor/tor", - f"{app_dir}/Contents/Resources/lib/onionshare/resources/tor/libevent-2.1.7.dylib", - f"{app_dir}/Contents/MacOS/onionshare", - f"{app_dir}/Contents/MacOS/onionshare-cli", - f"{app_dir}", - ], - ): - codesign(path, entitlements_plist_path, identity_name_application) - - print(f"○ Signed app bundle: {app_dir}") - - if not os.path.exists("/usr/local/bin/create-dmg"): - print("○ Error: create-dmg is not installed") - return - - print("○ Create DMG") - version_filename = f"{root}/cli/onionshare_cli/resources/version.txt" - with open(version_filename) as f: - version = f.read().strip() - - os.makedirs(f"{desktop_dir}/dist", exist_ok=True) - dmg_path = f"{desktop_dir}/dist/OnionShare-{version}.dmg" - run( - [ - "create-dmg", - "--volname", - "OnionShare", - "--volicon", - f"{desktop_dir}/onionshare/resources/onionshare.icns", - "--window-size", - "400", - "200", - "--icon-size", - "100", - "--icon", - "OnionShare.app", - "100", - "70", - "--hide-extension", - "OnionShare.app", - "--app-drop-link", - "300", - "70", - dmg_path, - f"{app_dir}", - "--identity", - identity_name_application, - ] - ) - - print(f"○ Finished building DMG: {dmg_path}") - - -if __name__ == "__main__": - main() diff --git a/desktop/package/build-windows.py b/desktop/package/build-windows.py deleted file mode 100644 index eb7d0218..00000000 --- a/desktop/package/build-windows.py +++ /dev/null @@ -1,596 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import inspect -import subprocess -import shutil -import uuid -import xml.etree.ElementTree as ET - - -root = os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) - ) -) - - -def run(cmd, cwd=None, error_ok=False): - print(cmd) - try: - subprocess.run(cmd, cwd=cwd, check=True) - except subprocess.CalledProcessError as e: - if not error_ok: - raise subprocess.CalledProcessError(e) - - -def sign(filename, cwd=None): - run( - [ - shutil.which("signtool"), - "sign", - "/v", - "/d", - "OnionShare", - "/sha1", - "bb1d265ab02272e8fc742f149dcf8751cac63f50", - "/fd", - "SHA256", - "/td", - "SHA256", - "/tr", - "http://timestamp.digicert.com", - filename, - ], - cwd, - ) - - -def get_size(dir): - size = 0 - for path, dirs, files in os.walk(dir): - for f in files: - fp = os.path.join(path, f) - size += os.path.getsize(fp) - return size - - -def wix_build_data(dirname, dir_prefix, id_, name): - data = { - "id": id_, - "name": name, - "files": [], - "dirs": [], - } - - for basename in os.listdir(dirname): - filename = os.path.join(dirname, basename) - if os.path.isfile(filename): - data["files"].append(os.path.join(dir_prefix, basename)) - elif os.path.isdir(filename): - if id_ == "INSTALLDIR": - id_prefix = "Folder" - else: - id_prefix = id_ - - # Skip lib/Pyside2/Examples folder - if "\\lib\\PySide2\\examples" in dirname: - continue - - id_value = f"{id_prefix}{basename.capitalize().replace('-', '_')}" - data["dirs"].append( - wix_build_data( - os.path.join(dirname, basename), - os.path.join(dir_prefix, basename), - id_value, - basename, - ) - ) - - if len(data["files"]) > 0: - if id_ == "INSTALLDIR": - data["component_id"] = "ApplicationFiles" - else: - data["component_id"] = "FolderComponent" + id_[len("Folder") :] - data["component_guid"] = str(uuid.uuid4()) - - return data - - -def wix_build_dir_xml(root, data): - attrs = {} - if "id" in data: - attrs["Id"] = data["id"] - if "name" in data: - attrs["Name"] = data["name"] - el = ET.SubElement(root, "Directory", attrs) - for subdata in data["dirs"]: - wix_build_dir_xml(el, subdata) - - # If this is the ProgramMenuFolder, add the menu component - if "id" in data and data["id"] == "ProgramMenuFolder": - component_el = ET.SubElement( - el, - "Component", - Id="ApplicationShortcuts", - Guid="539e7de8-a124-4c09-aa55-0dd516aad7bc", - ) - ET.SubElement( - component_el, - "Shortcut", - Id="ApplicationShortcut1", - Name="OnionShare", - Description="OnionShare", - Target="[INSTALLDIR]onionshare.exe", - WorkingDirectory="INSTALLDIR", - ) - ET.SubElement( - component_el, - "RegistryValue", - Root="HKCU", - Key="Software\OnionShare", - Name="installed", - Type="integer", - Value="1", - KeyPath="yes", - ) - - -def wix_build_components_xml(root, data): - component_ids = [] - if "component_id" in data: - component_ids.append(data["component_id"]) - - for subdata in data["dirs"]: - if "component_guid" in subdata: - dir_ref_el = ET.SubElement(root, "DirectoryRef", Id=subdata["id"]) - component_el = ET.SubElement( - dir_ref_el, - "Component", - Id=subdata["component_id"], - Guid=subdata["component_guid"], - ) - for filename in subdata["files"]: - file_el = ET.SubElement( - component_el, "File", Source=filename, Id="file_" + uuid.uuid4().hex - ) - - component_ids += wix_build_components_xml(root, subdata) - - return component_ids - - -def main(): - # Figure out the architecture and python path - if "64 bit" in sys.version: - python_arch = "win-amd64" - else: - python_arch = "win32" - - if os.getlogin() == "circleci" and python_arch == "win32": - python_path = "C:\\Python-32bit\\python" - else: - python_path = shutil.which("python") - - desktop_dir = os.path.join(root, "desktop") - - print("> Clean up from last build") - if os.path.exists(os.path.join(desktop_dir, "build")): - shutil.rmtree(os.path.join(desktop_dir, "build")) - if os.path.exists(os.path.join(desktop_dir, "dist")): - shutil.rmtree(os.path.join(desktop_dir, "dist")) - - print("> Building binaries") - run( - [ - python_path, - "setup-freeze.py", - "build", - ], - desktop_dir, - ) - - build_path = os.path.join(desktop_dir, "build", f"exe.{python_arch}-3.9") - - before_size = get_size(build_path) - - print("> Delete unused PySide2 stuff to save space") - for dirname in ["examples", "qml"]: - shutil.rmtree(os.path.join(build_path, "lib", "PySide2", dirname)) - for filename in [ - "lconvert.exe", - "linguist.exe", - "lrelease.exe", - "lupdate.exe", - "plugins/assetimporters/assimp.dll", - "plugins/assetimporters/uip.dll", - "plugins/audio/qtaudio_wasapi.dll", - "plugins/audio/qtaudio_windows.dll", - "plugins/bearer/qgenericbearer.dll", - "plugins/canbus/qtpassthrucanbus.dll", - "plugins/canbus/qtpeakcanbus.dll", - "plugins/canbus/qtsysteccanbus.dll", - "plugins/canbus/qttinycanbus.dll", - "plugins/canbus/qtvectorcanbus.dll", - "plugins/canbus/qtvirtualcanbus.dll", - "plugins/gamepads/xinputgamepad.dll", - "plugins/generic/qtuiotouchplugin.dll", - "plugins/geometryloaders/defaultgeometryloader.dll", - "plugins/geometryloaders/gltfgeometryloader.dll", - "plugins/geoservices/qtgeoservices_esri.dll", - "plugins/geoservices/qtgeoservices_itemsoverlay.dll", - "plugins/geoservices/qtgeoservices_mapbox.dll", - "plugins/geoservices/qtgeoservices_nokia.dll", - "plugins/geoservices/qtgeoservices_osm.dll", - "plugins/mediaservice/dsengine.dll", - "plugins/mediaservice/qtmedia_audioengine.dll", - "plugins/mediaservice/wmfengine.dll", - "plugins/platforminputcontexts/qtvirtualkeyboardplugin.dll", - "plugins/platforms/qdirect2d.dll", - "plugins/platforms/qoffscreen.dll", - "plugins/platforms/qwebgl.dll", - "plugins/platformthemes/qxdgdesktopportal.dll", - "plugins/playlistformats/qtmultimedia_m3u.dll", - "plugins/position/qtposition_positionpoll.dll", - "plugins/position/qtposition_serialnmea.dll", - "plugins/position/qtposition_winrt.dll", - "plugins/printsupport/windowsprintersupport.dll", - "plugins/qmltooling/qmldbg_debugger.dll", - "plugins/qmltooling/qmldbg_inspector.dll", - "plugins/qmltooling/qmldbg_local.dll", - "plugins/qmltooling/qmldbg_messages.dll", - "plugins/qmltooling/qmldbg_native.dll", - "plugins/qmltooling/qmldbg_nativedebugger.dll", - "plugins/qmltooling/qmldbg_preview.dll", - "plugins/qmltooling/qmldbg_profiler.dll", - "plugins/qmltooling/qmldbg_quickprofiler.dll", - "plugins/qmltooling/qmldbg_server.dll", - "plugins/qmltooling/qmldbg_tcp.dll", - "plugins/renderers/openglrenderer.dll", - "plugins/renderplugins/scene2d.dll", - "plugins/scenegraph/qsgd3d12backend.dll", - "plugins/sceneparsers/gltfsceneexport.dll", - "plugins/sceneparsers/gltfsceneimport.dll", - "plugins/sensorgestures/qtsensorgestures_plugin.dll", - "plugins/sensorgestures/qtsensorgestures_shakeplugin.dll", - "plugins/sensors/qtsensors_generic.dll", - "plugins/sqldrivers/qsqlite.dll", - "plugins/sqldrivers/qsqlodbc.dll", - "plugins/sqldrivers/qsqlpsql.dll", - "plugins/styles/qwindowsvistastyle.dll", - "plugins/texttospeech/qtexttospeech_sapi.dll", - "plugins/virtualkeyboard/qtvirtualkeyboard_hangul.dll", - "plugins/virtualkeyboard/qtvirtualkeyboard_openwnn.dll", - "plugins/virtualkeyboard/qtvirtualkeyboard_pinyin.dll", - "plugins/virtualkeyboard/qtvirtualkeyboard_tcime.dll", - "plugins/virtualkeyboard/qtvirtualkeyboard_thai.dll", - "plugins/webview/qtwebview_webengine.dll", - "pyside2-lupdate.exe", - "Qt3DAnimation.pyd", - "Qt3DAnimation.pyi", - "Qt3DCore.pyd", - "Qt3DCore.pyi", - "Qt3DExtras.pyd", - "Qt3DExtras.pyi", - "Qt3DInput.pyd", - "Qt3DInput.pyi", - "Qt3DLogic.pyd", - "Qt3DLogic.pyi", - "Qt3DRender.pyd", - "Qt3DRender.pyi", - "Qt53DAnimation.dll", - "Qt53DCore.dll", - "Qt53DExtras.dll", - "Qt53DInput.dll", - "Qt53DLogic.dll", - "Qt53DQuick.dll", - "Qt53DQuickAnimation.dll", - "Qt53DQuickExtras.dll", - "Qt53DQuickInput.dll", - "Qt53DQuickRender.dll", - "Qt53DQuickScene2D.dll", - "Qt53DRender.dll", - "Qt5Bluetooth.dll", - "Qt5Bodymovin.dll", - "Qt5Charts.dll", - "Qt5Concurrent.dll", - "Qt5DataVisualization.dll", - "Qt5DBus.dll", - "Qt5Designer.dll", - "Qt5DesignerComponents.dll", - "Qt5Gamepad.dll", - "Qt5Help.dll", - "Qt5Location.dll", - "Qt5Multimedia.dll", - "Qt5MultimediaQuick.dll", - "Qt5MultimediaWidgets.dll", - "Qt5Nfc.dll", - "Qt5OpenGL.dll", - "Qt5Pdf.dll", - "Qt5PdfWidgets.dll", - "Qt5Positioning.dll", - "Qt5PositioningQuick.dll", - "Qt5PrintSupport.dll", - "Qt5Purchasing.dll", - "Qt5Quick.dll", - "Qt5Quick3D.dll", - "Qt5Quick3DAssetImport.dll", - "Qt5Quick3DRender.dll", - "Qt5Quick3DRuntimeRender.dll", - "Qt5Quick3DUtils.dll", - "Qt5QuickControls2.dll", - "Qt5QuickParticles.dll", - "Qt5QuickShapes.dll", - "Qt5QuickTemplates2.dll", - "Qt5QuickTest.dll", - "Qt5QuickWidgets.dll", - "Qt5RemoteObjects.dll", - "Qt5Script.dll", - "Qt5ScriptTools.dll", - "Qt5Scxml.dll", - "Qt5Sensors.dll", - "Qt5SerialBus.dll", - "Qt5SerialPort.dll", - "Qt5Sql.dll", - "Qt5Svg.dll", - "Qt5Test.dll", - "Qt5TextToSpeech.dll", - "Qt5VirtualKeyboard.dll", - "Qt5WebChannel.dll", - "Qt5WebEngine.dll", - "Qt5WebEngineCore.dll", - "Qt5WebEngineWidgets.dll", - "Qt5WebSockets.dll", - "Qt5WebView.dll", - "Qt5Xml.dll", - "Qt5XmlPatterns.dll", - "QtAxContainer.pyd", - "QtAxContainer.pyi", - "QtCharts.pyd", - "QtCharts.pyi", - "QtConcurrent.pyd", - "QtConcurrent.pyi", - "QtDataVisualization.pyd", - "QtDataVisualization.pyi", - "qtdiag.exe", - "QtHelp.pyd", - "QtHelp.pyi", - "QtLocation.pyd", - "QtLocation.pyi", - "QtMultimedia.pyd", - "QtMultimedia.pyi", - "QtMultimediaWidgets.pyd", - "QtMultimediaWidgets.pyi", - "QtNetwork.pyd", - "QtNetwork.pyi", - "QtOpenGL.pyd", - "QtOpenGL.pyi", - "QtOpenGLFunctions.pyd", - "QtOpenGLFunctions.pyi", - "QtPositioning.pyd", - "QtPositioning.pyi", - "QtPrintSupport.pyd", - "QtPrintSupport.pyi", - "QtQml.pyd", - "QtQml.pyi", - "QtQuick.pyd", - "QtQuick.pyi", - "QtQuickControls2.pyd", - "QtQuickControls2.pyi", - "QtQuickWidgets.pyd", - "QtQuickWidgets.pyi", - "QtRemoteObjects.pyd", - "QtRemoteObjects.pyi", - "QtScript.pyd", - "QtScript.pyi", - "QtScriptTools.pyd", - "QtScriptTools.pyi", - "QtScxml.pyd", - "QtScxml.pyi", - "QtSensors.pyd", - "QtSensors.pyi", - "QtSerialPort.pyd", - "QtSerialPort.pyi", - "QtSql.pyd", - "QtSql.pyi", - "QtSvg.pyd", - "QtSvg.pyi", - "QtTest.pyd", - "QtTest.pyi", - "QtTextToSpeech.pyd", - "QtTextToSpeech.pyi", - "QtUiTools.pyd", - "QtUiTools.pyi", - "QtWebChannel.pyd", - "QtWebChannel.pyi", - "QtWebEngine.pyd", - "QtWebEngine.pyi", - "QtWebEngineCore.pyd", - "QtWebEngineCore.pyi", - "QtWebEngineProcess.exe", - "QtWebEngineWidgets.pyd", - "QtWebEngineWidgets.pyi", - "QtWebSockets.pyd", - "QtWebSockets.pyi", - "QtWinExtras.pyd", - "QtWinExtras.pyi", - "QtXml.pyd", - "QtXml.pyi", - "QtXmlPatterns.pyd", - "QtXmlPatterns.pyi", - "rcc.exe", - "uic.exe", - ]: - os.remove( - os.path.join( - build_path, - "lib", - "PySide2", - filename.replace("/", "\\"), - ) - ) - - after_size = get_size(build_path) - freed_bytes = before_size - after_size - freed_mb = int(freed_bytes / 1024 / 1024) - print(f"> Freed {freed_mb} mb") - - print(f"> Signing onionshare.exe") - sign(os.path.join(build_path, "onionshare.exe"), desktop_dir) - - print(f"> Signing onionshare-cli.exe") - sign(os.path.join(build_path, "onionshare-cli.exe"), desktop_dir) - - print(f"> Build the WiX file") - version_filename = os.path.join( - root, "cli", "onionshare_cli", "resources", "version.txt" - ) - with open(version_filename) as f: - version = f.read().strip() - - dist_dir = os.path.join(root, build_path) - - data = { - "id": "TARGETDIR", - "name": "SourceDir", - "dirs": [ - { - "id": "ProgramFilesFolder", - "dirs": [], - }, - { - "id": "ProgramMenuFolder", - "dirs": [], - }, - ], - } - - data["dirs"][0]["dirs"].append( - wix_build_data( - build_path, - "INSTALLDIR", - "OnionShare", - ) - ) - - root_el = ET.Element("Wix", xmlns="http://schemas.microsoft.com/wix/2006/wi") - product_el = ET.SubElement( - root_el, - "Product", - Name="OnionShare", - Manufacturer="Micah Lee, et al.", - Id="*", - UpgradeCode="$(var.ProductUpgradeCode)", - Language="1033", - Codepage="1252", - Version="$(var.ProductVersion)", - ) - ET.SubElement( - product_el, - "Package", - Id="*", - Keywords="Installer", - Description="OnionShare $(var.ProductVersion) Installer", - Manufacturer="Micah Lee, et al.", - InstallerVersion="100", - Languages="1033", - Compressed="yes", - SummaryCodepage="1252", - ) - ET.SubElement( - product_el, "Media", Id="1", Cabinet="product.cab", EmbedCab="yes" - ) - ET.SubElement( - product_el, - "Icon", - Id="ProductIcon", - SourceFile="..\\onionshare\\resources\\onionshare.ico", - ) - ET.SubElement(product_el, "Property", Id="ARPPRODUCTICON", Value="ProductIcon") - ET.SubElement( - product_el, - "Property", - Id="ARPHELPLINK", - Value="https://docs.onionshare.org", - ) - ET.SubElement( - product_el, - "Property", - Id="ARPURLINFOABOUT", - Value="https://onionshare.org", - ) - ET.SubElement(product_el, "UIRef", Id="WixUI_Minimal") - ET.SubElement(product_el, "UIRef", Id="WixUI_ErrorProgressText") - ET.SubElement( - product_el, - "WixVariable", - Id="WixUILicenseRtf", - Value="..\\package\\license.rtf", - ) - ET.SubElement( - product_el, - "WixVariable", - Id="WixUIDialogBmp", - Value="..\\package\\dialog.bmp", - ) - ET.SubElement( - product_el, - "MajorUpgrade", - AllowSameVersionUpgrades="yes", - DowngradeErrorMessage="A newer version of [ProductName] is already installed. If you are sure you want to downgrade, remove the existing installation via Programs and Features.", - ) - - wix_build_dir_xml(product_el, data) - component_ids = wix_build_components_xml(product_el, data) - - feature_el = ET.SubElement( - product_el, "Feature", Id="DefaultFeature", Level="1" - ) - for component_id in component_ids: - ET.SubElement(feature_el, "ComponentRef", Id=component_id) - ET.SubElement(feature_el, "ComponentRef", Id="ApplicationShortcuts") - - with open(os.path.join(root, "desktop", "build", "OnionShare.wxs"), "w") as f: - f.write('\n') - f.write(f'\n') - f.write( - '\n' - ) - - ET.indent(root_el) - f.write(ET.tostring(root_el).decode()) - - print(f"> Build the MSI") - run( - [shutil.which("candle.exe"), "OnionShare.wxs"], - os.path.join(desktop_dir, "build"), - ) - run( - [shutil.which("light.exe"), "-ext", "WixUIExtension", "OnionShare.wixobj"], - os.path.join(desktop_dir, "build"), - ) - - print(f"> Prepare OnionShare.msi for signing") - run( - [ - shutil.which("insignia.exe"), - "-im", - os.path.join(desktop_dir, "build", "OnionShare.msi"), - ], - error_ok=True, - ) - sign(os.path.join(desktop_dir, "build", "OnionShare.msi")) - - final_msi_filename = os.path.join( - desktop_dir, "dist", f"OnionShare-{version}.msi" - ) - print(f"> Final MSI: {final_msi_filename}") - os.makedirs(os.path.join(desktop_dir, "dist"), exist_ok=True) - os.rename( - os.path.join(desktop_dir, "build", "OnionShare.msi"), - final_msi_filename, - ) - - -if __name__ == "__main__": - main() diff --git a/desktop/package/macos.py b/desktop/package/macos.py index 307e8ddc..f9fc25f1 100644 --- a/desktop/package/macos.py +++ b/desktop/package/macos.py @@ -120,7 +120,6 @@ def cleanup_build(): "QtQuickTest", "QtQuickWidgets", "QtRemoteObjects", - "QtRepParser", "QtScript", "QtScriptTools", "QtScxml", @@ -131,11 +130,9 @@ def cleanup_build(): "QtSvg", "QtTest", "QtTextToSpeech", - "QtUiPlugin", "QtVirtualKeyboard", "QtWebChannel", "QtWebEngine", - "QtWebEngineCore", "QtWebEngineWidgets", "QtWebSockets", "QtWebView", From fa5d5b7cdd4a79f2a20d6b765a2c5093af00a856 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 21:04:08 -0700 Subject: [PATCH 023/154] Use docker for snapcraft --- .circleci/config.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fd83ed5e..82baf0bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -209,19 +209,16 @@ jobs: path: ~/onionshare-macos.zip build-snapcraft: - machine: - image: ubuntu-2004:current + docker: + - image: snapcore/snapcraft:edge working_directory: ~/repo steps: - checkout - - run: - name: Install snapcraft - command: sudo snap install snapcraft --classic - run: name: Build the snap command: | - cd ~/repo + cd ~/repo/desktop snapcraft - mv onionshare-*_amd64.snap onionshare_amd64.snap + mv onionshare-*_amd64.snap ~/onionshare_amd64.snap - store_artifacts: - path: ~/repo/onionshare_amd64.snap + path: ~/onionshare_amd64.snap From 562d213ec5dab7724d069203f11e6f2cdd23ff0f Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 21:13:42 -0700 Subject: [PATCH 024/154] Add some exception handling to the symlink creation --- desktop/package/macos.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/desktop/package/macos.py b/desktop/package/macos.py index f9fc25f1..4d4c02e0 100644 --- a/desktop/package/macos.py +++ b/desktop/package/macos.py @@ -184,10 +184,13 @@ def cleanup_build(): cwd=f"{app_path}/Contents/Frameworks/{basename}", ) - run( - ["ln", "-s", "5", "Current"], - cwd=f"{app_path}/Contents/Frameworks/{basename}/Versions", - ) + try: + run( + ["ln", "-s", "5", "Current"], + cwd=f"{app_path}/Contents/Frameworks/{basename}/Versions", + ) + except: + pass # Move Qt plugins os.rename( From ee9a5269bb45499772a904616c7585f5318bf320 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 31 Mar 2022 21:15:48 -0700 Subject: [PATCH 025/154] Oops, change to correct folder for snapcraft --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 82baf0bb..5cf11bfa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -217,7 +217,7 @@ jobs: - run: name: Build the snap command: | - cd ~/repo/desktop + cd ~/repo snapcraft mv onionshare-*_amd64.snap ~/onionshare_amd64.snap - store_artifacts: From 3fd96cd472c81e3a765b79adc0bcd104c4da42fe Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 11:15:53 -0700 Subject: [PATCH 026/154] Combine all get-tor scripts into a single get-tor.py --- desktop/README.md | 50 ++--- desktop/scripts/bridges/__init__.py | 74 ------- desktop/scripts/get-tor-linux.py | 135 ------------ desktop/scripts/get-tor-osx.py | 123 ----------- desktop/scripts/get-tor-windows.py | 108 --------- desktop/scripts/get-tor.py | 326 ++++++++++++++++++++++++++++ 6 files changed, 341 insertions(+), 475 deletions(-) delete mode 100644 desktop/scripts/bridges/__init__.py delete mode 100755 desktop/scripts/get-tor-linux.py delete mode 100755 desktop/scripts/get-tor-osx.py delete mode 100644 desktop/scripts/get-tor-windows.py create mode 100644 desktop/scripts/get-tor.py diff --git a/desktop/README.md b/desktop/README.md index af6a83f1..31071782 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -9,62 +9,42 @@ git clone https://github.com/onionshare/onionshare.git cd onionshare/desktop ``` -Make sure you have Python 3 installed. If you're using Windows or macOS, install version 3.9.9 [from python.org](https://www.python.org/downloads/release/python-3912/). For Windows, make sure to install the 32-bit (x86) version, and to check the box to add python to the path on the first page of the installer. +Make sure you have Python 3 installed. If you're using Windows or macOS, install version 3.9.12 [from python.org](https://www.python.org/downloads/release/python-3912/). For Windows, make sure to check the box to add python to the path on the first page of the installer. -Make sure you have [poetry installed](https://python-poetry.org/docs/#installation), and then install the dependencies: +Make sure you have [poetry](https://python-poetry.org/) installed: + +``` +pip3 install poetry +``` + +And install the poetry dependencies: ```sh poetry install ``` -In Windows, you may need to install [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/), making sure to check "Desktop development with C++", before `poetry install` will work properly. +**Windows users:** You may need to install [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/), making sure to check "Desktop development with C++", before `poetry install` will work properly. -### Install platform-specific dependencies +### Get Tor -#### Linux +**Linux users:** In Ubuntu 20.04 you need the `libxcb-xinerama0` package installed. -In Ubuntu 20.04 you need the `libxcb-xinerama0` package installed. +**Windows users:** Download and install 7-Zip from https://7-zip.org/download.html. [Add](https://medium.com/@kevinmarkvi/how-to-add-executables-to-your-path-in-windows-5ffa4ce61a53) `C:\Program Files (x86)\7-Zip` to your path. Download Tor Browser and extract the binaries: ```sh -poetry run ./scripts/get-tor-linux.py -``` - -#### macOS - -Download Tor Browser and extract the binaries: - -```sh -poetry run ./scripts/get-tor-osx.py -``` - -#### Windows - -These instructions include adding folders to the path in Windows. To do this, go to Start and type "advanced system settings", and open "View advanced system settings" in the Control Panel. Click Environment Variables. Under "System variables" double-click on Path. From there you can add and remove folders that are available in the PATH. - -Download and install 7-Zip from https://7-zip.org/download.html. I downloaded `7z1900.exe`. Add `C:\Program Files (x86)\7-Zip` to your path. - -Download Tor Browser and extract the binaries: - -```sh -poetry run python scripts\get-tor-windows.py +poetry run python ./scripts/get-tor.py ``` ### Compile dependencies -Install Go. The simplest way to make sure everything works is to install Go by following [these instructions](https://golang.org/doc/install). (In Windows, make sure to install the 32-bit version of Go, such as `go1.17.5.windows-386.msi`.) +Install Go. The simplest way to make sure everything works is to install Go by following [these instructions](https://golang.org/doc/install). Download and compile `meek-client`: ```sh -./scripts/build-meek-client.py -``` - -Or in Windows: - -```powershell -python .\scripts\build-meek-client.py +poetry run python ./scripts/build-meek-client.py ``` ### Running OnionShare from the source code tree diff --git a/desktop/scripts/bridges/__init__.py b/desktop/scripts/bridges/__init__.py deleted file mode 100644 index f9d3c9a9..00000000 --- a/desktop/scripts/bridges/__init__.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -OnionShare | https://onionshare.org/ - -Copyright (C) 2014-2022 Micah Lee, et al. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -import os -import requests - - -class UpdateTorBridges: - """ - Update the built-in Tor Bridges in OnionShare's torrc templates. - """ - - def __init__(self, root_path): - self.root_path = root_path - torrc_template_dir = os.path.join( - self.root_path, os.pardir, "cli/onionshare_cli/resources" - ) - endpoint = "https://bridges.torproject.org/moat/circumvention/builtin" - r = requests.post( - endpoint, - headers={"Content-Type": "application/vnd.api+json"}, - ) - if r.status_code != 200: - print( - f"There was a problem fetching the latest built-in bridges: status_code={r.status_code}" - ) - return False - - result = r.json() - - if "errors" in result: - print( - f"There was a problem fetching the latest built-in bridges: errors={result['errors']}" - ) - return False - - for bridge_type in ["meek-azure", "obfs4", "snowflake"]: - if result[bridge_type]: - if bridge_type == "meek-azure": - torrc_template_extension = "meek_lite_azure" - else: - torrc_template_extension = bridge_type - torrc_template = os.path.join( - self.root_path, - torrc_template_dir, - f"torrc_template-{torrc_template_extension}", - ) - - with open(torrc_template, "w") as f: - f.write(f"# Enable built-in {bridge_type} bridge\n") - bridges = result[bridge_type] - # Sorts the bridges numerically by IP, since they come back in - # random order from the API each time, and create noisy git diff. - bridges.sort(key=lambda s: s.split()[1]) - for item in bridges: - f.write(f"Bridge {item}\n") diff --git a/desktop/scripts/get-tor-linux.py b/desktop/scripts/get-tor-linux.py deleted file mode 100755 index f7b96b2e..00000000 --- a/desktop/scripts/get-tor-linux.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -OnionShare | https://onionshare.org/ - -Copyright (C) 2014-2022 Micah Lee, et al. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -""" -This script downloads a pre-built tor binary to bundle with OnionShare. -In order to avoid a Mac gnupg dependency, I manually verify the signature -and hard-code the sha256 hash. -""" -import inspect -import os -import sys -import hashlib -import shutil -import subprocess -import requests -from bridges import UpdateTorBridges - - -def main(): - tarball_url = "https://dist.torproject.org/torbrowser/11.0.9/tor-browser-linux64-11.0.9_en-US.tar.xz" - tarball_filename = "tor-browser-linux64-11.0.9_en-US.tar.xz" - expected_tarball_sha256 = ( - "baa5ccafb5c68f1c46f9ae983b9b0a0419f66d41e0483ba5aacb3462fa0a8032" - ) - - # Build paths - root_path = os.path.dirname( - os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) - ) - working_path = os.path.join(root_path, "build", "tor") - tarball_path = os.path.join(working_path, tarball_filename) - dist_path = os.path.join(root_path, "onionshare", "resources", "tor") - - # Make sure dirs exist - if not os.path.exists(working_path): - os.makedirs(working_path, exist_ok=True) - - if not os.path.exists(dist_path): - os.makedirs(dist_path, exist_ok=True) - - # Make sure the tarball is downloaded - if not os.path.exists(tarball_path): - print("Downloading {}".format(tarball_url)) - r = requests.get(tarball_url) - open(tarball_path, "wb").write(r.content) - tarball_sha256 = hashlib.sha256(r.content).hexdigest() - else: - tarball_data = open(tarball_path, "rb").read() - tarball_sha256 = hashlib.sha256(tarball_data).hexdigest() - - # Compare the hash - if tarball_sha256 != expected_tarball_sha256: - print("ERROR! The sha256 doesn't match:") - print("expected: {}".format(expected_tarball_sha256)) - print(" actual: {}".format(tarball_sha256)) - sys.exit(-1) - - # Delete extracted tarball, if it's there - shutil.rmtree(os.path.join(working_path, "tor-browser_en-US"), ignore_errors=True) - - # Extract the tarball - subprocess.call(["tar", "-xvf", tarball_path], cwd=working_path) - tarball_tor_path = os.path.join( - working_path, "tor-browser_en-US", "Browser", "TorBrowser" - ) - - # Copy into dist - shutil.copyfile( - os.path.join(tarball_tor_path, "Data", "Tor", "geoip"), - os.path.join(dist_path, "geoip"), - ) - shutil.copyfile( - os.path.join(tarball_tor_path, "Data", "Tor", "geoip6"), - os.path.join(dist_path, "geoip6"), - ) - shutil.copyfile( - os.path.join(tarball_tor_path, "Tor", "tor"), - os.path.join(dist_path, "tor"), - ) - os.chmod(os.path.join(dist_path, "tor"), 0o755) - shutil.copyfile( - os.path.join(tarball_tor_path, "Tor", "libcrypto.so.1.1"), - os.path.join(dist_path, "libcrypto.so.1.1"), - ) - shutil.copyfile( - os.path.join(tarball_tor_path, "Tor", "libevent-2.1.so.7"), - os.path.join(dist_path, "libevent-2.1.so.7"), - ) - shutil.copyfile( - os.path.join(tarball_tor_path, "Tor", "libssl.so.1.1"), - os.path.join(dist_path, "libssl.so.1.1"), - ) - shutil.copyfile( - os.path.join(tarball_tor_path, "Tor", "libstdc++", "libstdc++.so.6"), - os.path.join(dist_path, "libstdc++.so.6"), - ) - shutil.copyfile( - os.path.join(tarball_tor_path, "Tor", "PluggableTransports", "obfs4proxy"), - os.path.join(dist_path, "obfs4proxy"), - ) - os.chmod(os.path.join(dist_path, "obfs4proxy"), 0o755) - shutil.copyfile( - os.path.join( - tarball_tor_path, "Tor", "PluggableTransports", "snowflake-client" - ), - os.path.join(dist_path, "snowflake-client"), - ) - os.chmod(os.path.join(dist_path, "snowflake-client"), 0o755) - - print(f"Tor binaries extracted to: {dist_path}") - - # Fetch the built-in bridges - UpdateTorBridges(root_path) - - -if __name__ == "__main__": - main() diff --git a/desktop/scripts/get-tor-osx.py b/desktop/scripts/get-tor-osx.py deleted file mode 100755 index 95065cf1..00000000 --- a/desktop/scripts/get-tor-osx.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -OnionShare | https://onionshare.org/ - -Copyright (C) 2014-2022 Micah Lee, et al. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -""" -This script downloads a pre-built tor binary to bundle with OnionShare. -In order to avoid a Mac gnupg dependency, I manually verify the signature -and hard-code the sha256 hash. -""" -import inspect -import os -import sys -import hashlib -import shutil -import subprocess -import requests - -from bridges import UpdateTorBridges - - -def main(): - dmg_url = "https://dist.torproject.org/torbrowser/11.0.9/TorBrowser-11.0.9-osx64_en-US.dmg" - dmg_filename = "TorBrowser-11.0.9-osx64_en-US.dmg" - expected_dmg_sha256 = ( - "e34629a178a92983924a5a89c7a988285d2d27f21832413a7f7e33af7871c8d6" - ) - - # Build paths - root_path = os.path.dirname( - os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) - ) - working_path = os.path.join(root_path, "build", "tor") - dmg_tor_path = os.path.join( - "/Volumes", "Tor Browser", "Tor Browser.app", "Contents" - ) - dmg_path = os.path.join(working_path, dmg_filename) - dist_path = os.path.join(root_path, "onionshare", "resources", "tor") - if not os.path.exists(dist_path): - os.makedirs(dist_path, exist_ok=True) - - # Make sure the working folder exists - if not os.path.exists(working_path): - os.makedirs(working_path) - - # Make sure the zip is downloaded - if not os.path.exists(dmg_path): - print("Downloading {}".format(dmg_url)) - r = requests.get(dmg_url) - open(dmg_path, "wb").write(r.content) - dmg_sha256 = hashlib.sha256(r.content).hexdigest() - else: - dmg_data = open(dmg_path, "rb").read() - dmg_sha256 = hashlib.sha256(dmg_data).hexdigest() - - # Compare the hash - if dmg_sha256 != expected_dmg_sha256: - print("ERROR! The sha256 doesn't match:") - print("expected: {}".format(expected_dmg_sha256)) - print(" actual: {}".format(dmg_sha256)) - sys.exit(-1) - - # Mount the dmg, copy data to the working path - subprocess.call(["hdiutil", "attach", dmg_path]) - - # Copy into dist - shutil.copyfile( - os.path.join(dmg_tor_path, "Resources", "TorBrowser", "Tor", "geoip"), - os.path.join(dist_path, "geoip"), - ) - shutil.copyfile( - os.path.join(dmg_tor_path, "Resources", "TorBrowser", "Tor", "geoip6"), - os.path.join(dist_path, "geoip6"), - ) - shutil.copyfile( - os.path.join(dmg_tor_path, "MacOS", "Tor", "tor.real"), - os.path.join(dist_path, "tor"), - ) - os.chmod(os.path.join(dist_path, "tor"), 0o755) - shutil.copyfile( - os.path.join(dmg_tor_path, "MacOS", "Tor", "libevent-2.1.7.dylib"), - os.path.join(dist_path, "libevent-2.1.7.dylib"), - ) - # obfs4proxy binary - shutil.copyfile( - os.path.join(dmg_tor_path, "MacOS", "Tor", "PluggableTransports", "obfs4proxy"), - os.path.join(dist_path, "obfs4proxy"), - ) - os.chmod(os.path.join(dist_path, "obfs4proxy"), 0o755) - # snowflake-client binary - shutil.copyfile( - os.path.join( - dmg_tor_path, "MacOS", "Tor", "PluggableTransports", "snowflake-client" - ), - os.path.join(dist_path, "snowflake-client"), - ) - os.chmod(os.path.join(dist_path, "snowflake-client"), 0o755) - - # Eject dmg - subprocess.call(["diskutil", "eject", "/Volumes/Tor Browser"]) - - # Fetch the built-in bridges - UpdateTorBridges(root_path) - - -if __name__ == "__main__": - main() diff --git a/desktop/scripts/get-tor-windows.py b/desktop/scripts/get-tor-windows.py deleted file mode 100644 index 581a476b..00000000 --- a/desktop/scripts/get-tor-windows.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- -""" -OnionShare | https://onionshare.org/ - -Copyright (C) 2014-2022 Micah Lee, et al. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -""" -This script downloads a pre-built tor binary to bundle with OnionShare. -In order to avoid a Windows gnupg dependency, I manually verify the signature -and hard-code the sha256 hash. -""" -import inspect -import os -import sys -import hashlib -import shutil -import subprocess -import requests - -from bridges import UpdateTorBridges - - -def main(): - exe_url = "https://dist.torproject.org/torbrowser/11.0.9/torbrowser-install-11.0.9_en-US.exe" - exe_filename = "torbrowser-install-11.0.9_en-US.exe" - expected_exe_sha256 = ( - "e938433028b6ffb5d312db6268b19e419626b071f08209684c8e5b9f3d3df2bc" - ) - # Build paths - root_path = os.path.dirname( - os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) - ) - working_path = os.path.join(root_path, "build", "tor") - exe_path = os.path.join(working_path, exe_filename) - dist_path = os.path.join(root_path, "onionshare", "resources", "tor") - - # Make sure the working folder exists - if not os.path.exists(working_path): - os.makedirs(working_path) - - # Make sure Tor Browser is downloaded - if not os.path.exists(exe_path): - print("Downloading {}".format(exe_url)) - r = requests.get(exe_url) - open(exe_path, "wb").write(r.content) - exe_sha256 = hashlib.sha256(r.content).hexdigest() - else: - exe_data = open(exe_path, "rb").read() - exe_sha256 = hashlib.sha256(exe_data).hexdigest() - - # Compare the hash - if exe_sha256 != expected_exe_sha256: - print("ERROR! The sha256 doesn't match:") - print("expected: {}".format(expected_exe_sha256)) - print(" actual: {}".format(exe_sha256)) - sys.exit(-1) - - # Extract the bits we need from the exe - subprocess.Popen( - [ - "7z", - "e", - "-y", - exe_path, - "Browser\\TorBrowser\\Tor", - "-o%s" % os.path.join(working_path, "Tor"), - ] - ).wait() - subprocess.Popen( - [ - "7z", - "e", - "-y", - exe_path, - "Browser\\TorBrowser\\Data\\Tor\\geoip*", - "-o%s" % os.path.join(working_path, "Data"), - ] - ).wait() - - # Copy into the onionshare resources - if os.path.exists(dist_path): - shutil.rmtree(dist_path) - os.makedirs(dist_path) - shutil.copytree(os.path.join(working_path, "Tor"), os.path.join(dist_path, "Tor")) - shutil.copytree( - os.path.join(working_path, "Data"), os.path.join(dist_path, "Data", "Tor") - ) - - # Fetch the built-in bridges - UpdateTorBridges(root_path) - - -if __name__ == "__main__": - main() diff --git a/desktop/scripts/get-tor.py b/desktop/scripts/get-tor.py new file mode 100644 index 00000000..9d5b42da --- /dev/null +++ b/desktop/scripts/get-tor.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +import platform +import inspect +import os +import sys +import hashlib +import shutil +import subprocess +import requests + +# Windows +exe_url = ( + "https://dist.torproject.org/torbrowser/11.0.9/torbrowser-install-11.0.9_en-US.exe" +) +exe_filename = "torbrowser-install-11.0.9_en-US.exe" +expected_exe_sha256 = "e938433028b6ffb5d312db6268b19e419626b071f08209684c8e5b9f3d3df2bc" + +# macOS +dmg_url = ( + "https://dist.torproject.org/torbrowser/11.0.9/TorBrowser-11.0.9-osx64_en-US.dmg" +) +dmg_filename = "TorBrowser-11.0.9-osx64_en-US.dmg" +expected_dmg_sha256 = "e34629a178a92983924a5a89c7a988285d2d27f21832413a7f7e33af7871c8d6" + +# Linux +tarball_url = "https://dist.torproject.org/torbrowser/11.0.9/tor-browser-linux64-11.0.9_en-US.tar.xz" +tarball_filename = "tor-browser-linux64-11.0.9_en-US.tar.xz" +expected_tarball_sha256 = ( + "baa5ccafb5c68f1c46f9ae983b9b0a0419f66d41e0483ba5aacb3462fa0a8032" +) + + +# Common paths +root_path = os.path.dirname( + os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +) +working_path = os.path.join(root_path, "build", "tor") + + +def get_tor_windows(): + # Build paths + exe_path = os.path.join(working_path, exe_filename) + dist_path = os.path.join(root_path, "onionshare", "resources", "tor") + + # Make sure the working folder exists + if not os.path.exists(working_path): + os.makedirs(working_path) + + # Make sure Tor Browser is downloaded + if not os.path.exists(exe_path): + print("Downloading {}".format(exe_url)) + r = requests.get(exe_url) + open(exe_path, "wb").write(r.content) + exe_sha256 = hashlib.sha256(r.content).hexdigest() + else: + exe_data = open(exe_path, "rb").read() + exe_sha256 = hashlib.sha256(exe_data).hexdigest() + + # Compare the hash + if exe_sha256 != expected_exe_sha256: + print("ERROR! The sha256 doesn't match:") + print("expected: {}".format(expected_exe_sha256)) + print(" actual: {}".format(exe_sha256)) + sys.exit(-1) + + # Extract the bits we need from the exe + subprocess.Popen( + [ + "7z", + "e", + "-y", + exe_path, + "Browser\\TorBrowser\\Tor", + "-o%s" % os.path.join(working_path, "Tor"), + ] + ).wait() + subprocess.Popen( + [ + "7z", + "e", + "-y", + exe_path, + "Browser\\TorBrowser\\Data\\Tor\\geoip*", + "-o%s" % os.path.join(working_path, "Data"), + ] + ).wait() + + # Copy into the onionshare resources + if os.path.exists(dist_path): + shutil.rmtree(dist_path) + os.makedirs(dist_path) + shutil.copytree(os.path.join(working_path, "Tor"), os.path.join(dist_path, "Tor")) + shutil.copytree( + os.path.join(working_path, "Data"), os.path.join(dist_path, "Data", "Tor") + ) + + # Fetch the built-in bridges + update_tor_bridges() + + +def get_tor_macos(): + # Build paths + dmg_tor_path = os.path.join( + "/Volumes", "Tor Browser", "Tor Browser.app", "Contents" + ) + dmg_path = os.path.join(working_path, dmg_filename) + dist_path = os.path.join(root_path, "onionshare", "resources", "tor") + if not os.path.exists(dist_path): + os.makedirs(dist_path, exist_ok=True) + + # Make sure the working folder exists + if not os.path.exists(working_path): + os.makedirs(working_path) + + # Make sure the zip is downloaded + if not os.path.exists(dmg_path): + print("Downloading {}".format(dmg_url)) + r = requests.get(dmg_url) + open(dmg_path, "wb").write(r.content) + dmg_sha256 = hashlib.sha256(r.content).hexdigest() + else: + dmg_data = open(dmg_path, "rb").read() + dmg_sha256 = hashlib.sha256(dmg_data).hexdigest() + + # Compare the hash + if dmg_sha256 != expected_dmg_sha256: + print("ERROR! The sha256 doesn't match:") + print("expected: {}".format(expected_dmg_sha256)) + print(" actual: {}".format(dmg_sha256)) + sys.exit(-1) + + # Mount the dmg, copy data to the working path + subprocess.call(["hdiutil", "attach", dmg_path]) + + # Copy into dist + shutil.copyfile( + os.path.join(dmg_tor_path, "Resources", "TorBrowser", "Tor", "geoip"), + os.path.join(dist_path, "geoip"), + ) + shutil.copyfile( + os.path.join(dmg_tor_path, "Resources", "TorBrowser", "Tor", "geoip6"), + os.path.join(dist_path, "geoip6"), + ) + shutil.copyfile( + os.path.join(dmg_tor_path, "MacOS", "Tor", "tor.real"), + os.path.join(dist_path, "tor"), + ) + os.chmod(os.path.join(dist_path, "tor"), 0o755) + shutil.copyfile( + os.path.join(dmg_tor_path, "MacOS", "Tor", "libevent-2.1.7.dylib"), + os.path.join(dist_path, "libevent-2.1.7.dylib"), + ) + # obfs4proxy binary + shutil.copyfile( + os.path.join(dmg_tor_path, "MacOS", "Tor", "PluggableTransports", "obfs4proxy"), + os.path.join(dist_path, "obfs4proxy"), + ) + os.chmod(os.path.join(dist_path, "obfs4proxy"), 0o755) + # snowflake-client binary + shutil.copyfile( + os.path.join( + dmg_tor_path, "MacOS", "Tor", "PluggableTransports", "snowflake-client" + ), + os.path.join(dist_path, "snowflake-client"), + ) + os.chmod(os.path.join(dist_path, "snowflake-client"), 0o755) + + # Eject dmg + subprocess.call(["diskutil", "eject", "/Volumes/Tor Browser"]) + + # Fetch the built-in bridges + update_tor_bridges() + + +def get_tor_linux(): + # Build paths + tarball_path = os.path.join(working_path, tarball_filename) + dist_path = os.path.join(root_path, "onionshare", "resources", "tor") + + # Make sure dirs exist + if not os.path.exists(working_path): + os.makedirs(working_path, exist_ok=True) + + if not os.path.exists(dist_path): + os.makedirs(dist_path, exist_ok=True) + + # Make sure the tarball is downloaded + if not os.path.exists(tarball_path): + print("Downloading {}".format(tarball_url)) + r = requests.get(tarball_url) + open(tarball_path, "wb").write(r.content) + tarball_sha256 = hashlib.sha256(r.content).hexdigest() + else: + tarball_data = open(tarball_path, "rb").read() + tarball_sha256 = hashlib.sha256(tarball_data).hexdigest() + + # Compare the hash + if tarball_sha256 != expected_tarball_sha256: + print("ERROR! The sha256 doesn't match:") + print("expected: {}".format(expected_tarball_sha256)) + print(" actual: {}".format(tarball_sha256)) + sys.exit(-1) + + # Delete extracted tarball, if it's there + shutil.rmtree(os.path.join(working_path, "tor-browser_en-US"), ignore_errors=True) + + # Extract the tarball + subprocess.call(["tar", "-xvf", tarball_path], cwd=working_path) + tarball_tor_path = os.path.join( + working_path, "tor-browser_en-US", "Browser", "TorBrowser" + ) + + # Copy into dist + shutil.copyfile( + os.path.join(tarball_tor_path, "Data", "Tor", "geoip"), + os.path.join(dist_path, "geoip"), + ) + shutil.copyfile( + os.path.join(tarball_tor_path, "Data", "Tor", "geoip6"), + os.path.join(dist_path, "geoip6"), + ) + shutil.copyfile( + os.path.join(tarball_tor_path, "Tor", "tor"), + os.path.join(dist_path, "tor"), + ) + os.chmod(os.path.join(dist_path, "tor"), 0o755) + shutil.copyfile( + os.path.join(tarball_tor_path, "Tor", "libcrypto.so.1.1"), + os.path.join(dist_path, "libcrypto.so.1.1"), + ) + shutil.copyfile( + os.path.join(tarball_tor_path, "Tor", "libevent-2.1.so.7"), + os.path.join(dist_path, "libevent-2.1.so.7"), + ) + shutil.copyfile( + os.path.join(tarball_tor_path, "Tor", "libssl.so.1.1"), + os.path.join(dist_path, "libssl.so.1.1"), + ) + shutil.copyfile( + os.path.join(tarball_tor_path, "Tor", "libstdc++", "libstdc++.so.6"), + os.path.join(dist_path, "libstdc++.so.6"), + ) + shutil.copyfile( + os.path.join(tarball_tor_path, "Tor", "PluggableTransports", "obfs4proxy"), + os.path.join(dist_path, "obfs4proxy"), + ) + os.chmod(os.path.join(dist_path, "obfs4proxy"), 0o755) + shutil.copyfile( + os.path.join( + tarball_tor_path, "Tor", "PluggableTransports", "snowflake-client" + ), + os.path.join(dist_path, "snowflake-client"), + ) + os.chmod(os.path.join(dist_path, "snowflake-client"), 0o755) + + print(f"Tor binaries extracted to: {dist_path}") + + # Fetch the built-in bridges + update_tor_bridges() + + +def update_tor_bridges(): + """ + Update the built-in Tor Bridges in OnionShare's torrc templates. + """ + torrc_template_dir = os.path.join( + root_path, os.pardir, "cli/onionshare_cli/resources" + ) + endpoint = "https://bridges.torproject.org/moat/circumvention/builtin" + r = requests.post( + endpoint, + headers={"Content-Type": "application/vnd.api+json"}, + ) + if r.status_code != 200: + print( + f"There was a problem fetching the latest built-in bridges: status_code={r.status_code}" + ) + return False + + result = r.json() + + if "errors" in result: + print( + f"There was a problem fetching the latest built-in bridges: errors={result['errors']}" + ) + return False + + for bridge_type in ["meek-azure", "obfs4", "snowflake"]: + if result[bridge_type]: + if bridge_type == "meek-azure": + torrc_template_extension = "meek_lite_azure" + else: + torrc_template_extension = bridge_type + torrc_template = os.path.join( + root_path, + torrc_template_dir, + f"torrc_template-{torrc_template_extension}", + ) + + with open(torrc_template, "w") as f: + f.write(f"# Enable built-in {bridge_type} bridge\n") + bridges = result[bridge_type] + # Sorts the bridges numerically by IP, since they come back in + # random order from the API each time, and create noisy git diff. + bridges.sort(key=lambda s: s.split()[1]) + for item in bridges: + f.write(f"Bridge {item}\n") + + +def main(): + """ + Download Tor Browser and extract tor binaries + """ + system = platform.system() + if system == "Windows": + get_tor_windows() + elif system == "Darwin": + get_tor_macos() + elif system == "Linux": + get_tor_linux() + else: + print("Platform not supported") + + +if __name__ == "__main__": + main() From 28655d28c80e3dadcb4c599ee5c8677bd0876ce0 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 11:20:57 -0700 Subject: [PATCH 027/154] Make CircleCI use the new get-tor.py --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5cf11bfa..93690482 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,7 +93,7 @@ jobs: name: Get tor command: | cd ~\project\desktop - poetry run python .\scripts\get-tor-windows.py + poetry run python .\scripts\get-tor.py - run: name: Build meek command: | @@ -141,7 +141,7 @@ jobs: name: Get tor command: | cd ~\project\desktop - C:\Python-32bit\Scripts\poetry run python .\scripts\get-tor-windows.py + C:\Python-32bit\Scripts\poetry run python .\scripts\get-tor.py - run: name: Build meek command: | @@ -190,7 +190,7 @@ jobs: name: Get tor command: | cd ~/project/desktop - poetry run ./scripts/get-tor-osx.py + poetry run python ./scripts/get-tor.py - run: name: Build meek command: | From 9b28a9393aec28d94ec89771238c404d1bc89c20 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 17:20:31 -0700 Subject: [PATCH 028/154] Move windows and mac build scripts into scripts folder --- .circleci/config.yml | 4 ++-- RELEASE.md | 12 ++++++------ desktop/{package/macos.py => scripts/build-macos.py} | 0 .../{package/windows.py => scripts/build-windows.py} | 0 4 files changed, 8 insertions(+), 8 deletions(-) rename desktop/{package/macos.py => scripts/build-macos.py} (100%) rename desktop/{package/windows.py => scripts/build-windows.py} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 93690482..17e50a66 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -104,7 +104,7 @@ jobs: command: | cd ~\project\desktop poetry run python .\setup-freeze.py build - poetry run python .\package\windows.py cleanup-build + poetry run python .\script\build-windows.py cleanup-build - run: name: Compress command: | @@ -201,7 +201,7 @@ jobs: command: | cd ~/project/desktop poetry run python ./setup-freeze.py bdist_mac - poetry run python ./package/macos.py cleanup-build + poetry run python ./install/build-macos.py cleanup-build - run: name: Compress command: zip -r ~/onionshare-macos.zip ~/project/desktop/build/OnionShare.app diff --git a/RELEASE.md b/RELEASE.md index cab2909b..14b6599c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -110,9 +110,9 @@ Build the Windows binaries, delete extra files, codesign, and create an MSI pack ``` poetry run python .\setup-freeze.py build -poetry run python .\package\windows.py cleanup-build -poetry run python .\package\windows.py codesign [build_dir] -poetry run python .\package\windows.py package [build_dir] +poetry run python .\scripts\build-windows.py cleanup-build +poetry run python .\scripts\build-windows.py codesign [build_dir] +poetry run python .\scripts\build-windows.py package [build_dir] ``` This will create `desktop/dist/OnionShare-$VERSION.msi`, signed. @@ -125,9 +125,9 @@ Then build an executable, make it a macOS app bundle, and package it in a dmg: ```sh poetry run python ./setup-freeze.py bdist_mac -poetry run python ./package/macos.py cleanup-build -poetry run python ./package/macos.py codesign [app_path] -poetry run python ./package/macos.py package [app_path] +poetry run python ./scripts/build-macos.py cleanup-build +poetry run python ./scripts/build-macos.py codesign [app_path] +poetry run python ./scripts/build-macos.py package [app_path] ``` The will create `dist/OnionShare-$VERSION.dmg`. diff --git a/desktop/package/macos.py b/desktop/scripts/build-macos.py similarity index 100% rename from desktop/package/macos.py rename to desktop/scripts/build-macos.py diff --git a/desktop/package/windows.py b/desktop/scripts/build-windows.py similarity index 100% rename from desktop/package/windows.py rename to desktop/scripts/build-windows.py From 49471a835a4284582a5bc437690040b8dc33be5a Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 17:35:41 -0700 Subject: [PATCH 029/154] Fix macOS cleanup script --- .circleci/config.yml | 6 +++--- desktop/scripts/build-macos.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 17e50a66..40288c2d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -104,7 +104,7 @@ jobs: command: | cd ~\project\desktop poetry run python .\setup-freeze.py build - poetry run python .\script\build-windows.py cleanup-build + poetry run python .\scripts\build-windows.py cleanup-build - run: name: Compress command: | @@ -152,7 +152,7 @@ jobs: command: | cd ~\project\desktop C:\Python-32bit\Scripts\poetry run python .\setup-freeze.py build - C:\Python-32bit\Scripts\poetry run python .\package\windows.py cleanup-build + C:\Python-32bit\Scripts\poetry run python .\scripts\build-windows.py cleanup-build - run: name: Compress command: | @@ -201,7 +201,7 @@ jobs: command: | cd ~/project/desktop poetry run python ./setup-freeze.py bdist_mac - poetry run python ./install/build-macos.py cleanup-build + poetry run python ./scripts/build-macos.py cleanup-build - run: name: Compress command: zip -r ~/onionshare-macos.zip ~/project/desktop/build/OnionShare.app diff --git a/desktop/scripts/build-macos.py b/desktop/scripts/build-macos.py index 4d4c02e0..f9586de0 100644 --- a/desktop/scripts/build-macos.py +++ b/desktop/scripts/build-macos.py @@ -142,9 +142,14 @@ def cleanup_build(): shutil.rmtree( f"{app_path}/Contents/MacOS/lib/PySide2/Qt/lib/{framework}.framework" ) + print( + f"Deleted: {app_path}/Contents/MacOS/lib/PySide2/Qt/lib/{framework}.framework" + ) try: os.remove(f"{app_path}/Contents/MacOS/lib/PySide2/{framework}.abi3.so") os.remove(f"{app_path}/Contents/MacOS/lib/PySide2/{framework}.pyi") + print(f"Deleted: {app_path}/Contents/MacOS/lib/PySide2/{framework}.abi3.so") + print(f"Deleted: {app_path}/Contents/MacOS/lib/PySide2/{framework}.pyi") except FileNotFoundError: pass @@ -219,10 +224,12 @@ def cleanup_build(): f"{app_path}/Contents/Resources/lib/PySide2/rcc", f"{app_path}/Contents/Resources/lib/PySide2/uic", ]: - if os.path.isdir(filename): - shutil.rmtree(filename) - elif os.path.isfile(filename): + if os.path.isfile(filename) or os.path.islink(filename): os.remove(filename) + print(f"Deleted: {filename}") + elif os.path.isdir(filename): + shutil.rmtree(filename) + print(f"Deleted: {filename}") else: print(f"Cannot delete, filename not found: {filename}") From fc754584537ea9189725e936fe23bb129cd9a73d Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 20:21:14 -0700 Subject: [PATCH 030/154] Build exe.macos as well as the app bundle, and upload them both --- .circleci/config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 40288c2d..2243fc04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -200,13 +200,18 @@ jobs: name: Build OnionShare command: | cd ~/project/desktop + poetry run python ./setup-freeze.py build poetry run python ./setup-freeze.py bdist_mac poetry run python ./scripts/build-macos.py cleanup-build - run: name: Compress - command: zip -r ~/onionshare-macos.zip ~/project/desktop/build/OnionShare.app + command: | + zip -r ~/onionshare-macos.zip ~/project/desktop/build/exe.macosx-10.9-x86_64-3.9 + zip -r ~/onionshare-app-macos.zip ~/project/desktop/build/OnionShare.app - store_artifacts: path: ~/onionshare-macos.zip + - store_artifacts: + path: ~/onionshare-app-macos.zip build-snapcraft: docker: From eec2f7727b5d2c324d18c97304c3ab9e9ae2816d Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 20:22:08 -0700 Subject: [PATCH 031/154] Try stable snapcore/snapcraft container --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2243fc04..8d2f3e3f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,7 +215,7 @@ jobs: build-snapcraft: docker: - - image: snapcore/snapcraft:edge + - image: snapcore/snapcraft:stable working_directory: ~/repo steps: - checkout From 61db951789b560d8a2afd2b3942d2e81f42ff25e Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 20:26:16 -0700 Subject: [PATCH 032/154] Try install extra snaps first --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d2f3e3f..f21a61b1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -219,6 +219,9 @@ jobs: working_directory: ~/repo steps: - checkout + - run: + name: Install extra snaps + command: sudo snap install core18 go/stable go/latest/stable gnome-3-34-1804-sdk/latest/stable - run: name: Build the snap command: | From cc11737a75525d0ede511443eca6f8df8d60811d Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 20:29:55 -0700 Subject: [PATCH 033/154] Install one snap at a time --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f21a61b1..aa2ed0e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -221,7 +221,11 @@ jobs: - checkout - run: name: Install extra snaps - command: sudo snap install core18 go/stable go/latest/stable gnome-3-34-1804-sdk/latest/stable + command: | + sudo snap install core18 + sudo snap install go/stable + sudo snap install go/latest/stable + sudo snap install gnome-3-34-1804-sdk/latest/stable - run: name: Build the snap command: | From 8d810f87c0146a8303d75d59eea1fb3bcef696c7 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 20:59:33 -0700 Subject: [PATCH 034/154] Try installing go and gnome-3-34-1804-sdk without snapd --- .circleci/config.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aa2ed0e6..6a8df2db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -220,12 +220,17 @@ jobs: steps: - checkout - run: - name: Install extra snaps + name: Install extra snaps without snapd command: | - sudo snap install core18 - sudo snap install go/stable - sudo snap install go/latest/stable - sudo snap install gnome-3-34-1804-sdk/latest/stable + mkdir ~/tmp && cd ~/tmp + # snap install go + snap download go + mkdir -p /snap/go && unsquashfs -d /snap/go/current go_*.snap + rm ~/tmp/* + # snap install gnome-3-34-1804-sdk + snap download gnome-3-34-1804-sdk + mkdir -p /snap/gnome-3-34-1804-sdk && unsquashfs -d /snap/gnome-3-34-1804-sdk/current gnome-3-34-1804-sdk_*.snap + rm ~/tmp/* - run: name: Build the snap command: | From 8076c64b5edf4ab431c777451f5dbc52000e4366 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 21:06:48 -0700 Subject: [PATCH 035/154] Install even more snaps --- .circleci/config.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a8df2db..034c571c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,15 +222,12 @@ jobs: - run: name: Install extra snaps without snapd command: | - mkdir ~/tmp && cd ~/tmp - # snap install go - snap download go - mkdir -p /snap/go && unsquashfs -d /snap/go/current go_*.snap - rm ~/tmp/* - # snap install gnome-3-34-1804-sdk - snap download gnome-3-34-1804-sdk - mkdir -p /snap/gnome-3-34-1804-sdk && unsquashfs -d /snap/gnome-3-34-1804-sdk/current gnome-3-34-1804-sdk_*.snap - rm ~/tmp/* + mkdir ~/tmp + install_snap() { cd ~/tmp && snap download $1 && mkdir -p /snap/$1 && unsquashfs -d /snap/$1/current *.snap && rm ~/tmp/* } + install_snap "core18" + install_snap "go" + install_snap "gnome-3-34-1804" + install_snap "gnome-3-34-1804-sdk" - run: name: Build the snap command: | From af19ad405be34e2852f9720daf27193b00ff3035 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 21:11:47 -0700 Subject: [PATCH 036/154] bash function syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 034c571c..ee526d26 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -223,7 +223,7 @@ jobs: name: Install extra snaps without snapd command: | mkdir ~/tmp - install_snap() { cd ~/tmp && snap download $1 && mkdir -p /snap/$1 && unsquashfs -d /snap/$1/current *.snap && rm ~/tmp/* } + install_snap () { cd ~/tmp && snap download $1 && mkdir -p /snap/$1 && unsquashfs -d /snap/$1/current *.snap && rm ~/tmp/*; } install_snap "core18" install_snap "go" install_snap "gnome-3-34-1804" From 8a450c7db72f656d3ff218a28e95f8d3a1dc9099 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 3 Apr 2022 21:22:13 -0700 Subject: [PATCH 037/154] Give up on snapcraft --- .circleci/config.yml | 60 ++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ee526d26..86fb3a68 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,19 +12,19 @@ workflows: - build-win64: requires: - test-cli - # - test-gui + - test-gui - build-win32: requires: - test-cli - # - test-gui + - test-gui - build-macos: requires: - test-cli - # - test-gui - - build-snapcraft: - requires: - - test-cli - # - test-gui + - test-gui + # - build-snapcraft: + # requires: + # - test-cli + # - test-gui jobs: test-cli: @@ -213,26 +213,26 @@ jobs: - store_artifacts: path: ~/onionshare-app-macos.zip - build-snapcraft: - docker: - - image: snapcore/snapcraft:stable - working_directory: ~/repo - steps: - - checkout - - run: - name: Install extra snaps without snapd - command: | - mkdir ~/tmp - install_snap () { cd ~/tmp && snap download $1 && mkdir -p /snap/$1 && unsquashfs -d /snap/$1/current *.snap && rm ~/tmp/*; } - install_snap "core18" - install_snap "go" - install_snap "gnome-3-34-1804" - install_snap "gnome-3-34-1804-sdk" - - run: - name: Build the snap - command: | - cd ~/repo - snapcraft - mv onionshare-*_amd64.snap ~/onionshare_amd64.snap - - store_artifacts: - path: ~/onionshare_amd64.snap + # build-snapcraft: + # docker: + # - image: snapcore/snapcraft:stable + # working_directory: ~/repo + # steps: + # - checkout + # - run: + # name: Install extra snaps without snapd + # command: | + # mkdir ~/tmp + # install_snap () { cd ~/tmp && snap download $1 && mkdir -p /snap/$1 && unsquashfs -d /snap/$1/current *.snap && rm ~/tmp/*; } + # install_snap "core18" + # install_snap "go" + # install_snap "gnome-3-34-1804" + # install_snap "gnome-3-34-1804-sdk" + # - run: + # name: Build the snap + # command: | + # cd ~/repo + # snapcraft + # mv onionshare-*_amd64.snap ~/onionshare_amd64.snap + # - store_artifacts: + # path: ~/onionshare_amd64.snap From 3d81c890bc21da26f6501763b4d35750f93f021f Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Mon, 4 Apr 2022 20:41:24 -0700 Subject: [PATCH 038/154] Fix macos build bug --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 86fb3a68..b834dc23 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -206,7 +206,7 @@ jobs: - run: name: Compress command: | - zip -r ~/onionshare-macos.zip ~/project/desktop/build/exe.macosx-10.9-x86_64-3.9 + zip -r ~/onionshare-macos.zip ~/project/desktop/build/exe.macosx-10.9-universal2-3.9 zip -r ~/onionshare-app-macos.zip ~/project/desktop/build/OnionShare.app - store_artifacts: path: ~/onionshare-macos.zip From c23a7856eb50ed8a670fc214d239a8d1e698db2c Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Mon, 4 Apr 2022 20:45:08 -0700 Subject: [PATCH 039/154] Try the python3 choco project instead of python, to see if it fixes the win64 build --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b834dc23..4b88bcfc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,7 +80,7 @@ jobs: - run: name: Install Python 3.9.12 command: | - choco install python --version=3.9.12 + choco install python3 --version=3.9.12 - run: name: Install poetry command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - From a2b1f8b578d9baad07521be74829bd7fd00c05d2 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 5 Apr 2022 12:41:21 -0700 Subject: [PATCH 040/154] Complete codesigning and packaging for Windows --- RELEASE.md | 23 ++- desktop/scripts/build-windows.py | 327 +++++++++++++++++-------------- 2 files changed, 195 insertions(+), 155 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 14b6599c..91fa2e91 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -102,20 +102,35 @@ snapcraft upload --release=stable onionshare_${VERSION}_amd64.snap Set up the development environment described in desktop `README.md`. -- To get `signtool.exe`, install the [Windows 10 SDK](https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk) and add `C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x86` to your path. +- To get `signtool.exe`, install the [Windows SDK](https://developer.microsoft.com/en-us/windows/downloads/windows-sdk/) and add `C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64` to your path. - Go to https://dotnet.microsoft.com/download/dotnet-framework and download and install .NET Framework 3.5 SP1 Runtime. I downloaded `dotnetfx35.exe`. - Go to https://wixtoolset.org/releases/ and download and install WiX toolset. I downloaded `wix311.exe`. Add `C:\Program Files (x86)\WiX Toolset v3.11\bin` to the path. Build the Windows binaries, delete extra files, codesign, and create an MSI package: +CircleCI will build the binary and delete extra files, basically running these: + ``` poetry run python .\setup-freeze.py build poetry run python .\scripts\build-windows.py cleanup-build -poetry run python .\scripts\build-windows.py codesign [build_dir] -poetry run python .\scripts\build-windows.py package [build_dir] ``` -This will create `desktop/dist/OnionShare-$VERSION.msi`, signed. +Find the CircleCI jobs `build-win32` and `build-win64`, switch to the artifacts tab, and download: + +- `onionshare-win32.zip` +- `onionshare-win64.zip` + +Extract these files, then run: + +``` +poetry run python .\scripts\build-windows.py codesign [onionshare_win32_path] [onionshare_win64_path] +poetry run python .\scripts\build-windows.py package [onionshare_win32_path] [onionshare_win64_path] +``` + +This will create: + +- `desktop/dist/OnionShare-win32-$VERSION.msi` += `desktop/dist/OnionShare-win64-$VERSION.msi` ## macOS diff --git a/desktop/scripts/build-windows.py b/desktop/scripts/build-windows.py index 215496bd..d1729a6f 100644 --- a/desktop/scripts/build-windows.py +++ b/desktop/scripts/build-windows.py @@ -52,7 +52,7 @@ def sign(filename): "/d", "OnionShare", "/sha1", - "bb1d265ab02272e8fc742f149dcf8751cac63f50", + "1a0345732140749bdaa03efe8591b2c2a036884c", "/fd", "SHA256", "/td", @@ -169,6 +169,150 @@ def wix_build_components_xml(root, data): return component_ids +def msi_package(build_path, msi_path, product_update_code): + print(f"> Build the WiX file") + version_filename = os.path.join( + build_path, "lib", "onionshare_cli", "resources", "version.txt" + ) + with open(version_filename) as f: + version = f.read().strip() + + data = { + "id": "TARGETDIR", + "name": "SourceDir", + "dirs": [ + { + "id": "ProgramFilesFolder", + "dirs": [], + }, + { + "id": "ProgramMenuFolder", + "dirs": [], + }, + ], + } + + data["dirs"][0]["dirs"].append( + wix_build_data( + build_path, + ".", + "INSTALLDIR", + "OnionShare", + ) + ) + + root_el = ET.Element("Wix", xmlns="http://schemas.microsoft.com/wix/2006/wi") + product_el = ET.SubElement( + root_el, + "Product", + Name="OnionShare", + Manufacturer="Micah Lee, et al.", + Id="*", + UpgradeCode="$(var.ProductUpgradeCode)", + Language="1033", + Codepage="1252", + Version="$(var.ProductVersion)", + ) + ET.SubElement( + product_el, + "Package", + Id="*", + Keywords="Installer", + Description="OnionShare $(var.ProductVersion) Installer", + Manufacturer="Micah Lee, et al.", + InstallerVersion="100", + Languages="1033", + Compressed="yes", + SummaryCodepage="1252", + ) + ET.SubElement(product_el, "Media", Id="1", Cabinet="product.cab", EmbedCab="yes") + ET.SubElement( + product_el, + "Icon", + Id="ProductIcon", + SourceFile=os.path.join( + desktop_dir, "onionshare", "resources", "onionshare.ico" + ), + ) + ET.SubElement(product_el, "Property", Id="ARPPRODUCTICON", Value="ProductIcon") + ET.SubElement( + product_el, + "Property", + Id="ARPHELPLINK", + Value="https://docs.onionshare.org", + ) + ET.SubElement( + product_el, + "Property", + Id="ARPURLINFOABOUT", + Value="https://onionshare.org", + ) + ET.SubElement(product_el, "UIRef", Id="WixUI_Minimal") + ET.SubElement(product_el, "UIRef", Id="WixUI_ErrorProgressText") + ET.SubElement( + product_el, + "WixVariable", + Id="WixUILicenseRtf", + Value=os.path.join(desktop_dir, "package", "license.rtf"), + ) + ET.SubElement( + product_el, + "WixVariable", + Id="WixUIDialogBmp", + Value=os.path.join(desktop_dir, "package", "dialog.bmp"), + ) + ET.SubElement( + product_el, + "MajorUpgrade", + AllowSameVersionUpgrades="yes", + DowngradeErrorMessage="A newer version of [ProductName] is already installed. If you are sure you want to downgrade, remove the existing installation via Programs and Features.", + ) + + wix_build_dir_xml(product_el, data) + component_ids = wix_build_components_xml(product_el, data) + + feature_el = ET.SubElement(product_el, "Feature", Id="DefaultFeature", Level="1") + for component_id in component_ids: + ET.SubElement(feature_el, "ComponentRef", Id=component_id) + ET.SubElement(feature_el, "ComponentRef", Id="ApplicationShortcuts") + + with open(os.path.join(build_path, "OnionShare.wxs"), "w") as f: + f.write('\n') + f.write(f'\n') + f.write(f'\n') + + ET.indent(root_el) + f.write(ET.tostring(root_el).decode()) + + print(f"> Build the MSI") + run( + [shutil.which("candle.exe"), "OnionShare.wxs"], + build_path, + ) + run( + [shutil.which("light.exe"), "-ext", "WixUIExtension", "OnionShare.wixobj"], + build_path, + ) + + print(f"> Prepare OnionShare.msi for signing") + run( + [ + shutil.which("insignia.exe"), + "-im", + os.path.join(build_path, "OnionShare.msi"), + ], + error_ok=True, + ) + sign(os.path.join(build_path, "OnionShare.msi")) + + print(f"> Final MSI: {msi_path}") + os.makedirs(os.path.join(desktop_dir, "dist"), exist_ok=True) + os.rename( + os.path.join(build_path, "OnionShare.msi"), + msi_path, + ) + + @click.group() def main(): """ @@ -424,23 +568,28 @@ def cleanup_build(): @main.command() -@click.argument("build_path") -def codesign(build_path): +@click.argument("win32_path") +@click.argument("win64_path") +def codesign(win32_path, win64_path): """Sign Windows binaries before packaging""" - if not os.path.isdir(build_path): - click.echo("Invalid build path") - return + paths = [win32_path, win64_path] - click.echo("> Signing onionshare.exe") - sign(os.path.join(build_path, "onionshare.exe")) + for path in paths: + if not os.path.isdir(path): + click.echo("Invalid build path") + return - click.echo("> Signing onionshare-cli.exe") - sign(os.path.join(build_path, "onionshare-cli.exe")) + for path in paths: + bin_path = os.path.join(path, "onionshare.exe") + click.echo(f"> Signing {bin_path}") + sign(bin_path) - click.echo("> Signing meek-client.exe") - sign( - os.path.join( - build_path, + bin_path = os.path.join(path, "onionshare-cli.exe") + click.echo(f"> Signing {bin_path}") + sign(bin_path) + + bin_path = os.path.join( + path, "lib", "onionshare", "resources", @@ -448,154 +597,30 @@ def codesign(build_path): "Tor", "meek-client.exe", ) - ) + click.echo(f"> Signing {bin_path}") + sign(bin_path) @main.command() -@click.argument("build_path") -def package(build_path): +@click.argument("win32_path") +@click.argument("win64_path") +def package(win32_path, win64_path): """Build the MSI package""" - - print(f"> Build the WiX file") version_filename = os.path.join( root, "cli", "onionshare_cli", "resources", "version.txt" ) with open(version_filename) as f: version = f.read().strip() - data = { - "id": "TARGETDIR", - "name": "SourceDir", - "dirs": [ - { - "id": "ProgramFilesFolder", - "dirs": [], - }, - { - "id": "ProgramMenuFolder", - "dirs": [], - }, - ], - } - - data["dirs"][0]["dirs"].append( - wix_build_data( - build_path, - "INSTALLDIR", - "OnionShare", - ) + msi_package( + win32_path, + os.path.join(desktop_dir, "dist", f"OnionShare-win32-{version}.msi"), + "12b9695c-965b-4be0-bc33-21274e809576", ) - - root_el = ET.Element("Wix", xmlns="http://schemas.microsoft.com/wix/2006/wi") - product_el = ET.SubElement( - root_el, - "Product", - Name="OnionShare", - Manufacturer="Micah Lee, et al.", - Id="*", - UpgradeCode="$(var.ProductUpgradeCode)", - Language="1033", - Codepage="1252", - Version="$(var.ProductVersion)", - ) - ET.SubElement( - product_el, - "Package", - Id="*", - Keywords="Installer", - Description="OnionShare $(var.ProductVersion) Installer", - Manufacturer="Micah Lee, et al.", - InstallerVersion="100", - Languages="1033", - Compressed="yes", - SummaryCodepage="1252", - ) - ET.SubElement(product_el, "Media", Id="1", Cabinet="product.cab", EmbedCab="yes") - ET.SubElement( - product_el, - "Icon", - Id="ProductIcon", - SourceFile="..\\onionshare\\resources\\onionshare.ico", - ) - ET.SubElement(product_el, "Property", Id="ARPPRODUCTICON", Value="ProductIcon") - ET.SubElement( - product_el, - "Property", - Id="ARPHELPLINK", - Value="https://docs.onionshare.org", - ) - ET.SubElement( - product_el, - "Property", - Id="ARPURLINFOABOUT", - Value="https://onionshare.org", - ) - ET.SubElement(product_el, "UIRef", Id="WixUI_Minimal") - ET.SubElement(product_el, "UIRef", Id="WixUI_ErrorProgressText") - ET.SubElement( - product_el, - "WixVariable", - Id="WixUILicenseRtf", - Value="..\\package\\license.rtf", - ) - ET.SubElement( - product_el, - "WixVariable", - Id="WixUIDialogBmp", - Value="..\\package\\dialog.bmp", - ) - ET.SubElement( - product_el, - "MajorUpgrade", - AllowSameVersionUpgrades="yes", - DowngradeErrorMessage="A newer version of [ProductName] is already installed. If you are sure you want to downgrade, remove the existing installation via Programs and Features.", - ) - - wix_build_dir_xml(product_el, data) - component_ids = wix_build_components_xml(product_el, data) - - feature_el = ET.SubElement(product_el, "Feature", Id="DefaultFeature", Level="1") - for component_id in component_ids: - ET.SubElement(feature_el, "ComponentRef", Id=component_id) - ET.SubElement(feature_el, "ComponentRef", Id="ApplicationShortcuts") - - with open(os.path.join(root, "desktop", "build", "OnionShare.wxs"), "w") as f: - f.write('\n') - f.write(f'\n') - f.write( - '\n' - ) - - ET.indent(root_el) - f.write(ET.tostring(root_el).decode()) - - print(f"> Build the MSI") - run( - [shutil.which("candle.exe"), "OnionShare.wxs"], - os.path.join(desktop_dir, "build"), - ) - run( - [shutil.which("light.exe"), "-ext", "WixUIExtension", "OnionShare.wixobj"], - os.path.join(desktop_dir, "build"), - ) - - print(f"> Prepare OnionShare.msi for signing") - run( - [ - shutil.which("insignia.exe"), - "-im", - os.path.join(desktop_dir, "build", "OnionShare.msi"), - ], - error_ok=True, - ) - sign(os.path.join(desktop_dir, "build", "OnionShare.msi")) - - final_msi_filename = os.path.join(desktop_dir, "dist", f"OnionShare-{version}.msi") - print(f"> Final MSI: {final_msi_filename}") - os.makedirs(os.path.join(desktop_dir, "dist"), exist_ok=True) - os.rename( - os.path.join(desktop_dir, "build", "OnionShare.msi"), - final_msi_filename, + msi_package( + win64_path, + os.path.join(desktop_dir, "dist", f"OnionShare-win64-{version}.msi"), + "ed7f9243-3528-4b4a-b85c-9943982e75eb", ) From 301aa19df2bc9eb50b6601efb60a4bae8ba00a4f Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 5 Apr 2022 17:34:04 -0700 Subject: [PATCH 041/154] Properly zip up the app bundle --- .circleci/config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b88bcfc..14202362 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -206,12 +206,10 @@ jobs: - run: name: Compress command: | - zip -r ~/onionshare-macos.zip ~/project/desktop/build/exe.macosx-10.9-universal2-3.9 - zip -r ~/onionshare-app-macos.zip ~/project/desktop/build/OnionShare.app + cd ~/project/desktop/build + zip -r ~/onionshare-macos.zip OnionShare.app - store_artifacts: path: ~/onionshare-macos.zip - - store_artifacts: - path: ~/onionshare-app-macos.zip # build-snapcraft: # docker: From de4dbb54a0e74390f41d5b96bec0e4949e2a36a6 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 7 Apr 2022 19:29:04 -0700 Subject: [PATCH 042/154] Fix macOS package cleanup and codesigning --- RELEASE.md | 31 ++++++-- desktop/scripts/build-macos.py | 141 +++++++++++++++++---------------- 2 files changed, 97 insertions(+), 75 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 91fa2e91..b85e82f3 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -130,28 +130,47 @@ poetry run python .\scripts\build-windows.py package [onionshare_win32_path] [on This will create: - `desktop/dist/OnionShare-win32-$VERSION.msi` -= `desktop/dist/OnionShare-win64-$VERSION.msi` +- `desktop/dist/OnionShare-win64-$VERSION.msi` ## macOS Set up the development environment described in `README.md`. -Then build an executable, make it a macOS app bundle, and package it in a dmg: +Then build an executable, make it a macOS app bundle, and package it in a dmg. + +CircleCI will build the binary and delete extra files, basically running these: ```sh poetry run python ./setup-freeze.py bdist_mac poetry run python ./scripts/build-macos.py cleanup-build +``` + +Find the CircleCI job `build-macos`, switch to the artifacts tab, and download: + +- `onionshare-macos.zip` + +Extract this file, then run: + +```sh poetry run python ./scripts/build-macos.py codesign [app_path] poetry run python ./scripts/build-macos.py package [app_path] ``` The will create `dist/OnionShare-$VERSION.dmg`. -Now, notarize the release. You must have an app-specific Apple ID password saved in the login keychain called `onionshare-notarize`. +Now, notarize the release. -- Notarize it: `xcrun altool --notarize-app --primary-bundle-id "com.micahflee.onionshare" -u "micah@micahflee.com" -p "@keychain:onionshare-notarize" --file dist/OnionShare-$VERSION.dmg` -- Wait for it to get approved, check status with: `xcrun altool --notarization-history 0 -u "micah@micahflee.com" -p "@keychain:onionshare-notarize"` -- After it's approved, staple the ticket: `xcrun stapler staple dist/OnionShare-$VERSION.dmg` +```sh +export APPLE_PASSWORD="changeme" # app-specific Apple ID password +export VERSION=$(cat ../cli/onionshare_cli/resources/version.txt) + +# Notarize it +xcrun altool --notarize-app --primary-bundle-id "com.micahflee.onionshare" -u "micah@micahflee.com" -p "$APPLE_PASSWORD" --file dist/OnionShare-$VERSION.dmg +# Wait for it to get approved, ceck status with +xcrun altool --notarization-history 0 -u "micah@micahflee.com" -p "$APPLE_PASSWORD" +# After it's approved, staple the ticket +xcrun stapler staple dist/OnionShare-$VERSION.dmg +``` This will create `desktop/dist/OnionShare-$VERSION.dmg`, signed and notarized. diff --git a/desktop/scripts/build-macos.py b/desktop/scripts/build-macos.py index f9586de0..10af26ce 100644 --- a/desktop/scripts/build-macos.py +++ b/desktop/scripts/build-macos.py @@ -69,75 +69,78 @@ def cleanup_build(): print("> Delete unused Qt Frameworks") for framework in [ - "Qt3DAnimation", - "Qt3DCore", - "Qt3DExtras", - "Qt3DInput", - "Qt3DLogic", - "Qt3DQuick", - "Qt3DQuickAnimation", - "Qt3DQuickExtras", - "Qt3DQuickInput", - "Qt3DQuickRender", - "Qt3DQuickScene2D", - "Qt3DRender", - "QtBluetooth", - "QtBodymovin", - "QtCharts", - "QtConcurrent", - "QtDataVisualization", - "QtDesigner", - "QtDesignerComponents", - "QtGamepad", - "QtHelp", - "QtLocation", - "QtMultimedia", "QtMultimediaQuick", - "QtMultimediaWidgets", - "QtNetwork", - "QtNetworkAuth", - "QtNfc", - "QtOpenGL", - "QtPdf", - "QtPdfWidgets", - "QtPositioning", - "QtPositioningQuick", - "QtPrintSupport", - "QtPurchasing", - "QtQml", - "QtQmlModels", - "QtQmlWorkerScript", - "QtQuick", - "QtQuick3D", - "QtQuick3DAssetImport", - "QtQuick3DRender", - "QtQuick3DRuntimeRender", - "QtQuick3DUtils", "QtQuickControls2", "QtQuickParticles", - "QtQuickShapes", - "QtQuickTemplates2", - "QtQuickTest", - "QtQuickWidgets", "QtRemoteObjects", - "QtScript", + "Qt3DInput", + "QtPdfWidgets", "QtScriptTools", - "QtScxml", - "QtSensors", - "QtSerialBus", - "QtSerialPort", - "QtSql", - "QtSvg", - "QtTest", - "QtTextToSpeech", - "QtVirtualKeyboard", - "QtWebChannel", - "QtWebEngine", + "QtNetworkAuth", + "QtDataVisualization", + "QtWebEngineCore", + "Qt3DQuickRender", + "Qt3DQuickExtras", + "QtQuick3DRender", + "QtDesigner", + "QtNfc", + "QtQuick3DAssetImport", + "QtBodymovin", "QtWebEngineWidgets", + "QtQuickWidgets", + "Qt3DQuickInput", + "Qt3DQuickScene2D", + "QtUiPlugin", + "QtPdf", + "Qt3DRender", + "QtQuick3DRuntimeRender", + "QtHelp", + "QtPrintSupport", + "QtCharts", "QtWebSockets", - "QtWebView", + "QtQuick3DUtils", + "QtQuickTemplates2", + "QtScript", + "QtPositioningQuick", + "Qt3DCore", + "QtLocation", "QtXml", + "QtSerialPort", + "QtWebView", + "QtQuick", + "QtScxml", + "QtQml", + "Qt3DExtras", + "QtWebChannel", + "QtMultimedia", + "QtQmlWorkerScript", + "QtVirtualKeyboard", + "QtPurchasing", + "QtOpenGL", + "QtWebEngine", + "Qt3DQuick", + "QtTest", + "QtPositioning", + "QtBluetooth", + "QtQuick3D", + "Qt3DLogic", + "QtQuickShapes", + "QtQuickTest", + "QtNetwork", "QtXmlPatterns", + "QtSvg", + "QtDesignerComponents", + "QtMultimediaWidgets", + "QtQmlModels", + "Qt3DQuickAnimation", + "QtSensors", + "Qt3DAnimation", + "QtRepParser", + "QtTextToSpeech", + "QtGamepad", + "QtSerialBus", + "QtSql", + "QtConcurrent" ]: shutil.rmtree( f"{app_path}/Contents/MacOS/lib/PySide2/Qt/lib/{framework}.framework" @@ -147,8 +150,11 @@ def cleanup_build(): ) try: os.remove(f"{app_path}/Contents/MacOS/lib/PySide2/{framework}.abi3.so") - os.remove(f"{app_path}/Contents/MacOS/lib/PySide2/{framework}.pyi") print(f"Deleted: {app_path}/Contents/MacOS/lib/PySide2/{framework}.abi3.so") + except FileNotFoundError: + pass + try: + os.remove(f"{app_path}/Contents/MacOS/lib/PySide2/{framework}.pyi") print(f"Deleted: {app_path}/Contents/MacOS/lib/PySide2/{framework}.pyi") except FileNotFoundError: pass @@ -214,15 +220,12 @@ def cleanup_build(): f"{app_path}/Contents/Resources/lib/PySide2/glue", f"{app_path}/Contents/Resources/lib/PySide2/include", f"{app_path}/Contents/Resources/lib/PySide2/pyside2-lupdate", - f"{app_path}/Contents/Resources/lib/PySide2/Qt/qml", - f"{app_path}/Contents/Resources/lib/PySide2/libpyside2.abi3.5.15.dylib", - f"{app_path}/Contents/Resources/lib/PySide2/Qt/lib/QtRepParser.framework", - f"{app_path}/Contents/Resources/lib/PySide2/Qt/lib/QtUiPlugin.framework", - f"{app_path}/Contents/Resources/lib/PySide2/Qt/lib/QtWebEngineCore.framework/Helpers", - f"{app_path}/Contents/Resources/lib/shiboken2/libshiboken2.abi3.5.15.dylib", - f"{app_path}/Contents/Resources/lib/shiboken2/docs", f"{app_path}/Contents/Resources/lib/PySide2/rcc", f"{app_path}/Contents/Resources/lib/PySide2/uic", + f"{app_path}/Contents/Resources/lib/PySide2/libpyside2.abi3.5.15.dylib", + f"{app_path}/Contents/Resources/lib/PySide2/Qt/qml", + f"{app_path}/Contents/Resources/lib/shiboken2/libshiboken2.abi3.5.15.dylib", + f"{app_path}/Contents/Resources/lib/shiboken2/docs", ]: if os.path.isfile(filename) or os.path.islink(filename): os.remove(filename) @@ -263,7 +266,7 @@ def codesign(app_path): f"{app_path}", ], ): - codesign(path, entitlements_plist_path, identity_name_application) + sign(path, entitlements_plist_path, identity_name_application) print(f"> Signed app bundle: {app_path}") From e184ca5d0304c065d173eddcab73d9a66f18ec6e Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 10 Apr 2022 19:07:35 -0400 Subject: [PATCH 043/154] In CI, download Tor from for windows Tor Expert Bundle, and build meek, obfs4proxy, and snowflake ourselves --- .circleci/config.yml | 71 +++++++++++++++++++++++++++----- desktop/scripts/build-windows.py | 53 ++++++++++++++++-------- 2 files changed, 96 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 14202362..f2d46c6e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,15 +90,38 @@ jobs: cd ~\project\desktop poetry install - run: - name: Get tor - command: | - cd ~\project\desktop - poetry run python .\scripts\get-tor.py + name: Download tor (Windows Expert Bundle) + command: | + mkdir ~\Downloads\tor + cd ~\Downloads\tor + $URL = "https://www.torproject.org/dist/torbrowser/11.0.10/tor-win32-0.4.6.10.zip" + $Filename = "tor-win32-0.4.6.10.zip" + $ExpectedHash = "d3f62317507dbe1a1aa74b9e0e03996dbded2143f94409270828f6a8bcdda16a" + Invoke-WebRequest -Uri $URL -OutFile $Filename + $FileHash = Get-FileHash $Filename + if($FileHash.Hash -ne $ExpectedHash) { + throw "Invalid hash" + } + Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor - run: - name: Build meek + name: Build obfs4proxy command: | - cd ~\project\desktop - python .\scripts\build-meek-client.py + mkdir ~\Downloads\obfs4proxy + cd Downloads\obfs4proxy + git clone https://gitlab.com/yawning/obfs4 + cd obfs4 + go build .\obfs4proxy + Move-Item -Path .\obfs4proxy.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + - run: + name: Build snowflake-client + command: | + go install git.torproject.org/pluggable-transports/snowflake.git/client@latest + Move-Item -Path ~\go\bin\client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + - run: + name: Build meek-client + command: | + go install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 + Move-Item -Path ~\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - run: name: Build OnionShare command: | @@ -138,10 +161,38 @@ jobs: cd ~\project\desktop C:\Python-32bit\Scripts\poetry install - run: - name: Get tor + name: Download tor (Windows Expert Bundle) + command: | + mkdir ~\Downloads\tor + cd ~\Downloads\tor + $URL = "https://www.torproject.org/dist/torbrowser/11.0.10/tor-win32-0.4.6.10.zip" + $Filename = "tor-win32-0.4.6.10.zip" + $ExpectedHash = "d3f62317507dbe1a1aa74b9e0e03996dbded2143f94409270828f6a8bcdda16a" + Invoke-WebRequest -Uri $URL -OutFile $Filename + $FileHash = Get-FileHash $Filename + if($FileHash.Hash -ne $ExpectedHash) { + throw "Invalid hash" + } + Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor + - run: + name: Build obfs4proxy command: | - cd ~\project\desktop - C:\Python-32bit\Scripts\poetry run python .\scripts\get-tor.py + mkdir ~\Downloads\obfs4proxy + cd Downloads\obfs4proxy + git clone https://gitlab.com/yawning/obfs4 + cd obfs4 + "C:\\Program Files (x86)\\Go\\bin\\go" build .\obfs4proxy + Move-Item -Path .\obfs4proxy.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + - run: + name: Build snowflake-client + command: | + "C:\\Program Files (x86)\\Go\\bin\\go" install git.torproject.org/pluggable-transports/snowflake.git/client@latest + Move-Item -Path ~\go\bin\client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + - run: + name: Build meek-client + command: | + "C:\\Program Files (x86)\\Go\\bin\\go" install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 + Move-Item -Path ~\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - run: name: Build meek command: | diff --git a/desktop/scripts/build-windows.py b/desktop/scripts/build-windows.py index d1729a6f..a0cb3994 100644 --- a/desktop/scripts/build-windows.py +++ b/desktop/scripts/build-windows.py @@ -44,6 +44,7 @@ def run(cmd, cwd=None, error_ok=False): def sign(filename): + click.echo(f"> Signing {filename}") run( [ shutil.which("signtool"), @@ -580,25 +581,41 @@ def codesign(win32_path, win64_path): return for path in paths: - bin_path = os.path.join(path, "onionshare.exe") - click.echo(f"> Signing {bin_path}") - sign(bin_path) - - bin_path = os.path.join(path, "onionshare-cli.exe") - click.echo(f"> Signing {bin_path}") - sign(bin_path) - - bin_path = os.path.join( - path, - "lib", - "onionshare", - "resources", - "tor", - "Tor", - "meek-client.exe", + sign(os.path.join(path, "onionshare.exe")) + sign(os.path.join(path, "onionshare-cli.exe")) + sign( + os.path.join( + path, + "lib", + "onionshare", + "resources", + "tor", + "Tor", + "meek-client.exe", + ) + ) + sign( + os.path.join( + path, + "lib", + "onionshare", + "resources", + "tor", + "Tor", + "obfs4proxy.exe", + ) + ) + sign( + os.path.join( + path, + "lib", + "onionshare", + "resources", + "tor", + "Tor", + "snowflake-client.exe", + ) ) - click.echo(f"> Signing {bin_path}") - sign(bin_path) @main.command() From bf53fbd721a3e2dd060f45031f3f2416ea44102d Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 10 Apr 2022 23:46:40 -0400 Subject: [PATCH 044/154] Work on making all macOS binaries universal2 --- .circleci/config.yml | 162 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 141 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f2d46c6e..3d56b474 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,9 +99,7 @@ jobs: $ExpectedHash = "d3f62317507dbe1a1aa74b9e0e03996dbded2143f94409270828f6a8bcdda16a" Invoke-WebRequest -Uri $URL -OutFile $Filename $FileHash = Get-FileHash $Filename - if($FileHash.Hash -ne $ExpectedHash) { - throw "Invalid hash" - } + if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor - run: name: Build obfs4proxy @@ -170,9 +168,7 @@ jobs: $ExpectedHash = "d3f62317507dbe1a1aa74b9e0e03996dbded2143f94409270828f6a8bcdda16a" Invoke-WebRequest -Uri $URL -OutFile $Filename $FileHash = Get-FileHash $Filename - if($FileHash.Hash -ne $ExpectedHash) { - throw "Invalid hash" - } + if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor - run: name: Build obfs4proxy @@ -193,11 +189,6 @@ jobs: command: | "C:\\Program Files (x86)\\Go\\bin\\go" install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 Move-Item -Path ~\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - - run: - name: Build meek - command: | - cd ~\project\desktop - C:\Python-32bit\python .\scripts\build-meek-client.py - run: name: Build OnionShare command: | @@ -215,6 +206,10 @@ jobs: build-macos: macos: xcode: 12.5.1 + environment: + - INSTALL_X86_64: /Users/distiller/install/x86_64 + - INSTALL_ARM64: /Users/distiller/install/arm64 + - INSTALL_UNIVERSAL2: /Users/distiller/install/universal2 steps: - checkout - run: @@ -222,6 +217,141 @@ jobs: command: | brew install wget brew install go + - run: + name: Make folders for x86_64 and arm64 tor deps + command: mkdir -p $INSTALL_X86_64 $INSTALL_ARM64 $INSTALL_UNIVERSAL2 + - run: + name: Build openssl (x86_64 and arm64) + command: | + # Download and verify + cd ~/Downloads + URL=https://www.openssl.org/source/openssl-3.0.2.tar.gz + FILENAME=openssl-3.0.2.tar.gz + EXPECTED_SHA256=98e91ccead4d4756ae3c9cde5e09191a8e586d9f4d50838e7ec09d6411dfdb63 + wget $URL + SHA256=$(shasum -a 256 $FILENAME | cut -d" " -f1) + if [ "$SHA256" != "$EXPECTED_SHA256" ]; then echo "Failed" && exit -1; fi + tar -xvf $FILENAME + rm $FILENAME + mv openssl-* openssl-x86_64 + cp -r openssl-x86_64 openssl-arm64 + # Build x86_64 + cd openssl-x86_64 + export MACOSX_DEPLOYMENT_TARGET=10.9 + ./Configure darwin64-x86_64-cc shared --prefix=$INSTALL_X86_64 + make + make install + # Build arm64 + cd ../openssl-arm64 + export MACOSX_DEPLOYMENT_TARGET=10.15 + ./Configure enable-rc5 zlib darwin64-arm64-cc no-asm --prefix=$INSTALL_ARM64 + make + make install + - run: + name: Build libevent (x86_64 and arm64) + command: | + # Download and verify + cd ~/Downloads + URL=https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz + FILENAME=libevent-2.1.12-stable.tar.gz + EXPECTED_SHA256=92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb + wget $URL + SHA256=$(shasum -a 256 $FILENAME | cut -d" " -f1) + if [ "$SHA256" != "$EXPECTED_SHA256" ]; then echo "Failed" && exit -1; fi + tar -xvf $FILENAME + rm $FILENAME + mv libevent-* libevent-x86_64 + cp -r libevent-x86_64 libevent-arm64 + # Build x86_64 + cd libevent-x86_64 + export CFLAGS="-arch x86_64" + ./configure --disable-openssl --prefix=$INSTALL_X86_64 + make + make install + # Build arm64 + cd ../libevent-arm64 + export CFLAGS="-arch x86_64" # temporarily, change after the fact + ./configure --disable-openssl --prefix=$INSTALL_ARM64 + # Fix the Makefile so it builds for arm64 + sed -i '' -e 's/host_triplet = x86_64-apple-darwin20.5.0/host_triplet = arm64-apple-darwin20.5.0/g' Makefile + sed -i '' -e 's/CFLAGS = -arch x86_64/CFLAGS = -arch arm64/g' Makefile + sed -i '' -e 's/host = x86_64-apple-darwin20.5.0/host = arm64-apple-darwin20.5.0/g' Makefile + sed -i '' -e 's/host_cpu = x86_64/host_cpu = arm64/g' Makefile + make + make install + - run: + name: Build tor (x86_64 and arm64) + command: | + # Download and verify + cd ~/Downloads + URL=https://dist.torproject.org/tor-0.4.6.10.tar.gz + FILENAME=tor-0.4.6.10.tar.gz + EXPECTED_SHA256=94ccd60e04e558f33be73032bc84ea241660f92f58cfb88789bda6893739e31c + wget $URL + SHA256=$(shasum -a 256 $FILENAME | cut -d" " -f1) + if [ "$SHA256" != "$EXPECTED_SHA256" ]; then echo "Failed" && exit -1; fi + tar -xvf $FILENAME + rm $FILENAME + mv tor-* tor-x86_64 + cp -r tor-x86_64 tor-arm64 + mkdir -p /Users/distiller/compiled/tor-x86-64 /Users/distiller/compiled/tor-amd64 /Users/distiller/compiled/tor-universal + # Build x86_64 + cd tor-x86_64 + export CFLAGS="-arch x86_64" + ./configure --prefix=$INSTALL_X86_64 + make + make install + # Build arm64 + cd ../tor-arm64 + export CFLAGS="-arch arm64" + ./configure --host=x86_64-apple-darwin20.5.0 --disable-tool-name-check --prefix=$INSTALL_ARM64 + make + make install + - run: + name: Make tor universal2 binary + command: | + cp $INSTALL_ARM64/share/tor/* $INSTALL_UNIVERSAL2 + lipo -create -output $INSTALL_UNIVERSAL2/libevent-2.1.7.dylib $INSTALL_ARM64/lib/libevent-2.1.7.dylib $INSTALL_X86_64/lib/libevent-2.1.7.dylib + lipo -create -output $INSTALL_UNIVERSAL2/tor $INSTALL_ARM64/bin/tor $INSTALL_X86_64/bin/tor + - run: + name: Build obfs4proxy (universal2) + command: | + # Get source code + mkdir ~/Downloads/obfs4proxy + cd ~/Downloads/obfs4proxy + git clone https://gitlab.com/yawning/obfs4 + cd obfs4 + # Build x86_64 and arm64 + GOOS=darwin GOARCH=amd64 go build -o obfs4proxy_x86_64 ./obfs4proxy + GOOS=darwin GOARCH=arm64 go build -o obfs4proxy_arm64 ./obfs4proxy + # Combine them + lipo -create -output $INSTALL_UNIVERSAL2/obfs4proxy obfs4proxy_arm64 obfs4proxy_x86_64 + - run: + name: Build snowflake-client (universal2) + command: | + # Get source code + mkdir ~/Downloads/snowflake-client + cd ~/Downloads/snowflake-client + git clone https://git.torproject.org/pluggable-transports/snowflake.git + cd snowflake + # Build x86_64 and arm64 + GOOS=darwin GOARCH=amd64 go build -o snowflake-client_x86_64 ./client + GOOS=darwin GOARCH=arm64 go build -o snowflake-client_arm64 ./client + # Combine them + lipo -create -output $INSTALL_UNIVERSAL2/snowflake-client snowflake-client_arm64 snowflake-client_x86_64 + - run: + name: Build meek-client (universal2) + command: | + # Get source code + mkdir ~/Downloads/meek-client + cd ~/Downloads/meek-client + git clone https://git.torproject.org/pluggable-transports/meek.git + cd meek + # Build x86_64 and arm64 + GOOS=darwin GOARCH=amd64 go build -o meek-client_x86_64 ./meek-client + GOOS=darwin GOARCH=arm64 go build -o meel-client_arm64 ./meek-client + # Combine them + lipo -create -output $INSTALL_UNIVERSAL2/meek-client meek-client_arm64 meek-client_x86_64 - run: name: Install Python 3.9.12 command: | @@ -237,16 +367,6 @@ jobs: command: | cd ~/project/desktop poetry install - - run: - name: Get tor - command: | - cd ~/project/desktop - poetry run python ./scripts/get-tor.py - - run: - name: Build meek - command: | - cd ~/project/desktop - ./scripts/build-meek-client.py - run: name: Build OnionShare command: | From c6c06a34bc14eee1ebad585aae493079b29404b4 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 10 Apr 2022 23:49:29 -0400 Subject: [PATCH 045/154] Fix yaml indention --- .circleci/config.yml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d56b474..041ecf5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,17 +90,17 @@ jobs: cd ~\project\desktop poetry install - run: - name: Download tor (Windows Expert Bundle) - command: | - mkdir ~\Downloads\tor - cd ~\Downloads\tor - $URL = "https://www.torproject.org/dist/torbrowser/11.0.10/tor-win32-0.4.6.10.zip" - $Filename = "tor-win32-0.4.6.10.zip" - $ExpectedHash = "d3f62317507dbe1a1aa74b9e0e03996dbded2143f94409270828f6a8bcdda16a" - Invoke-WebRequest -Uri $URL -OutFile $Filename - $FileHash = Get-FileHash $Filename - if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } - Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor + name: Download tor (Windows Expert Bundle) + command: | + mkdir ~\Downloads\tor + cd ~\Downloads\tor + $URL = "https://www.torproject.org/dist/torbrowser/11.0.10/tor-win32-0.4.6.10.zip" + $Filename = "tor-win32-0.4.6.10.zip" + $ExpectedHash = "d3f62317507dbe1a1aa74b9e0e03996dbded2143f94409270828f6a8bcdda16a" + Invoke-WebRequest -Uri $URL -OutFile $Filename + $FileHash = Get-FileHash $Filename + if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } + Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor - run: name: Build obfs4proxy command: | @@ -159,17 +159,17 @@ jobs: cd ~\project\desktop C:\Python-32bit\Scripts\poetry install - run: - name: Download tor (Windows Expert Bundle) - command: | - mkdir ~\Downloads\tor - cd ~\Downloads\tor - $URL = "https://www.torproject.org/dist/torbrowser/11.0.10/tor-win32-0.4.6.10.zip" - $Filename = "tor-win32-0.4.6.10.zip" - $ExpectedHash = "d3f62317507dbe1a1aa74b9e0e03996dbded2143f94409270828f6a8bcdda16a" - Invoke-WebRequest -Uri $URL -OutFile $Filename - $FileHash = Get-FileHash $Filename - if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } - Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor + name: Download tor (Windows Expert Bundle) + command: | + mkdir ~\Downloads\tor + cd ~\Downloads\tor + $URL = "https://www.torproject.org/dist/torbrowser/11.0.10/tor-win32-0.4.6.10.zip" + $Filename = "tor-win32-0.4.6.10.zip" + $ExpectedHash = "d3f62317507dbe1a1aa74b9e0e03996dbded2143f94409270828f6a8bcdda16a" + Invoke-WebRequest -Uri $URL -OutFile $Filename + $FileHash = Get-FileHash $Filename + if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } + Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor - run: name: Build obfs4proxy command: | From 6fa7a22f62fe01f3548a0c34e3e727d29d2d259d Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Mon, 11 Apr 2022 01:16:06 -0400 Subject: [PATCH 046/154] We don't need openssl --- .circleci/config.yml | 33 +++------------------------------ 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 041ecf5c..1efd207c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -207,9 +207,9 @@ jobs: macos: xcode: 12.5.1 environment: - - INSTALL_X86_64: /Users/distiller/install/x86_64 - - INSTALL_ARM64: /Users/distiller/install/arm64 - - INSTALL_UNIVERSAL2: /Users/distiller/install/universal2 + INSTALL_X86_64: /Users/distiller/install/x86_64 + INSTALL_ARM64: /Users/distiller/install/arm64 + INSTALL_UNIVERSAL2: /Users/distiller/install/universal2 steps: - checkout - run: @@ -220,33 +220,6 @@ jobs: - run: name: Make folders for x86_64 and arm64 tor deps command: mkdir -p $INSTALL_X86_64 $INSTALL_ARM64 $INSTALL_UNIVERSAL2 - - run: - name: Build openssl (x86_64 and arm64) - command: | - # Download and verify - cd ~/Downloads - URL=https://www.openssl.org/source/openssl-3.0.2.tar.gz - FILENAME=openssl-3.0.2.tar.gz - EXPECTED_SHA256=98e91ccead4d4756ae3c9cde5e09191a8e586d9f4d50838e7ec09d6411dfdb63 - wget $URL - SHA256=$(shasum -a 256 $FILENAME | cut -d" " -f1) - if [ "$SHA256" != "$EXPECTED_SHA256" ]; then echo "Failed" && exit -1; fi - tar -xvf $FILENAME - rm $FILENAME - mv openssl-* openssl-x86_64 - cp -r openssl-x86_64 openssl-arm64 - # Build x86_64 - cd openssl-x86_64 - export MACOSX_DEPLOYMENT_TARGET=10.9 - ./Configure darwin64-x86_64-cc shared --prefix=$INSTALL_X86_64 - make - make install - # Build arm64 - cd ../openssl-arm64 - export MACOSX_DEPLOYMENT_TARGET=10.15 - ./Configure enable-rc5 zlib darwin64-arm64-cc no-asm --prefix=$INSTALL_ARM64 - make - make install - run: name: Build libevent (x86_64 and arm64) command: | From 51133e20fd4199a49c0ae890c4988280be687884 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 19 Apr 2022 21:12:42 -0400 Subject: [PATCH 047/154] De-universal2-ize the macOS circleCI config --- .circleci/config.yml | 89 ++++++++++++-------------------------------- 1 file changed, 23 insertions(+), 66 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1efd207c..38a9072a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -206,22 +206,20 @@ jobs: build-macos: macos: xcode: 12.5.1 - environment: - INSTALL_X86_64: /Users/distiller/install/x86_64 - INSTALL_ARM64: /Users/distiller/install/arm64 - INSTALL_UNIVERSAL2: /Users/distiller/install/universal2 steps: - checkout + - environment: + BUILD_PREFIX: /Users/distiller/root + - run: + name: Make BUILD_PREFIX directory + command: mkdir $BUILD_PREFIX - run: name: Install Homebrew dependencies command: | brew install wget brew install go - run: - name: Make folders for x86_64 and arm64 tor deps - command: mkdir -p $INSTALL_X86_64 $INSTALL_ARM64 $INSTALL_UNIVERSAL2 - - run: - name: Build libevent (x86_64 and arm64) + name: Build libevent command: | # Download and verify cd ~/Downloads @@ -232,28 +230,13 @@ jobs: SHA256=$(shasum -a 256 $FILENAME | cut -d" " -f1) if [ "$SHA256" != "$EXPECTED_SHA256" ]; then echo "Failed" && exit -1; fi tar -xvf $FILENAME - rm $FILENAME - mv libevent-* libevent-x86_64 - cp -r libevent-x86_64 libevent-arm64 - # Build x86_64 - cd libevent-x86_64 - export CFLAGS="-arch x86_64" - ./configure --disable-openssl --prefix=$INSTALL_X86_64 - make - make install - # Build arm64 - cd ../libevent-arm64 - export CFLAGS="-arch x86_64" # temporarily, change after the fact - ./configure --disable-openssl --prefix=$INSTALL_ARM64 - # Fix the Makefile so it builds for arm64 - sed -i '' -e 's/host_triplet = x86_64-apple-darwin20.5.0/host_triplet = arm64-apple-darwin20.5.0/g' Makefile - sed -i '' -e 's/CFLAGS = -arch x86_64/CFLAGS = -arch arm64/g' Makefile - sed -i '' -e 's/host = x86_64-apple-darwin20.5.0/host = arm64-apple-darwin20.5.0/g' Makefile - sed -i '' -e 's/host_cpu = x86_64/host_cpu = arm64/g' Makefile + # Build + cd libevent-2.1.12-stable + ./configure --disable-openssl --prefix=$BUILD_PREFIX make make install - run: - name: Build tor (x86_64 and arm64) + name: Build tor command: | # Download and verify cd ~/Downloads @@ -264,71 +247,45 @@ jobs: SHA256=$(shasum -a 256 $FILENAME | cut -d" " -f1) if [ "$SHA256" != "$EXPECTED_SHA256" ]; then echo "Failed" && exit -1; fi tar -xvf $FILENAME - rm $FILENAME - mv tor-* tor-x86_64 - cp -r tor-x86_64 tor-arm64 - mkdir -p /Users/distiller/compiled/tor-x86-64 /Users/distiller/compiled/tor-amd64 /Users/distiller/compiled/tor-universal - # Build x86_64 - cd tor-x86_64 - export CFLAGS="-arch x86_64" - ./configure --prefix=$INSTALL_X86_64 - make - make install - # Build arm64 - cd ../tor-arm64 - export CFLAGS="-arch arm64" - ./configure --host=x86_64-apple-darwin20.5.0 --disable-tool-name-check --prefix=$INSTALL_ARM64 + # Build + cd tor-0.4.6.10 + ./configure --prefix=$BUILD_PREFIX make make install - run: - name: Make tor universal2 binary - command: | - cp $INSTALL_ARM64/share/tor/* $INSTALL_UNIVERSAL2 - lipo -create -output $INSTALL_UNIVERSAL2/libevent-2.1.7.dylib $INSTALL_ARM64/lib/libevent-2.1.7.dylib $INSTALL_X86_64/lib/libevent-2.1.7.dylib - lipo -create -output $INSTALL_UNIVERSAL2/tor $INSTALL_ARM64/bin/tor $INSTALL_X86_64/bin/tor - - run: - name: Build obfs4proxy (universal2) + name: Build obfs4proxy command: | # Get source code mkdir ~/Downloads/obfs4proxy cd ~/Downloads/obfs4proxy git clone https://gitlab.com/yawning/obfs4 cd obfs4 - # Build x86_64 and arm64 - GOOS=darwin GOARCH=amd64 go build -o obfs4proxy_x86_64 ./obfs4proxy - GOOS=darwin GOARCH=arm64 go build -o obfs4proxy_arm64 ./obfs4proxy - # Combine them - lipo -create -output $INSTALL_UNIVERSAL2/obfs4proxy obfs4proxy_arm64 obfs4proxy_x86_64 + # Build + go build -o obfs4proxy ./obfs4proxy - run: - name: Build snowflake-client (universal2) + name: Build snowflake-client command: | # Get source code mkdir ~/Downloads/snowflake-client cd ~/Downloads/snowflake-client git clone https://git.torproject.org/pluggable-transports/snowflake.git cd snowflake - # Build x86_64 and arm64 - GOOS=darwin GOARCH=amd64 go build -o snowflake-client_x86_64 ./client - GOOS=darwin GOARCH=arm64 go build -o snowflake-client_arm64 ./client - # Combine them - lipo -create -output $INSTALL_UNIVERSAL2/snowflake-client snowflake-client_arm64 snowflake-client_x86_64 + # Build + go build -o snowflake-client ./client - run: - name: Build meek-client (universal2) + name: Build meek-client command: | # Get source code mkdir ~/Downloads/meek-client cd ~/Downloads/meek-client git clone https://git.torproject.org/pluggable-transports/meek.git cd meek - # Build x86_64 and arm64 - GOOS=darwin GOARCH=amd64 go build -o meek-client_x86_64 ./meek-client - GOOS=darwin GOARCH=arm64 go build -o meel-client_arm64 ./meek-client - # Combine them - lipo -create -output $INSTALL_UNIVERSAL2/meek-client meek-client_arm64 meek-client_x86_64 + # Build + go build -o meek-client ./meek-client - run: name: Install Python 3.9.12 command: | - wget https://www.python.org/ftp/python/3.9.12/python-3.9.12-macos11.pkg -O ~/Downloads/python.pkg + wget https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg -O ~/Downloads/python.pkg sudo installer -pkg ~/Downloads/python.pkg -target / - run: name: Install poetry From e1122e8a06520fd735ddb81bfde497dd58a530b0 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 19 Apr 2022 21:14:04 -0400 Subject: [PATCH 048/154] Fix environment --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 38a9072a..59926387 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -206,10 +206,10 @@ jobs: build-macos: macos: xcode: 12.5.1 + environment: + BUILD_PREFIX: /Users/distiller/root steps: - checkout - - environment: - BUILD_PREFIX: /Users/distiller/root - run: name: Make BUILD_PREFIX directory command: mkdir $BUILD_PREFIX From b86de120e665556af93147c0f20f807d13cb4730 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 19 Apr 2022 21:24:23 -0400 Subject: [PATCH 049/154] Try caching win32 tor download --- .circleci/config.yml | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59926387..b8ff8c1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,10 +21,6 @@ workflows: requires: - test-cli - test-gui - # - build-snapcraft: - # requires: - # - test-cli - # - test-gui jobs: test-cli: @@ -89,6 +85,8 @@ jobs: command: | cd ~\project\desktop poetry install + - restore_cache: + key: download-tor-win32-0.4.6.10 - run: name: Download tor (Windows Expert Bundle) command: | @@ -101,6 +99,10 @@ jobs: $FileHash = Get-FileHash $Filename if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor + - save_cache: + key: download-tor-win32-0.4.6.10 + paths: + - ~\project\desktop\onionshare\resources\tor - run: name: Build obfs4proxy command: | @@ -311,27 +313,3 @@ jobs: zip -r ~/onionshare-macos.zip OnionShare.app - store_artifacts: path: ~/onionshare-macos.zip - - # build-snapcraft: - # docker: - # - image: snapcore/snapcraft:stable - # working_directory: ~/repo - # steps: - # - checkout - # - run: - # name: Install extra snaps without snapd - # command: | - # mkdir ~/tmp - # install_snap () { cd ~/tmp && snap download $1 && mkdir -p /snap/$1 && unsquashfs -d /snap/$1/current *.snap && rm ~/tmp/*; } - # install_snap "core18" - # install_snap "go" - # install_snap "gnome-3-34-1804" - # install_snap "gnome-3-34-1804-sdk" - # - run: - # name: Build the snap - # command: | - # cd ~/repo - # snapcraft - # mv onionshare-*_amd64.snap ~/onionshare_amd64.snap - # - store_artifacts: - # path: ~/onionshare_amd64.snap From e1da7fc7022fb6a080f3ecc1898bfe6b332de184 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 19 Apr 2022 21:42:37 -0400 Subject: [PATCH 050/154] Cache deps in win64 and win32 --- .circleci/config.yml | 60 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b8ff8c1a..78d0d4a6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,11 +98,17 @@ jobs: Invoke-WebRequest -Uri $URL -OutFile $Filename $FileHash = Get-FileHash $Filename if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } - Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor + mkdir ~\Downloads\tor\tor-win32 + Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\Downloads\tor\tor-win32 - save_cache: key: download-tor-win32-0.4.6.10 paths: - ~\project\desktop\onionshare\resources\tor + - run: + name: Copy tor binary into app + command: cp ~\Downloads\tor\tor-win32\* ~\project\desktop\onionshare\resources\tor + - restore_cache: + key: build-win64-obfs4proxy-v0.0.13 - run: name: Build obfs4proxy command: | @@ -110,18 +116,35 @@ jobs: cd Downloads\obfs4proxy git clone https://gitlab.com/yawning/obfs4 cd obfs4 + git checkout obfs4proxy-0.0.13 go build .\obfs4proxy Move-Item -Path .\obfs4proxy.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + - save_cache: + key: build-win64-obfs4proxy-v0.0.13 + paths: + - ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + - restore_cache: + key: build-win64-snowflake-v2.1.0 - run: name: Build snowflake-client command: | - go install git.torproject.org/pluggable-transports/snowflake.git/client@latest + go install git.torproject.org/pluggable-transports/snowflake.git/client@v2.1.0 Move-Item -Path ~\go\bin\client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + - save_cache: + key: build-win64-snowflake-v2.1.0 + paths: + - ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + - restore_cache: + key: build-win64-meek-v0.37.0 - run: name: Build meek-client command: | go install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 Move-Item -Path ~\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe + - save_cache: + key: build-win64-meek-v0.37.0 + paths: + - ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - run: name: Build OnionShare command: | @@ -160,6 +183,8 @@ jobs: command: | cd ~\project\desktop C:\Python-32bit\Scripts\poetry install + - restore_cache: + key: download-tor-win32-0.4.6.10 - run: name: Download tor (Windows Expert Bundle) command: | @@ -171,26 +196,53 @@ jobs: Invoke-WebRequest -Uri $URL -OutFile $Filename $FileHash = Get-FileHash $Filename if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } - Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\project\desktop\onionshare\resources\tor + mkdir ~\Downloads\tor\tor-win32 + Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\Downloads\tor\tor-win32 + - save_cache: + key: download-tor-win32-0.4.6.10 + paths: + - ~\project\desktop\onionshare\resources\tor + - run: + name: Copy tor binary into app + command: cp ~\Downloads\tor\tor-win32\* ~\project\desktop\onionshare\resources\tor + - restore_cache: + key: build-win32-obfs4proxy-v0.0.13 - run: name: Build obfs4proxy command: | mkdir ~\Downloads\obfs4proxy cd Downloads\obfs4proxy git clone https://gitlab.com/yawning/obfs4 + git checkout obfs4proxy-0.0.13 cd obfs4 "C:\\Program Files (x86)\\Go\\bin\\go" build .\obfs4proxy Move-Item -Path .\obfs4proxy.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + - save_cache: + key: build-win32-obfs4proxy-v0.0.13 + paths: + - ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + - restore_cache: + key: build-win32-snowflake-v2.1.0 - run: name: Build snowflake-client command: | - "C:\\Program Files (x86)\\Go\\bin\\go" install git.torproject.org/pluggable-transports/snowflake.git/client@latest + "C:\\Program Files (x86)\\Go\\bin\\go" install git.torproject.org/pluggable-transports/snowflake.git/client@v2.1.0 Move-Item -Path ~\go\bin\client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + - save_cache: + key: build-win32-snowflake-v2.1.0 + paths: + - ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + - restore_cache: + key: build-win32-meek-v0.37.0 - run: name: Build meek-client command: | "C:\\Program Files (x86)\\Go\\bin\\go" install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 Move-Item -Path ~\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe + - save_cache: + key: build-win32-meek-v0.37.0 + paths: + - ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - run: name: Build OnionShare command: | From f118204cb7a40592de31223a278c98c259db7538 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 19 Apr 2022 22:02:47 -0400 Subject: [PATCH 051/154] Fix some windows cache paths, and in macOS build copy binaries into app --- .circleci/config.yml | 70 ++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 78d0d4a6..bc4948a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,7 +103,7 @@ jobs: - save_cache: key: download-tor-win32-0.4.6.10 paths: - - ~\project\desktop\onionshare\resources\tor + - ~\Downloads\tor\tor-win32 - run: name: Copy tor binary into app command: cp ~\Downloads\tor\tor-win32\* ~\project\desktop\onionshare\resources\tor @@ -201,7 +201,7 @@ jobs: - save_cache: key: download-tor-win32-0.4.6.10 paths: - - ~\project\desktop\onionshare\resources\tor + - ~\Downloads\tor\tor-win32 - run: name: Copy tor binary into app command: cp ~\Downloads\tor\tor-win32\* ~\project\desktop\onionshare\resources\tor @@ -261,21 +261,21 @@ jobs: macos: xcode: 12.5.1 environment: - BUILD_PREFIX: /Users/distiller/root + BINARY_DIR: /Users/distiller/bin steps: - checkout - - run: - name: Make BUILD_PREFIX directory - command: mkdir $BUILD_PREFIX - run: name: Install Homebrew dependencies command: | brew install wget brew install go + - restore_cache: + key: build-libevent-2.1.12-tor-0.4.6.10 - run: - name: Build libevent + name: Build libevent and tor command: | - # Download and verify + mkdir -p $BINARY_DIR/tor + # Download and verify libevent cd ~/Downloads URL=https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz FILENAME=libevent-2.1.12-stable.tar.gz @@ -284,15 +284,12 @@ jobs: SHA256=$(shasum -a 256 $FILENAME | cut -d" " -f1) if [ "$SHA256" != "$EXPECTED_SHA256" ]; then echo "Failed" && exit -1; fi tar -xvf $FILENAME - # Build + # Build libevent cd libevent-2.1.12-stable - ./configure --disable-openssl --prefix=$BUILD_PREFIX + ./configure --disable-openssl --prefix=$BINARY_DIR/tor make make install - - run: - name: Build tor - command: | - # Download and verify + # Download and verify tor cd ~/Downloads URL=https://dist.torproject.org/tor-0.4.6.10.tar.gz FILENAME=tor-0.4.6.10.tar.gz @@ -301,11 +298,17 @@ jobs: SHA256=$(shasum -a 256 $FILENAME | cut -d" " -f1) if [ "$SHA256" != "$EXPECTED_SHA256" ]; then echo "Failed" && exit -1; fi tar -xvf $FILENAME - # Build + # Build tor cd tor-0.4.6.10 - ./configure --prefix=$BUILD_PREFIX + ./configure --prefix=$BINARY_DIR/tor make make install + - save_cache: + key: build-libevent-2.1.12-tor-0.4.6.10 + paths: + - /Users/distiller/bin/tor + - restore_cache: + key: build-macos-obfs4proxy-v0.0.13 - run: name: Build obfs4proxy command: | @@ -314,8 +317,15 @@ jobs: cd ~/Downloads/obfs4proxy git clone https://gitlab.com/yawning/obfs4 cd obfs4 + git checkout obfs4proxy-0.0.13 # Build - go build -o obfs4proxy ./obfs4proxy + go build -o $BINARY_DIR/obfs4proxy ./obfs4proxy + - save_cache: + key: build-macos-obfs4proxy-v0.0.13 + paths: + - /Users/distiller/bin/obfs4proxy + - restore_cache: + key: build-macos-snowflake-v2.1.0 - run: name: Build snowflake-client command: | @@ -324,8 +334,15 @@ jobs: cd ~/Downloads/snowflake-client git clone https://git.torproject.org/pluggable-transports/snowflake.git cd snowflake + git checkout v2.1.0 # Build - go build -o snowflake-client ./client + go build -o $BINARY_DIR/snowflake-client ./client + - save_cache: + key: build-macos-snowflake-v2.1.0 + paths: + - /Users/distiller/bin/snowflake-client + - restore_cache: + key: build-macos-meek-v0.37.0 - run: name: Build meek-client command: | @@ -334,8 +351,23 @@ jobs: cd ~/Downloads/meek-client git clone https://git.torproject.org/pluggable-transports/meek.git cd meek + git checkout v0.37.0 # Build - go build -o meek-client ./meek-client + go build -o $BINARY_DIR/meek-client ./meek-client + - save_cache: + key: build-macos-meek-v0.37.0 + paths: + - /Users/distiller/bin/meek-client + - run: + name: Copy binaries into app + command: | + export DEST=~\project\desktop\onionshare\resources\tor + cp $BINARY_DIR/tor/bin/tor $DEST + cp $BINARY_DIR/tor/lib/libevent-2.1.7.dylib $DEST + cp $BINARY_DIR/tor/share/tor/geoip* $DEST + cp $BINARY_DIR/obfs4proxy $DEST + cp $BINARY_DIR/snowflake-client $DEST + cp $BINARY_DIR/meek-client $DEST - run: name: Install Python 3.9.12 command: | From 45adc8674d6644d34510f1eb4fb3aa2e1ec58ce4 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 19 Apr 2022 22:12:30 -0400 Subject: [PATCH 052/154] Cache poetry deps when running tests --- .circleci/config.yml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc4948a9..5c52be83 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,6 @@ jobs: test-cli: docker: - image: cimg/python:3.9 - working_directory: ~/repo steps: - checkout - run: @@ -34,13 +33,21 @@ jobs: command: | sudo apt-get update sudo apt-get -y install tor obfs4proxy - pip install poetry - cd ~/repo/cli + - restore_cache: + key: test-cli-poetry-deps-{{ checksum "~/project/cli/poetry.lock" }} + - run: + name: Install poetry dependencies + command: | + cd ~/project/cli poetry install + - save_cache: + key: test-cli-poetry-deps-{{ checksum "~/project/cli/poetry.lock" }} + paths: + - /home/circleci/.cache/pypoetry/virtualenvs - run: name: Run tests command: | - cd ~/repo/cli + cd ~/project/cli poetry run pytest -v ./tests poetry run onionshare-cli --local-only ./tests --auto-stop-timer 2 poetry run onionshare-cli --local-only --receive --auto-stop-timer 2 @@ -50,7 +57,6 @@ jobs: test-gui: docker: - image: cimg/python:3.9 - working_directory: ~/repo steps: - checkout - run: @@ -59,12 +65,21 @@ jobs: sudo apt-get update sudo apt-get install -y tor obfs4proxy gcc python3-dev python3-pyside2.qtcore python3-pyside2.qtwidgets python3-pyside2.qtgui sudo apt-get install -y xvfb x11-utils libxkbcommon-x11-0 libxcb-randr0-dev libxcb-xtest0-dev libxcb-xinerama0-dev libxcb-shape0-dev libxcb-xkb-dev libxcb-render-util0 libxcb-icccm4 libxcb-keysyms1 libxcb-image0 - cd ~/repo/desktop + - restore_cache: + key: test-gui-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }} + - run: + name: Install poetry dependencies + command: | + cd ~/project/cli poetry install + - save_cache: + key: test-gui-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }} + paths: + - /home/circleci/.cache/pypoetry/virtualenvs - run: name: Run tests command: | - cd ~/repo/desktop + cd ~/project/desktop QT_DEBUG_PLUGINS=1 xvfb-run poetry run pytest -v ./tests/test_gui_*.py build-win64: From fa8c99038065d873c09bd1982000c75144413a25 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 19 Apr 2022 22:15:49 -0400 Subject: [PATCH 053/154] Oops, install correct poetry deps for GUI tests --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5c52be83..0e70b162 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,14 +66,14 @@ jobs: sudo apt-get install -y tor obfs4proxy gcc python3-dev python3-pyside2.qtcore python3-pyside2.qtwidgets python3-pyside2.qtgui sudo apt-get install -y xvfb x11-utils libxkbcommon-x11-0 libxcb-randr0-dev libxcb-xtest0-dev libxcb-xinerama0-dev libxcb-shape0-dev libxcb-xkb-dev libxcb-render-util0 libxcb-icccm4 libxcb-keysyms1 libxcb-image0 - restore_cache: - key: test-gui-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }} + key: test-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }} - run: name: Install poetry dependencies command: | - cd ~/project/cli + cd ~/project/desktop poetry install - save_cache: - key: test-gui-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }} + key: test-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }} paths: - /home/circleci/.cache/pypoetry/virtualenvs - run: From 17b0b2b3181d73c15eb3b90f0953f43d636d2584 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Tue, 19 Apr 2022 22:26:03 -0400 Subject: [PATCH 054/154] Make gui tests optional because they keep failing --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e70b162..32cc49eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,15 +12,15 @@ workflows: - build-win64: requires: - test-cli - - test-gui + # - test-gui - build-win32: requires: - test-cli - - test-gui + # - test-gui - build-macos: requires: - test-cli - - test-gui + # - test-gui jobs: test-cli: From ad6e7f05be7b4575546f215517cfba7c84258a7f Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 25 May 2022 21:46:12 -0700 Subject: [PATCH 055/154] Update tor to 0.4.7.7, and focus on just build-win64 --- .circleci/config.yml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 32cc49eb..aa0dc712 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,19 +8,19 @@ workflows: test_and_build: jobs: - test-cli - - test-gui + # - test-gui - build-win64: requires: - test-cli # - test-gui - - build-win32: - requires: - - test-cli - # - test-gui - - build-macos: - requires: - - test-cli - # - test-gui + # - build-win32: + # requires: + # - test-cli + # # - test-gui + # - build-macos: + # requires: + # - test-cli + # # - test-gui jobs: test-cli: @@ -101,22 +101,22 @@ jobs: cd ~\project\desktop poetry install - restore_cache: - key: download-tor-win32-0.4.6.10 + key: download-tor-win32-0.4.7.7 - run: name: Download tor (Windows Expert Bundle) command: | mkdir ~\Downloads\tor cd ~\Downloads\tor - $URL = "https://www.torproject.org/dist/torbrowser/11.0.10/tor-win32-0.4.6.10.zip" - $Filename = "tor-win32-0.4.6.10.zip" - $ExpectedHash = "d3f62317507dbe1a1aa74b9e0e03996dbded2143f94409270828f6a8bcdda16a" + $URL = "https://www.torproject.org/dist/torbrowser/11.0.13/tor-win32-0.4.7.7.zip" + $Filename = "tor-win32-0.4.7.7.zip" + $ExpectedHash = "6af0088460d61a5e60971aea94562a3bed2fb597da6b8ac77948a5f6424dfa29" Invoke-WebRequest -Uri $URL -OutFile $Filename $FileHash = Get-FileHash $Filename if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } mkdir ~\Downloads\tor\tor-win32 - Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\Downloads\tor\tor-win32 + Expand-Archive -LiteralPath tor-win32-0.4.7.7.zip -DestinationPath ~\Downloads\tor\tor-win32 - save_cache: - key: download-tor-win32-0.4.6.10 + key: download-tor-win32-0.4.7.7 paths: - ~\Downloads\tor\tor-win32 - run: @@ -199,22 +199,22 @@ jobs: cd ~\project\desktop C:\Python-32bit\Scripts\poetry install - restore_cache: - key: download-tor-win32-0.4.6.10 + key: download-tor-win32-0.4.7.7 - run: name: Download tor (Windows Expert Bundle) command: | mkdir ~\Downloads\tor cd ~\Downloads\tor - $URL = "https://www.torproject.org/dist/torbrowser/11.0.10/tor-win32-0.4.6.10.zip" - $Filename = "tor-win32-0.4.6.10.zip" - $ExpectedHash = "d3f62317507dbe1a1aa74b9e0e03996dbded2143f94409270828f6a8bcdda16a" + $URL = "https://www.torproject.org/dist/torbrowser/11.0.13/tor-win32-0.4.7.7.zip" + $Filename = "tor-win32-0.4.7.7.zip" + $ExpectedHash = "6af0088460d61a5e60971aea94562a3bed2fb597da6b8ac77948a5f6424dfa29" Invoke-WebRequest -Uri $URL -OutFile $Filename $FileHash = Get-FileHash $Filename if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } mkdir ~\Downloads\tor\tor-win32 - Expand-Archive -LiteralPath tor-win32-0.4.6.10.zip -DestinationPath ~\Downloads\tor\tor-win32 + Expand-Archive -LiteralPath tor-win32-0.4.7.7.zip -DestinationPath ~\Downloads\tor\tor-win32 - save_cache: - key: download-tor-win32-0.4.6.10 + key: download-tor-win32-0.4.7.7 paths: - ~\Downloads\tor\tor-win32 - run: From bfd7e970e0c48f5cecf232de49e72b1e44cca22b Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 26 May 2022 18:01:37 -0700 Subject: [PATCH 056/154] Comment everything out outside of building libevent --- .circleci/config.yml | 248 +++++++++++++++++++++++++++++-------------- 1 file changed, 166 insertions(+), 82 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aa0dc712..46d49676 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ orbs: workflows: version: 2 - test_and_build: + ci: jobs: - test-cli # - test-gui @@ -88,91 +88,175 @@ jobs: shell: powershell.exe steps: - checkout + # # Install python + # - run: + # name: Install Python 3.9.12 + # command: | + # choco install python3 --version=3.9.12 + + # # Install poetry + # - run: + # name: Install poetry + # command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - + + # # Install poetry dependencies + # - restore_cache: + # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }} + # - run: + # name: Install poetry dependencies + # command: | + # cd C:\Users\circleci\project\desktop + # poetry install + # - save_cache: + # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }} + # paths: + # - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs + - run: - name: Install Python 3.9.12 + name: Install msys2 and dependencies command: | - choco install python3 --version=3.9.12 + choco install msys2 --params "/NoUpdate /InstallDir:C:\msys2" + refreshenv + C:\msys2\usr\bin\bash -c "/c/msys2/usr/bin/pacman -S --needed --noconfirm autoconf automake libtool make clang" + mkdir C:\Users\circleci\src + + # Build libevent - run: - name: Install poetry - command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - - - run: - name: Install poetry dependencies + name: Build libevent command: | - cd ~\project\desktop - poetry install - - restore_cache: - key: download-tor-win32-0.4.7.7 - - run: - name: Download tor (Windows Expert Bundle) - command: | - mkdir ~\Downloads\tor - cd ~\Downloads\tor - $URL = "https://www.torproject.org/dist/torbrowser/11.0.13/tor-win32-0.4.7.7.zip" - $Filename = "tor-win32-0.4.7.7.zip" - $ExpectedHash = "6af0088460d61a5e60971aea94562a3bed2fb597da6b8ac77948a5f6424dfa29" - Invoke-WebRequest -Uri $URL -OutFile $Filename - $FileHash = Get-FileHash $Filename - if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } - mkdir ~\Downloads\tor\tor-win32 - Expand-Archive -LiteralPath tor-win32-0.4.7.7.zip -DestinationPath ~\Downloads\tor\tor-win32 - - save_cache: - key: download-tor-win32-0.4.7.7 - paths: - - ~\Downloads\tor\tor-win32 - - run: - name: Copy tor binary into app - command: cp ~\Downloads\tor\tor-win32\* ~\project\desktop\onionshare\resources\tor - - restore_cache: - key: build-win64-obfs4proxy-v0.0.13 - - run: - name: Build obfs4proxy - command: | - mkdir ~\Downloads\obfs4proxy - cd Downloads\obfs4proxy - git clone https://gitlab.com/yawning/obfs4 - cd obfs4 - git checkout obfs4proxy-0.0.13 - go build .\obfs4proxy - Move-Item -Path .\obfs4proxy.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe - - save_cache: - key: build-win64-obfs4proxy-v0.0.13 - paths: - - ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe - - restore_cache: - key: build-win64-snowflake-v2.1.0 - - run: - name: Build snowflake-client - command: | - go install git.torproject.org/pluggable-transports/snowflake.git/client@v2.1.0 - Move-Item -Path ~\go\bin\client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe - - save_cache: - key: build-win64-snowflake-v2.1.0 - paths: - - ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe - - restore_cache: - key: build-win64-meek-v0.37.0 - - run: - name: Build meek-client - command: | - go install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 - Move-Item -Path ~\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - - save_cache: - key: build-win64-meek-v0.37.0 - paths: - - ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - - run: - name: Build OnionShare - command: | - cd ~\project\desktop - poetry run python .\setup-freeze.py build - poetry run python .\scripts\build-windows.py cleanup-build - - run: - name: Compress - command: | - mv ~\project\desktop\build\exe.win-amd64-3.9\ ~\onionshare-win64 - Compress-Archive -LiteralPath ~\onionshare-win64 -DestinationPath ~\onionshare-win64.zip - - store_artifacts: - path: ~\onionshare-win64.zip + $LibeventTag = "release-2.1.12-stable" + + # Get source + gpg --recv-key 9E3AC83A27974B84D1B3401DB86086848EF8686D + cd C:\Users\circleci\src + git clone https://github.com/libevent/libevent.git + cd libevent + git tag -v $LibeventTag + if($? -ne $true) { throw "libevent tag doesn't verify" } + git checkout $LibeventTag + + # Build + $BashScript = @" + #!/bin/bash + export PATH=/c/msys2/usr/bin:$PATH + cd /c/Users/circleci/src/libevent + ./autogen.sh + ./configure --prefix=/opt/libevent-build --with-pic + make -j8 install + "@ + + # # Build tor + # - run: + # name: Build tor + # command: | + # $LibeventTag = "release-2.1.12-stable" + # $TorTag = "tor-0.4.7.7" + + # choco install msys2 --params "/NoUpdate /InstallDir:C:\msys2" + # C:\msys2\usr\bin\bash -c "pacman --sync --noconfirm msys/autoconf" + # C:\msys2\usr\bin\bash -c "pacman --sync --noconfirm msys/automake" + + # # Get libevent source + # gpg --recv-key 9E3AC83A27974B84D1B3401DB86086848EF8686D + # cd C:\Users\circleci\src + # git clone https://github.com/libevent/libevent.git + # cd libevent + # git tag -v $LibeventTag + # if($? -ne $true) { throw "libevent tag doesn't verify" } + # git checkout $LibeventTag + + # # Build libevent + # $BashScript = @" + # #!/bin/bash + # export PATH=/c/msys2/usr/bin:$PATH + # cd /c/Users/circleci/src/libevent + # ./autogen.sh + # ./configure --prefix=/opt/libevent-build --with-pic + # make -j8 install + # "@ + + # # Get tor source + # gpg --recv-key B74417EDDF22AC9F9E90F49142E86A2A11F48D36 + # cd C:\Users\circleci\src + # git clone https://git.torproject.org/tor.git + # cd tor + # git tag -v $TorTag + # if($? -ne $true) { throw "tor tag doesn't verify" } + # git checkout $TorTag + + # - restore_cache: + # key: download-tor-win32-0.4.7.7 + # - run: + # name: Download tor (Windows Expert Bundle) + # command: | + # mkdir ~\Downloads\tor + # cd ~\Downloads\tor + # $URL = "https://www.torproject.org/dist/torbrowser/11.0.13/tor-win32-0.4.7.7.zip" + # $Filename = "tor-win32-0.4.7.7.zip" + # $ExpectedHash = "6af0088460d61a5e60971aea94562a3bed2fb597da6b8ac77948a5f6424dfa29" + # Invoke-WebRequest -Uri $URL -OutFile $Filename + # $FileHash = Get-FileHash $Filename + # if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } + # mkdir ~\Downloads\tor\tor-win32 + # Expand-Archive -LiteralPath tor-win32-0.4.7.7.zip -DestinationPath ~\Downloads\tor\tor-win32 + # - save_cache: + # key: download-tor-win32-0.4.7.7 + # paths: + # - ~\Downloads\tor\tor-win32 + # - run: + # name: Copy tor binary into app + # command: cp ~\Downloads\tor\tor-win32\* ~\project\desktop\onionshare\resources\tor + # - restore_cache: + # key: build-win64-obfs4proxy-v0.0.13 + # - run: + # name: Build obfs4proxy + # command: | + # mkdir ~\Downloads\obfs4proxy + # cd Downloads\obfs4proxy + # git clone https://gitlab.com/yawning/obfs4 + # cd obfs4 + # git checkout obfs4proxy-0.0.13 + # go build .\obfs4proxy + # Move-Item -Path .\obfs4proxy.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + # - save_cache: + # key: build-win64-obfs4proxy-v0.0.13 + # paths: + # - ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + # - restore_cache: + # key: build-win64-snowflake-v2.1.0 + # - run: + # name: Build snowflake-client + # command: | + # go install git.torproject.org/pluggable-transports/snowflake.git/client@v2.1.0 + # Move-Item -Path ~\go\bin\client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + # - save_cache: + # key: build-win64-snowflake-v2.1.0 + # paths: + # - ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + # - restore_cache: + # key: build-win64-meek-v0.37.0 + # - run: + # name: Build meek-client + # command: | + # go install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 + # Move-Item -Path ~\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe + # - save_cache: + # key: build-win64-meek-v0.37.0 + # paths: + # - ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe + # - run: + # name: Build OnionShare + # command: | + # cd ~\project\desktop + # poetry run python .\setup-freeze.py build + # poetry run python .\scripts\build-windows.py cleanup-build + # - run: + # name: Compress + # command: | + # mv ~\project\desktop\build\exe.win-amd64-3.9\ ~\onionshare-win64 + # Compress-Archive -LiteralPath ~\onionshare-win64 -DestinationPath ~\onionshare-win64.zip + # - store_artifacts: + # path: ~\onionshare-win64.zip build-win32: executor: From 3fea3019aa11f5db052fe694ac22180c9ea09610 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 26 May 2022 18:36:54 -0700 Subject: [PATCH 057/154] Try building openssl and libevent --- .circleci/config.yml | 91 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 46d49676..7fe94c4d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,6 +86,10 @@ jobs: executor: name: win/default shell: powershell.exe + environment: + OPENSSL_TAG: "openssl-3.0.3" + LIBEVENT_TAG: "release-2.1.12-stable" + TOR_TAG: "tor-0.4.7.7" steps: - checkout # # Install python @@ -119,31 +123,80 @@ jobs: refreshenv C:\msys2\usr\bin\bash -c "/c/msys2/usr/bin/pacman -S --needed --noconfirm autoconf automake libtool make clang" mkdir C:\Users\circleci\src + mkdir C:\Users\circleci\build + + # Build openssl + - restore_cache: + key: "build-win64-openssl-{{ .Environment.OPENSSL_TAG }}" + - run: + name: Build openssl + command: | + if (Test-Path -Path "C:\Users\circleci\build\openssl") { + Write-Output "openssl already compiled, skipping" + } else { + # Get source + gpg --recv-key 8657ABB260F056B1E5190839D9C4D26D0E604491 + cd C:\Users\circleci\src + git clone https://github.com/openssl/openssl.git + cd openssl + git tag -v $OPENSSL_TAG + if($? -ne $true) { throw "openssl tag doesn't verify" } + git checkout $OPENSSL_TAG + + # Build + $BuildScript = { + #!/bin/bash + export PATH=/c/msys2/usr/bin:$PATH + cd /c/Users/circleci/src/openssl + ./Configure --prefix=/opt/openssl-build --openssldir=/usr/local/ssl '-Wl,-rpath,$(LIBRPATH)' Cygwin-x86_64 + make -j$(nproc) + make install + cp -r /opt/openssl-build /c/Users/circleci/build/openssl + } + Set-Content -Path C:\Users\circleci\src\build-openssl.sh -Value $BuildScript.ToString() + C:\msys2\usr\bin\bash -c "/c/Users/circleci/src/build-openssl.sh" + } + - save_cache: + key: "build-win64-openssl-{{ .Environment.OPENSSL_TAG }}" + paths: + - C:\Users\circleci\build\openssl # Build libevent + - restore_cache: + key: "build-win64-livevent-{{ .Environment.LIBEVENT_TAG }}" - run: name: Build libevent command: | - $LibeventTag = "release-2.1.12-stable" + if (Test-Path -Path "C:\Users\circleci\build\openssl") { + Write-Output "openssl already compiled, skipping" + } else { + # Get source + gpg --recv-key 9E3AC83A27974B84D1B3401DB86086848EF8686D + cd C:\Users\circleci\src + git clone https://github.com/libevent/libevent.git + cd libevent + git tag -v $LIBEVENT_TAG + if($? -ne $true) { throw "libevent tag doesn't verify" } + git checkout $LIBEVENT_TAG - # Get source - gpg --recv-key 9E3AC83A27974B84D1B3401DB86086848EF8686D - cd C:\Users\circleci\src - git clone https://github.com/libevent/libevent.git - cd libevent - git tag -v $LibeventTag - if($? -ne $true) { throw "libevent tag doesn't verify" } - git checkout $LibeventTag - - # Build - $BashScript = @" - #!/bin/bash - export PATH=/c/msys2/usr/bin:$PATH - cd /c/Users/circleci/src/libevent - ./autogen.sh - ./configure --prefix=/opt/libevent-build --with-pic - make -j8 install - "@ + # Build + $BuildScript = { + #!/bin/bash + export PATH=/c/msys2/usr/bin:$PATH + cd /c/Users/circleci/src/libevent + ./autogen.sh + ./configure --prefix=/opt/libevent-build --with-pic + make -j$(nproc) + make install + cp -r /opt/libevent-build /c/Users/circleci/build/libevent + } + Set-Content -Path C:\Users\circleci\src\build-libevent.sh -Value $BuildScript.ToString() + C:\msys2\usr\bin\bash -c "/c/Users/circleci/src/build-libevent.sh" + } + - save_cache: + key: "build-win64-libevent-{{ .Environment.LIBEVENT_TAG }}" + paths: + - C:\Users\circleci\build\libevent # # Build tor # - run: From 1b67e6a6fd6a8e6a99477cb4399d9db52c86f1cd Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 26 May 2022 19:37:41 -0700 Subject: [PATCH 058/154] Install perl-Pod-Parser to get pod2man --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7fe94c4d..be5c4fa6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,7 +121,7 @@ jobs: command: | choco install msys2 --params "/NoUpdate /InstallDir:C:\msys2" refreshenv - C:\msys2\usr\bin\bash -c "/c/msys2/usr/bin/pacman -S --needed --noconfirm autoconf automake libtool make clang" + C:\msys2\usr\bin\bash -c "/c/msys2/usr/bin/pacman -S --needed --noconfirm autoconf automake libtool make clang perl-Pod-Parser" mkdir C:\Users\circleci\src mkdir C:\Users\circleci\build @@ -146,7 +146,7 @@ jobs: # Build $BuildScript = { #!/bin/bash - export PATH=/c/msys2/usr/bin:$PATH + export PATH=/c/msys2/usr/bin:/c/msys2/usr/bin/core_perl:$PATH cd /c/Users/circleci/src/openssl ./Configure --prefix=/opt/openssl-build --openssldir=/usr/local/ssl '-Wl,-rpath,$(LIBRPATH)' Cygwin-x86_64 make -j$(nproc) From cc0ab0ef799b7bcc0026e50c838bb2e3041d8ad4 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 26 May 2022 20:20:37 -0700 Subject: [PATCH 059/154] Switch to bash --- .circleci/config.yml | 109 +++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 56 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index be5c4fa6..838f6056 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,7 +85,7 @@ jobs: build-win64: executor: name: win/default - shell: powershell.exe + shell: bash.exe environment: OPENSSL_TAG: "openssl-3.0.3" LIBEVENT_TAG: "release-2.1.12-stable" @@ -121,9 +121,9 @@ jobs: command: | choco install msys2 --params "/NoUpdate /InstallDir:C:\msys2" refreshenv - C:\msys2\usr\bin\bash -c "/c/msys2/usr/bin/pacman -S --needed --noconfirm autoconf automake libtool make clang perl-Pod-Parser" - mkdir C:\Users\circleci\src - mkdir C:\Users\circleci\build + /c/msys2/usr/bin/pacman -S --needed --noconfirm autoconf automake libtool make clang perl-Pod-Parser + mkdir /c/Users/circleci/src + mkdir /c/Users/circleci/build # Build openssl - restore_cache: @@ -131,72 +131,69 @@ jobs: - run: name: Build openssl command: | - if (Test-Path -Path "C:\Users\circleci\build\openssl") { - Write-Output "openssl already compiled, skipping" - } else { + export PATH=/c/msys2/usr/bin:/c/msys2/usr/bin/core_perl:$PATH + if [[ -d "/c/Users/circleci/build/openssl" ]]; then + echo "openssl already compiled, skipping" + elif # Get source gpg --recv-key 8657ABB260F056B1E5190839D9C4D26D0E604491 - cd C:\Users\circleci\src + cd /c/Users/circleci/src git clone https://github.com/openssl/openssl.git cd openssl git tag -v $OPENSSL_TAG - if($? -ne $true) { throw "openssl tag doesn't verify" } + if [ $? -eq 0 ]; then + echo "openssl tag doesn't verify" + exit -1 + fi git checkout $OPENSSL_TAG # Build - $BuildScript = { - #!/bin/bash - export PATH=/c/msys2/usr/bin:/c/msys2/usr/bin/core_perl:$PATH - cd /c/Users/circleci/src/openssl - ./Configure --prefix=/opt/openssl-build --openssldir=/usr/local/ssl '-Wl,-rpath,$(LIBRPATH)' Cygwin-x86_64 - make -j$(nproc) - make install - cp -r /opt/openssl-build /c/Users/circleci/build/openssl - } - Set-Content -Path C:\Users\circleci\src\build-openssl.sh -Value $BuildScript.ToString() - C:\msys2\usr\bin\bash -c "/c/Users/circleci/src/build-openssl.sh" - } + ./Configure --prefix=/opt/openssl-build --openssldir=/usr/local/ssl '-Wl,-rpath,$(LIBRPATH)' Cygwin-x86_64 + make -j$(nproc) + make install + cp -r /opt/openssl-build /c/Users/circleci/build/openssl + fi - save_cache: key: "build-win64-openssl-{{ .Environment.OPENSSL_TAG }}" paths: - C:\Users\circleci\build\openssl - # Build libevent - - restore_cache: - key: "build-win64-livevent-{{ .Environment.LIBEVENT_TAG }}" - - run: - name: Build libevent - command: | - if (Test-Path -Path "C:\Users\circleci\build\openssl") { - Write-Output "openssl already compiled, skipping" - } else { - # Get source - gpg --recv-key 9E3AC83A27974B84D1B3401DB86086848EF8686D - cd C:\Users\circleci\src - git clone https://github.com/libevent/libevent.git - cd libevent - git tag -v $LIBEVENT_TAG - if($? -ne $true) { throw "libevent tag doesn't verify" } - git checkout $LIBEVENT_TAG + # # Build libevent + # - restore_cache: + # key: "build-win64-livevent-{{ .Environment.LIBEVENT_TAG }}" + # - run: + # name: Build libevent + # command: | + # if (Test-Path -Path "C:\Users\circleci\build\openssl") { + # Write-Output "openssl already compiled, skipping" + # } else { + # # Get source + # gpg --recv-key 9E3AC83A27974B84D1B3401DB86086848EF8686D + # cd C:\Users\circleci\src + # git clone https://github.com/libevent/libevent.git + # cd libevent + # git tag -v $LIBEVENT_TAG + # if($? -ne $true) { throw "libevent tag doesn't verify" } + # git checkout $LIBEVENT_TAG - # Build - $BuildScript = { - #!/bin/bash - export PATH=/c/msys2/usr/bin:$PATH - cd /c/Users/circleci/src/libevent - ./autogen.sh - ./configure --prefix=/opt/libevent-build --with-pic - make -j$(nproc) - make install - cp -r /opt/libevent-build /c/Users/circleci/build/libevent - } - Set-Content -Path C:\Users\circleci\src\build-libevent.sh -Value $BuildScript.ToString() - C:\msys2\usr\bin\bash -c "/c/Users/circleci/src/build-libevent.sh" - } - - save_cache: - key: "build-win64-libevent-{{ .Environment.LIBEVENT_TAG }}" - paths: - - C:\Users\circleci\build\libevent + # # Build + # $BuildScript = { + # #!/bin/bash + # export PATH=/c/msys2/usr/bin:$PATH + # cd /c/Users/circleci/src/libevent + # ./autogen.sh + # ./configure --prefix=/opt/libevent-build --with-pic + # make -j$(nproc) + # make install + # cp -r /opt/libevent-build /c/Users/circleci/build/libevent + # } + # Set-Content -Path C:\Users\circleci\src\build-libevent.sh -Value $BuildScript.ToString() + # C:\msys2\usr\bin\bash -c "/c/Users/circleci/src/build-libevent.sh" + # } + # - save_cache: + # key: "build-win64-libevent-{{ .Environment.LIBEVENT_TAG }}" + # paths: + # - C:\Users\circleci\build\libevent # # Build tor # - run: From fd32fb5cc4a1060c57c1a870d11d14805031200a Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 27 May 2022 07:16:39 -0700 Subject: [PATCH 060/154] Move tor stuff into build-tor-win64 --- .circleci/config.yml | 74 ++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 838f6056..40cae7f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,10 +9,14 @@ workflows: jobs: - test-cli # - test-gui - - build-win64: + - build-tor-win64: requires: - test-cli - # - test-gui + # - build-win64: + # requires: + # - build-tor-win64 + # - test-cli + # # - test-gui # - build-win32: # requires: # - test-cli @@ -82,7 +86,7 @@ jobs: cd ~/project/desktop QT_DEBUG_PLUGINS=1 xvfb-run poetry run pytest -v ./tests/test_gui_*.py - build-win64: + build-tor-win64: executor: name: win/default shell: bash.exe @@ -92,30 +96,6 @@ jobs: TOR_TAG: "tor-0.4.7.7" steps: - checkout - # # Install python - # - run: - # name: Install Python 3.9.12 - # command: | - # choco install python3 --version=3.9.12 - - # # Install poetry - # - run: - # name: Install poetry - # command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - - - # # Install poetry dependencies - # - restore_cache: - # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }} - # - run: - # name: Install poetry dependencies - # command: | - # cd C:\Users\circleci\project\desktop - # poetry install - # - save_cache: - # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }} - # paths: - # - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs - - run: name: Install msys2 and dependencies command: | @@ -127,7 +107,7 @@ jobs: # Build openssl - restore_cache: - key: "build-win64-openssl-{{ .Environment.OPENSSL_TAG }}" + key: "build-win64-openssl-3.0.3" - run: name: Build openssl command: | @@ -141,7 +121,7 @@ jobs: git clone https://github.com/openssl/openssl.git cd openssl git tag -v $OPENSSL_TAG - if [ $? -eq 0 ]; then + if [ $? -ne 0 ]; then echo "openssl tag doesn't verify" exit -1 fi @@ -154,7 +134,7 @@ jobs: cp -r /opt/openssl-build /c/Users/circleci/build/openssl fi - save_cache: - key: "build-win64-openssl-{{ .Environment.OPENSSL_TAG }}" + key: "build-win64-openssl-3.0.3" paths: - C:\Users\circleci\build\openssl @@ -234,6 +214,40 @@ jobs: # if($? -ne $true) { throw "tor tag doesn't verify" } # git checkout $TorTag + build-win64: + executor: + name: win/default + shell: bash.exe + environment: + OPENSSL_TAG: "openssl-3.0.3" + LIBEVENT_TAG: "release-2.1.12-stable" + TOR_TAG: "tor-0.4.7.7" + steps: + - checkout + # # Install python + # - run: + # name: Install Python 3.9.12 + # command: | + # choco install python3 --version=3.9.12 + + # # Install poetry + # - run: + # name: Install poetry + # command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - + + # # Install poetry dependencies + # - restore_cache: + # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }} + # - run: + # name: Install poetry dependencies + # command: | + # cd C:\Users\circleci\project\desktop + # poetry install + # - save_cache: + # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }} + # paths: + # - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs + # - restore_cache: # key: download-tor-win32-0.4.7.7 # - run: From b3928ea2e4e4c33a997775041be478308cdf3915 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 27 May 2022 07:27:05 -0700 Subject: [PATCH 061/154] Avoid if elif fi --- .circleci/config.yml | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 40cae7f8..13675b65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -114,25 +114,26 @@ jobs: export PATH=/c/msys2/usr/bin:/c/msys2/usr/bin/core_perl:$PATH if [[ -d "/c/Users/circleci/build/openssl" ]]; then echo "openssl already compiled, skipping" - elif - # Get source - gpg --recv-key 8657ABB260F056B1E5190839D9C4D26D0E604491 - cd /c/Users/circleci/src - git clone https://github.com/openssl/openssl.git - cd openssl - git tag -v $OPENSSL_TAG - if [ $? -ne 0 ]; then - echo "openssl tag doesn't verify" - exit -1 - fi - git checkout $OPENSSL_TAG - - # Build - ./Configure --prefix=/opt/openssl-build --openssldir=/usr/local/ssl '-Wl,-rpath,$(LIBRPATH)' Cygwin-x86_64 - make -j$(nproc) - make install - cp -r /opt/openssl-build /c/Users/circleci/build/openssl + exit 0 fi + + # Get source + gpg --recv-key 8657ABB260F056B1E5190839D9C4D26D0E604491 + cd /c/Users/circleci/src + git clone https://github.com/openssl/openssl.git + cd openssl + git tag -v $OPENSSL_TAG + if [ $? -ne 0 ]; then + echo "openssl tag doesn't verify" + exit -1 + fi + git checkout $OPENSSL_TAG + + # Build + ./Configure --prefix=/opt/openssl-build --openssldir=/usr/local/ssl '-Wl,-rpath,$(LIBRPATH)' Cygwin-x86_64 + make -j$(nproc) + make install + cp -r /opt/openssl-build /c/Users/circleci/build/openssl - save_cache: key: "build-win64-openssl-3.0.3" paths: From c7be4b7330ac343b504f8120cc87e6fc4b03afe4 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 27 May 2022 07:37:57 -0700 Subject: [PATCH 062/154] gpg --recv-key just before verifying the tag --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13675b65..ba56f8e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,10 +118,10 @@ jobs: fi # Get source - gpg --recv-key 8657ABB260F056B1E5190839D9C4D26D0E604491 cd /c/Users/circleci/src git clone https://github.com/openssl/openssl.git cd openssl + gpg --recv-key 8657ABB260F056B1E5190839D9C4D26D0E604491 git tag -v $OPENSSL_TAG if [ $? -ne 0 ]; then echo "openssl tag doesn't verify" From 56a4e6e99f6301a8b6024dbb2e93b21c5a2b689e Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 27 May 2022 07:47:10 -0700 Subject: [PATCH 063/154] Import signing keys in a different way --- .circleci/config.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba56f8e2..2899e316 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,13 +97,19 @@ jobs: steps: - checkout - run: - name: Install msys2 and dependencies + name: Install msys2 and dependencies, create folders, import signing keys command: | choco install msys2 --params "/NoUpdate /InstallDir:C:\msys2" refreshenv /c/msys2/usr/bin/pacman -S --needed --noconfirm autoconf automake libtool make clang perl-Pod-Parser mkdir /c/Users/circleci/src mkdir /c/Users/circleci/build + # openssl signing key + curl https://keys.openpgp.org/vks/v1/by-fingerprint/8657ABB260F056B1E5190839D9C4D26D0E604491 | gpg --import + # libevent signing key + curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import + # tor signing key + curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import # Build openssl - restore_cache: @@ -121,7 +127,6 @@ jobs: cd /c/Users/circleci/src git clone https://github.com/openssl/openssl.git cd openssl - gpg --recv-key 8657ABB260F056B1E5190839D9C4D26D0E604491 git tag -v $OPENSSL_TAG if [ $? -ne 0 ]; then echo "openssl tag doesn't verify" From efa2f51d498a55271b37b7221d121cf201052afb Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sat, 28 May 2022 21:40:52 -0700 Subject: [PATCH 064/154] Try building libevent with cygwin --- .circleci/config.yml | 118 ++++++++++++------------------------------- 1 file changed, 33 insertions(+), 85 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2899e316..7e577b8f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,14 +9,10 @@ workflows: jobs: - test-cli # - test-gui - - build-tor-win64: + - build-win64: requires: - test-cli - # - build-win64: - # requires: - # - build-tor-win64 - # - test-cli - # # - test-gui + # - test-gui # - build-win32: # requires: # - test-cli @@ -86,105 +82,67 @@ jobs: cd ~/project/desktop QT_DEBUG_PLUGINS=1 xvfb-run poetry run pytest -v ./tests/test_gui_*.py - build-tor-win64: + build-win64: executor: name: win/default - shell: bash.exe + shell: powershell.exe environment: OPENSSL_TAG: "openssl-3.0.3" LIBEVENT_TAG: "release-2.1.12-stable" TOR_TAG: "tor-0.4.7.7" steps: - checkout - - run: - name: Install msys2 and dependencies, create folders, import signing keys - command: | - choco install msys2 --params "/NoUpdate /InstallDir:C:\msys2" - refreshenv - /c/msys2/usr/bin/pacman -S --needed --noconfirm autoconf automake libtool make clang perl-Pod-Parser - mkdir /c/Users/circleci/src - mkdir /c/Users/circleci/build - # openssl signing key - curl https://keys.openpgp.org/vks/v1/by-fingerprint/8657ABB260F056B1E5190839D9C4D26D0E604491 | gpg --import - # libevent signing key - curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import - # tor signing key - curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import - # Build openssl - - restore_cache: - key: "build-win64-openssl-3.0.3" + # Install cygwin and required packages - run: - name: Build openssl + name: Install cygwin and required packages command: | - export PATH=/c/msys2/usr/bin:/c/msys2/usr/bin/core_perl:$PATH - if [[ -d "/c/Users/circleci/build/openssl" ]]; then + curl -o Downloads\setup-x86_64.exe https://cygwin.com/setup-x86_64.exe + .\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,clang,perl + mkdir C:\Users\circleci\src + mkdir C:\Users\circleci\build + + # Build libevent + - restore_cache: + key: "build-win64-livevent-2.1.12" + - run: + name: Build libevent + shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail + command: | + if [[ -d "/cygdrive/c/Users/circleci/build/openssl" ]]; then echo "openssl already compiled, skipping" exit 0 fi # Get source - cd /c/Users/circleci/src - git clone https://github.com/openssl/openssl.git - cd openssl - git tag -v $OPENSSL_TAG + curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import + cd /cygdrive/c/Users/circleci/src + git clone https://github.com/libevent/libevent.git + cd libevent + git tag -v $LIBEVENT_TAG if [ $? -ne 0 ]; then - echo "openssl tag doesn't verify" + echo "libevent tag doesn't verify" exit -1 fi - git checkout $OPENSSL_TAG + git checkout $LIBEVENT_TAG # Build - ./Configure --prefix=/opt/openssl-build --openssldir=/usr/local/ssl '-Wl,-rpath,$(LIBRPATH)' Cygwin-x86_64 + ./autogen.sh + ./configure --prefix=/opt/libevent-build --with-pic make -j$(nproc) make install - cp -r /opt/openssl-build /c/Users/circleci/build/openssl + cp -r /opt/libevent-build /cygdrive/c/Users/circleci/build/libevent - save_cache: - key: "build-win64-openssl-3.0.3" + key: "build-win64-libevent-2.1.12" paths: - - C:\Users\circleci\build\openssl - - # # Build libevent - # - restore_cache: - # key: "build-win64-livevent-{{ .Environment.LIBEVENT_TAG }}" - # - run: - # name: Build libevent - # command: | - # if (Test-Path -Path "C:\Users\circleci\build\openssl") { - # Write-Output "openssl already compiled, skipping" - # } else { - # # Get source - # gpg --recv-key 9E3AC83A27974B84D1B3401DB86086848EF8686D - # cd C:\Users\circleci\src - # git clone https://github.com/libevent/libevent.git - # cd libevent - # git tag -v $LIBEVENT_TAG - # if($? -ne $true) { throw "libevent tag doesn't verify" } - # git checkout $LIBEVENT_TAG - - # # Build - # $BuildScript = { - # #!/bin/bash - # export PATH=/c/msys2/usr/bin:$PATH - # cd /c/Users/circleci/src/libevent - # ./autogen.sh - # ./configure --prefix=/opt/libevent-build --with-pic - # make -j$(nproc) - # make install - # cp -r /opt/libevent-build /c/Users/circleci/build/libevent - # } - # Set-Content -Path C:\Users\circleci\src\build-libevent.sh -Value $BuildScript.ToString() - # C:\msys2\usr\bin\bash -c "/c/Users/circleci/src/build-libevent.sh" - # } - # - save_cache: - # key: "build-win64-libevent-{{ .Environment.LIBEVENT_TAG }}" - # paths: - # - C:\Users\circleci\build\libevent + - C:\Users\circleci\build\libevent # # Build tor # - run: # name: Build tor # command: | + # # tor signing key + # curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import # $LibeventTag = "release-2.1.12-stable" # $TorTag = "tor-0.4.7.7" @@ -220,16 +178,6 @@ jobs: # if($? -ne $true) { throw "tor tag doesn't verify" } # git checkout $TorTag - build-win64: - executor: - name: win/default - shell: bash.exe - environment: - OPENSSL_TAG: "openssl-3.0.3" - LIBEVENT_TAG: "release-2.1.12-stable" - TOR_TAG: "tor-0.4.7.7" - steps: - - checkout # # Install python # - run: # name: Install Python 3.9.12 From 68b3c1f37138ed04b49772110d12f80aa246b9bb Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sat, 28 May 2022 21:44:26 -0700 Subject: [PATCH 065/154] Fix cygwin download --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e577b8f..3387670b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,8 +97,8 @@ jobs: - run: name: Install cygwin and required packages command: | - curl -o Downloads\setup-x86_64.exe https://cygwin.com/setup-x86_64.exe - .\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,clang,perl + Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,clang,perl mkdir C:\Users\circleci\src mkdir C:\Users\circleci\build From 50b770cbc2c7b96306f2a3a58b2dbc4a67a58af8 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 09:27:10 -0700 Subject: [PATCH 066/154] Verify cygwin signature --- .circleci/config.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3387670b..abe6141c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,8 +97,17 @@ jobs: - run: name: Install cygwin and required packages command: | - Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,clang,perl + # Download cygwin installer, signature, and signing key + Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe -OutFile .\Downloads\setup-x86_64.exe + Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe.sig -OutFile .\Downloads\setup-x86_64.exe.sig + Invoke-WebRequest -Uri https://cygwin.com/key/pubring.asc -OutFile .\Downloads\pubring.asc + # Verify signature + gpg --import .\Downloads\pubring.asc + gpg --verify .\Downloads\setup-x86_64.exe.sig .\Downloads\setup-x86_64.exe + if($? -ne $true) { throw "cygwin failed to verify" } + # Install cygwin and required packaged + .\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,clang,perl + # Create folders mkdir C:\Users\circleci\src mkdir C:\Users\circleci\build From d1a063238d2b6fc48fa5328b3b739c68bef53162 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 09:42:44 -0700 Subject: [PATCH 067/154] Use absolute paths, install gnupg in cygwin, and use cygwin's gpg to verify git tags --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index abe6141c..11f1fd92 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,15 +98,15 @@ jobs: name: Install cygwin and required packages command: | # Download cygwin installer, signature, and signing key - Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe -OutFile .\Downloads\setup-x86_64.exe - Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe.sig -OutFile .\Downloads\setup-x86_64.exe.sig - Invoke-WebRequest -Uri https://cygwin.com/key/pubring.asc -OutFile .\Downloads\pubring.asc + Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe + Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe.sig -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe.sig + Invoke-WebRequest -Uri https://cygwin.com/key/pubring.asc -OutFile C:\Users\circleci\Downloads\pubring.asc # Verify signature - gpg --import .\Downloads\pubring.asc - gpg --verify .\Downloads\setup-x86_64.exe.sig .\Downloads\setup-x86_64.exe + gpg --import C:\Users\circleci\Downloads\pubring.asc + gpg --verify C:\Users\circleci\Downloads\setup-x86_64.exe.sig C:\Users\circleci\Downloads\setup-x86_64.exe if($? -ne $true) { throw "cygwin failed to verify" } # Install cygwin and required packaged - .\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,clang,perl + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg # Create folders mkdir C:\Users\circleci\src mkdir C:\Users\circleci\build From 09b45300f1e100fb25f6422dd8130fbea5f6fdab Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 10:40:36 -0700 Subject: [PATCH 068/154] Skip cygwin install if libevent and tor are compiled --- .circleci/config.yml | 48 ++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11f1fd92..5a2e19e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,39 +87,52 @@ jobs: name: win/default shell: powershell.exe environment: - OPENSSL_TAG: "openssl-3.0.3" + # NOTE: make sure to change when upgrading libevent LIBEVENT_TAG: "release-2.1.12-stable" + # NOTE: make sure to change when upgrading tor TOR_TAG: "tor-0.4.7.7" steps: - checkout + - restore_cache: + # NOTE: make sure to change when upgrading libevent + key: "build-win64-livevent-2.1.12" + - restore_cache: + # NOTE: make sure to change when upgrading tor + key: "build-win64-tor-0.4.7.7" + # Install cygwin and required packages - run: name: Install cygwin and required packages command: | - # Download cygwin installer, signature, and signing key - Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe - Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe.sig -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe.sig - Invoke-WebRequest -Uri https://cygwin.com/key/pubring.asc -OutFile C:\Users\circleci\Downloads\pubring.asc - # Verify signature - gpg --import C:\Users\circleci\Downloads\pubring.asc - gpg --verify C:\Users\circleci\Downloads\setup-x86_64.exe.sig C:\Users\circleci\Downloads\setup-x86_64.exe - if($? -ne $true) { throw "cygwin failed to verify" } - # Install cygwin and required packaged - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg - # Create folders - mkdir C:\Users\circleci\src - mkdir C:\Users\circleci\build + if (((Test-Path C:\Users\circleci\build\libevent -PathType Container) -and (Test-Path C:\Users\circleci\build\tor -PathType Container)) -eq $true) { + Write-Output "libevent and tor already compiled, skipping" + } else { + Write-Output "downloading cygwin installer, signature, and signing key" + Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe + Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe.sig -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe.sig + Invoke-WebRequest -Uri https://cygwin.com/key/pubring.asc -OutFile C:\Users\circleci\Downloads\pubring.asc + + Write-Output "verifying signature" + gpg --import C:\Users\circleci\Downloads\pubring.asc + gpg --verify C:\Users\circleci\Downloads\setup-x86_64.exe.sig C:\Users\circleci\Downloads\setup-x86_64.exe + if($? -ne $true) { throw "cygwin failed to verify" } + + Write-Output "installing cygwin and requited packages" + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg + + Write-Output "creating folders" + mkdir C:\Users\circleci\src + mkdir C:\Users\circleci\build + } # Build libevent - - restore_cache: - key: "build-win64-livevent-2.1.12" - run: name: Build libevent shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail command: | if [[ -d "/cygdrive/c/Users/circleci/build/openssl" ]]; then - echo "openssl already compiled, skipping" + echo "libevent already compiled, skipping" exit 0 fi @@ -142,6 +155,7 @@ jobs: make install cp -r /opt/libevent-build /cygdrive/c/Users/circleci/build/libevent - save_cache: + # NOTE: make sure to change when upgrading libevent key: "build-win64-libevent-2.1.12" paths: - C:\Users\circleci\build\libevent From 744c79db9683834ea704103aa89213ac21dba4ff Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 11:05:28 -0700 Subject: [PATCH 069/154] Try installing cygwin and packages separately --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a2e19e2..7cc805c6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -119,6 +119,7 @@ jobs: if($? -ne $true) { throw "cygwin failed to verify" } Write-Output "installing cygwin and requited packages" + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg Write-Output "creating folders" From 4ea8fd8d23a219f347a778746963f9c0486e1290 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 11:12:52 -0700 Subject: [PATCH 070/154] Try installing cygwin packages separately --- .circleci/config.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7cc805c6..139ebe4d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -119,8 +119,16 @@ jobs: if($? -ne $true) { throw "cygwin failed to verify" } Write-Output "installing cygwin and requited packages" + # C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P git + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P gcc-core + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P make + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P autoconf + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P automake + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libtool + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P gnupg Write-Output "creating folders" mkdir C:\Users\circleci\src From 02339eeda150ff318f795c20a875e65b698f5810 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 11:18:45 -0700 Subject: [PATCH 071/154] Debug output --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 139ebe4d..aaa3174d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -122,13 +122,21 @@ jobs: # C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel + Write-Output "[] done installing libssl-devel" C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P git + Write-Output "[] done installing git" C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P gcc-core + Write-Output "[] done installing gcc-core" C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P make + Write-Output "[] done installing make" C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P autoconf + Write-Output "[] done installing autoconf" C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P automake + Write-Output "[] done installing automake" C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libtool + Write-Output "[] done installing libtool" C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P gnupg + Write-Output "[] done installing gnupg" Write-Output "creating folders" mkdir C:\Users\circleci\src From 144b01d7e479e1d9b76ed312d75c35980070a2d1 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 11:25:40 -0700 Subject: [PATCH 072/154] Try using gpg2 --- .circleci/config.yml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aaa3174d..ea076a29 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -119,24 +119,9 @@ jobs: if($? -ne $true) { throw "cygwin failed to verify" } Write-Output "installing cygwin and requited packages" - # C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel - Write-Output "[] done installing libssl-devel" - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P git - Write-Output "[] done installing git" - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P gcc-core - Write-Output "[] done installing gcc-core" - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P make - Write-Output "[] done installing make" - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P autoconf - Write-Output "[] done installing autoconf" - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P automake - Write-Output "[] done installing automake" - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libtool - Write-Output "[] done installing libtool" - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P gnupg - Write-Output "[] done installing gnupg" + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg2 + C:\cygwin64\bin\bash.exe -c "ln -s /usr/bin/gpg2 /usr/bin/gpg" + C:\cygwin64\bin\bash.exe -c "gpg --list-keys" Write-Output "creating folders" mkdir C:\Users\circleci\src From 2eee62c705d33cd76a029ee40dd834cd3b4079eb Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 11:34:51 -0700 Subject: [PATCH 073/154] Download and install cygwin in separate tasks --- .circleci/config.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea076a29..be434921 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,9 +101,9 @@ jobs: # NOTE: make sure to change when upgrading tor key: "build-win64-tor-0.4.7.7" - # Install cygwin and required packages + # Download cygwin - run: - name: Install cygwin and required packages + name: Download cygwin command: | if (((Test-Path C:\Users\circleci\build\libevent -PathType Container) -and (Test-Path C:\Users\circleci\build\tor -PathType Container)) -eq $true) { Write-Output "libevent and tor already compiled, skipping" @@ -117,15 +117,16 @@ jobs: gpg --import C:\Users\circleci\Downloads\pubring.asc gpg --verify C:\Users\circleci\Downloads\setup-x86_64.exe.sig C:\Users\circleci\Downloads\setup-x86_64.exe if($? -ne $true) { throw "cygwin failed to verify" } + } - Write-Output "installing cygwin and requited packages" - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg2 - C:\cygwin64\bin\bash.exe -c "ln -s /usr/bin/gpg2 /usr/bin/gpg" - C:\cygwin64\bin\bash.exe -c "gpg --list-keys" - - Write-Output "creating folders" - mkdir C:\Users\circleci\src - mkdir C:\Users\circleci\build + # Install cygwin and required packages + - run: + name: Install cygwin and required packages + command: | + if (((Test-Path C:\Users\circleci\build\libevent -PathType Container) -and (Test-Path C:\Users\circleci\build\tor -PathType Container)) -eq $true) { + Write-Output "libevent and tor already compiled, skipping" + } else { + C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg } # Build libevent @@ -138,6 +139,9 @@ jobs: exit 0 fi + mkdir -p /cygdrive/c/Users/circleci/src + mkdir -p /cygdrive/c/Users/circleci/build + # Get source curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import cd /cygdrive/c/Users/circleci/src From c9598a6e6d08d412e9524f461f4989628840348d Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 11:45:34 -0700 Subject: [PATCH 074/154] Switch to bash --- .circleci/config.yml | 45 +++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index be434921..a95bd726 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -104,20 +104,39 @@ jobs: # Download cygwin - run: name: Download cygwin + shell: bash.exe --login -eo pipefail command: | - if (((Test-Path C:\Users\circleci\build\libevent -PathType Container) -and (Test-Path C:\Users\circleci\build\tor -PathType Container)) -eq $true) { - Write-Output "libevent and tor already compiled, skipping" - } else { - Write-Output "downloading cygwin installer, signature, and signing key" - Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe - Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe.sig -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe.sig - Invoke-WebRequest -Uri https://cygwin.com/key/pubring.asc -OutFile C:\Users\circleci\Downloads\pubring.asc + if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then + if [[ -d "/cygdrive/c/Users/circleci/build/tor" ]]; then + echo "libevent and tor already compiled, skipping" + exit 0 + fi + fi - Write-Output "verifying signature" - gpg --import C:\Users\circleci\Downloads\pubring.asc - gpg --verify C:\Users\circleci\Downloads\setup-x86_64.exe.sig C:\Users\circleci\Downloads\setup-x86_64.exe - if($? -ne $true) { throw "cygwin failed to verify" } - } + curl -o setup-x86_64.exe https://cygwin.com/setup-x86_64.exe + curl -o setup-x86_64.exe.sig https://cygwin.com/setup-x86_64.exe.sig + curl -o pubring.asc https://cygwin.com/key/pubring.asc + + gpg --import pubring.asc + gpg --verify setup-x86_64.exe.sig setup-x86_64.exe + if [ $? -ne 0 ]; then + echo "cygwin signature doesn't verify" + exit -1 + fi + + # if (((Test-Path C:\Users\circleci\build\libevent -PathType Container) -and (Test-Path C:\Users\circleci\build\tor -PathType Container)) -eq $true) { + # Write-Output "libevent and tor already compiled, skipping" + # } else { + # Write-Output "downloading cygwin installer, signature, and signing key" + # Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe + # Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe.sig -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe.sig + # Invoke-WebRequest -Uri https://cygwin.com/key/pubring.asc -OutFile C:\Users\circleci\Downloads\pubring.asc + + # Write-Output "verifying signature" + # gpg --import C:\Users\circleci\Downloads\pubring.asc + # gpg --verify C:\Users\circleci\Downloads\setup-x86_64.exe.sig C:\Users\circleci\Downloads\setup-x86_64.exe + # if($? -ne $true) { throw "cygwin failed to verify" } + # } # Install cygwin and required packages - run: @@ -134,7 +153,7 @@ jobs: name: Build libevent shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail command: | - if [[ -d "/cygdrive/c/Users/circleci/build/openssl" ]]; then + if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then echo "libevent already compiled, skipping" exit 0 fi From c8a20bc9c60d0a6e4ee2b204202a5147fd335136 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 11:49:59 -0700 Subject: [PATCH 075/154] Install cygwin in bash too --- .circleci/config.yml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a95bd726..a8eeac9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,9 +101,9 @@ jobs: # NOTE: make sure to change when upgrading tor key: "build-win64-tor-0.4.7.7" - # Download cygwin + # Install cygwin and required packages - run: - name: Download cygwin + name: Install cygwin and required packages shell: bash.exe --login -eo pipefail command: | if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then @@ -124,29 +124,7 @@ jobs: exit -1 fi - # if (((Test-Path C:\Users\circleci\build\libevent -PathType Container) -and (Test-Path C:\Users\circleci\build\tor -PathType Container)) -eq $true) { - # Write-Output "libevent and tor already compiled, skipping" - # } else { - # Write-Output "downloading cygwin installer, signature, and signing key" - # Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe - # Invoke-WebRequest -Uri https://cygwin.com/setup-x86_64.exe.sig -OutFile C:\Users\circleci\Downloads\setup-x86_64.exe.sig - # Invoke-WebRequest -Uri https://cygwin.com/key/pubring.asc -OutFile C:\Users\circleci\Downloads\pubring.asc - - # Write-Output "verifying signature" - # gpg --import C:\Users\circleci\Downloads\pubring.asc - # gpg --verify C:\Users\circleci\Downloads\setup-x86_64.exe.sig C:\Users\circleci\Downloads\setup-x86_64.exe - # if($? -ne $true) { throw "cygwin failed to verify" } - # } - - # Install cygwin and required packages - - run: - name: Install cygwin and required packages - command: | - if (((Test-Path C:\Users\circleci\build\libevent -PathType Container) -and (Test-Path C:\Users\circleci\build\tor -PathType Container)) -eq $true) { - Write-Output "libevent and tor already compiled, skipping" - } else { - C:\Users\circleci\Downloads\setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg - } + ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg # Build libevent - run: From 8693eb7c2f16d84414eb58edeb3b1c6a83c2a751 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 11:55:59 -0700 Subject: [PATCH 076/154] Import signing keys in a seprate task --- .circleci/config.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a8eeac9a..45e5e950 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -126,6 +126,21 @@ jobs: ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg + # Import libevent and tor signing keys + - run: + name: Import libevent and tor signing keys + shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail + command: | + if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then + echo "libevent already compiled, skipping" + exit 0 + fi + + gpg --list-keys + curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import + curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import + gpg --list-keys + # Build libevent - run: name: Build libevent @@ -140,7 +155,6 @@ jobs: mkdir -p /cygdrive/c/Users/circleci/build # Get source - curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import cd /cygdrive/c/Users/circleci/src git clone https://github.com/libevent/libevent.git cd libevent @@ -168,7 +182,7 @@ jobs: # name: Build tor # command: | # # tor signing key - # curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import + # $LibeventTag = "release-2.1.12-stable" # $TorTag = "tor-0.4.7.7" From 75d6f10f30af317b73f5d89c39305670a0999f5c Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 11:57:54 -0700 Subject: [PATCH 077/154] Don't import signing keys in a separate task, and fix paths in the cygwin task --- .circleci/config.yml | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 45e5e950..e9c5466a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -106,8 +106,8 @@ jobs: name: Install cygwin and required packages shell: bash.exe --login -eo pipefail command: | - if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then - if [[ -d "/cygdrive/c/Users/circleci/build/tor" ]]; then + if [[ -d "/c/Users/circleci/build/libevent" ]]; then + if [[ -d "/c/Users/circleci/build/tor" ]]; then echo "libevent and tor already compiled, skipping" exit 0 fi @@ -126,21 +126,6 @@ jobs: ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg - # Import libevent and tor signing keys - - run: - name: Import libevent and tor signing keys - shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail - command: | - if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then - echo "libevent already compiled, skipping" - exit 0 - fi - - gpg --list-keys - curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import - curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import - gpg --list-keys - # Build libevent - run: name: Build libevent @@ -155,6 +140,7 @@ jobs: mkdir -p /cygdrive/c/Users/circleci/build # Get source + curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import cd /cygdrive/c/Users/circleci/src git clone https://github.com/libevent/libevent.git cd libevent @@ -182,7 +168,7 @@ jobs: # name: Build tor # command: | # # tor signing key - + # curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import # $LibeventTag = "release-2.1.12-stable" # $TorTag = "tor-0.4.7.7" From 304f4cf8dc084847aa011bdbb12cee2eec90f41c Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 12:02:15 -0700 Subject: [PATCH 078/154] Try building tor as well --- .circleci/config.yml | 77 ++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9c5466a..b7f2be1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -153,7 +153,9 @@ jobs: # Build ./autogen.sh - ./configure --prefix=/opt/libevent-build --with-pic + ./configure \ + --prefix=/opt/libevent-build \ + --with-pic make -j$(nproc) make install cp -r /opt/libevent-build /cygdrive/c/Users/circleci/build/libevent @@ -163,46 +165,45 @@ jobs: paths: - C:\Users\circleci\build\libevent - # # Build tor - # - run: - # name: Build tor - # command: | - # # tor signing key - # curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import - # $LibeventTag = "release-2.1.12-stable" - # $TorTag = "tor-0.4.7.7" + # Build tor + - run: + name: Build tor + shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail + command: | + if [[ -d "/cygdrive/c/Users/circleci/build/tor" ]]; then + echo "tor already compiled, skipping" + exit 0 + fi - # choco install msys2 --params "/NoUpdate /InstallDir:C:\msys2" - # C:\msys2\usr\bin\bash -c "pacman --sync --noconfirm msys/autoconf" - # C:\msys2\usr\bin\bash -c "pacman --sync --noconfirm msys/automake" + mkdir -p /cygdrive/c/Users/circleci/src + mkdir -p /cygdrive/c/Users/circleci/build - # # Get libevent source - # gpg --recv-key 9E3AC83A27974B84D1B3401DB86086848EF8686D - # cd C:\Users\circleci\src - # git clone https://github.com/libevent/libevent.git - # cd libevent - # git tag -v $LibeventTag - # if($? -ne $true) { throw "libevent tag doesn't verify" } - # git checkout $LibeventTag + # Get source + curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import + cd /cygdrive/c/Users/circleci/src + git clone https://git.torproject.org/tor.git + cd tor + git tag -v $TOR_TAG + if [ $? -ne 0 ]; then + echo "tor tag doesn't verify" + exit -1 + fi + git checkout $TOR_TAG - # # Build libevent - # $BashScript = @" - # #!/bin/bash - # export PATH=/c/msys2/usr/bin:$PATH - # cd /c/Users/circleci/src/libevent - # ./autogen.sh - # ./configure --prefix=/opt/libevent-build --with-pic - # make -j8 install - # "@ - - # # Get tor source - # gpg --recv-key B74417EDDF22AC9F9E90F49142E86A2A11F48D36 - # cd C:\Users\circleci\src - # git clone https://git.torproject.org/tor.git - # cd tor - # git tag -v $TorTag - # if($? -ne $true) { throw "tor tag doesn't verify" } - # git checkout $TorTag + # Build + ./configure \ + --prefix=/opt/tor-build \ + --enable-static-libevent \ + --with-libevent-dir=/cygdrive/c/Users/circleci/build/libevent \ + --disable-asciidoc + make -j$(nproc) + make install + cp -r /opt/tor-build /cygdrive/c/Users/circleci/build/tor + - save_cache: + # NOTE: make sure to change when upgrading tor + key: "build-win64-tor-0.4.7.7" + paths: + - C:\Users\circleci\build\tor # # Install python # - run: From 7fd623446ff92a059ba74b1fc9c71f51ff5c28bb Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 12:13:20 -0700 Subject: [PATCH 079/154] Fix cache key name, and run ./autogen.sh when compiling tor --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b7f2be1f..ada5b5a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,7 +96,7 @@ jobs: - restore_cache: # NOTE: make sure to change when upgrading libevent - key: "build-win64-livevent-2.1.12" + key: "build-win64-libevent-2.1.12" - restore_cache: # NOTE: make sure to change when upgrading tor key: "build-win64-tor-0.4.7.7" @@ -191,6 +191,7 @@ jobs: git checkout $TOR_TAG # Build + ./autogen.sh ./configure \ --prefix=/opt/tor-build \ --enable-static-libevent \ From 9fc8ee51781b266d8ce315d35e0fe4466b1947b7 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 12:45:58 -0700 Subject: [PATCH 080/154] Add a cache version, to invalidate old circleci cache --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ada5b5a0..5f52cc5b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,14 +34,14 @@ jobs: sudo apt-get update sudo apt-get -y install tor obfs4proxy - restore_cache: - key: test-cli-poetry-deps-{{ checksum "~/project/cli/poetry.lock" }} + key: test-cli-poetry-deps-{{ .Environment.CACHE_VERSION }}-{{ checksum "~/project/cli/poetry.lock" }} - run: name: Install poetry dependencies command: | cd ~/project/cli poetry install - save_cache: - key: test-cli-poetry-deps-{{ checksum "~/project/cli/poetry.lock" }} + key: test-cli-poetry-deps-{{ .Environment.CACHE_VERSION }}-{{ checksum "~/project/cli/poetry.lock" }} paths: - /home/circleci/.cache/pypoetry/virtualenvs - run: @@ -66,14 +66,14 @@ jobs: sudo apt-get install -y tor obfs4proxy gcc python3-dev python3-pyside2.qtcore python3-pyside2.qtwidgets python3-pyside2.qtgui sudo apt-get install -y xvfb x11-utils libxkbcommon-x11-0 libxcb-randr0-dev libxcb-xtest0-dev libxcb-xinerama0-dev libxcb-shape0-dev libxcb-xkb-dev libxcb-render-util0 libxcb-icccm4 libxcb-keysyms1 libxcb-image0 - restore_cache: - key: test-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }} + key: test-desktop-poetry-deps-{{ .Environment.CACHE_VERSION }}-{{ checksum "~/project/desktop/poetry.lock" }} - run: name: Install poetry dependencies command: | cd ~/project/desktop poetry install - save_cache: - key: test-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }} + key: test-desktop-poetry-deps-{{ .Environment.CACHE_VERSION }}-{{ checksum "~/project/desktop/poetry.lock" }} paths: - /home/circleci/.cache/pypoetry/virtualenvs - run: @@ -96,10 +96,10 @@ jobs: - restore_cache: # NOTE: make sure to change when upgrading libevent - key: "build-win64-libevent-2.1.12" + key: "build-win64-libevent-2.1.12-{{ .Environment.CACHE_VERSION }}" - restore_cache: # NOTE: make sure to change when upgrading tor - key: "build-win64-tor-0.4.7.7" + key: "build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }}" # Install cygwin and required packages - run: @@ -161,7 +161,7 @@ jobs: cp -r /opt/libevent-build /cygdrive/c/Users/circleci/build/libevent - save_cache: # NOTE: make sure to change when upgrading libevent - key: "build-win64-libevent-2.1.12" + key: "build-win64-libevent-2.1.12-{{ .Environment.CACHE_VERSION }}" paths: - C:\Users\circleci\build\libevent @@ -202,7 +202,7 @@ jobs: cp -r /opt/tor-build /cygdrive/c/Users/circleci/build/tor - save_cache: # NOTE: make sure to change when upgrading tor - key: "build-win64-tor-0.4.7.7" + key: "build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }}" paths: - C:\Users\circleci\build\tor From 279951d5efebe0e0f8aaa3c583ab5b2d8ac2bccb Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 12:49:09 -0700 Subject: [PATCH 081/154] Build libevent and tor in the same task --- .circleci/config.yml | 53 ++++++++++++-------------------------------- 1 file changed, 14 insertions(+), 39 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f52cc5b..7c2e4e31 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,9 +94,6 @@ jobs: steps: - checkout - - restore_cache: - # NOTE: make sure to change when upgrading libevent - key: "build-win64-libevent-2.1.12-{{ .Environment.CACHE_VERSION }}" - restore_cache: # NOTE: make sure to change when upgrading tor key: "build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }}" @@ -106,11 +103,9 @@ jobs: name: Install cygwin and required packages shell: bash.exe --login -eo pipefail command: | - if [[ -d "/c/Users/circleci/build/libevent" ]]; then - if [[ -d "/c/Users/circleci/build/tor" ]]; then - echo "libevent and tor already compiled, skipping" - exit 0 - fi + if [[ -d "/c/Users/circleci/build/tor" ]]; then + echo "tor already compiled, skipping" + exit 0 fi curl -o setup-x86_64.exe https://cygwin.com/setup-x86_64.exe @@ -126,20 +121,20 @@ jobs: ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg - # Build libevent + # Build libevent tor - run: - name: Build libevent + name: Build libevent and tor shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail command: | - if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then - echo "libevent already compiled, skipping" + if [[ -d "/cygdrive/c/Users/circleci/build/tor" ]]; then + echo "tor already compiled, skipping" exit 0 fi mkdir -p /cygdrive/c/Users/circleci/src - mkdir -p /cygdrive/c/Users/circleci/build + mkdir -p /cygdrive/c/Users/circleci/build/tor - # Get source + # Get libevent source curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import cd /cygdrive/c/Users/circleci/src git clone https://github.com/libevent/libevent.git @@ -151,34 +146,15 @@ jobs: fi git checkout $LIBEVENT_TAG - # Build + # Build libevent ./autogen.sh ./configure \ --prefix=/opt/libevent-build \ --with-pic make -j$(nproc) make install - cp -r /opt/libevent-build /cygdrive/c/Users/circleci/build/libevent - - save_cache: - # NOTE: make sure to change when upgrading libevent - key: "build-win64-libevent-2.1.12-{{ .Environment.CACHE_VERSION }}" - paths: - - C:\Users\circleci\build\libevent - # Build tor - - run: - name: Build tor - shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail - command: | - if [[ -d "/cygdrive/c/Users/circleci/build/tor" ]]; then - echo "tor already compiled, skipping" - exit 0 - fi - - mkdir -p /cygdrive/c/Users/circleci/src - mkdir -p /cygdrive/c/Users/circleci/build - - # Get source + # Get tor source curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import cd /cygdrive/c/Users/circleci/src git clone https://git.torproject.org/tor.git @@ -190,16 +166,15 @@ jobs: fi git checkout $TOR_TAG - # Build + # Build tor ./autogen.sh ./configure \ - --prefix=/opt/tor-build \ + --prefix=/cygdrive/c/Users/circleci/build/tor \ --enable-static-libevent \ - --with-libevent-dir=/cygdrive/c/Users/circleci/build/libevent \ + --with-libevent-dir=/opt/libevent-build \ --disable-asciidoc make -j$(nproc) make install - cp -r /opt/tor-build /cygdrive/c/Users/circleci/build/tor - save_cache: # NOTE: make sure to change when upgrading tor key: "build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }}" From 1e9135f0c9145ff5b7313978eb7a7d29a37b9484 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 13:04:25 -0700 Subject: [PATCH 082/154] Try installing libevent-devel instead of compiling libevent --- .circleci/config.yml | 87 +++++++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 34 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c2e4e31..7d6b5a37 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,14 +66,14 @@ jobs: sudo apt-get install -y tor obfs4proxy gcc python3-dev python3-pyside2.qtcore python3-pyside2.qtwidgets python3-pyside2.qtgui sudo apt-get install -y xvfb x11-utils libxkbcommon-x11-0 libxcb-randr0-dev libxcb-xtest0-dev libxcb-xinerama0-dev libxcb-shape0-dev libxcb-xkb-dev libxcb-render-util0 libxcb-icccm4 libxcb-keysyms1 libxcb-image0 - restore_cache: - key: test-desktop-poetry-deps-{{ .Environment.CACHE_VERSION }}-{{ checksum "~/project/desktop/poetry.lock" }} + key: test-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - run: name: Install poetry dependencies command: | cd ~/project/desktop poetry install - save_cache: - key: test-desktop-poetry-deps-{{ .Environment.CACHE_VERSION }}-{{ checksum "~/project/desktop/poetry.lock" }} + key: test-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} paths: - /home/circleci/.cache/pypoetry/virtualenvs - run: @@ -100,11 +100,11 @@ jobs: # Install cygwin and required packages - run: - name: Install cygwin and required packages + name: Install cygwin and required packages (for building tor) shell: bash.exe --login -eo pipefail command: | if [[ -d "/c/Users/circleci/build/tor" ]]; then - echo "tor already compiled, skipping" + echo "libevent and tor already compiled, skipping" exit 0 fi @@ -119,11 +119,50 @@ jobs: exit -1 fi - ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg + ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,libevent-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg - # Build libevent tor + # # Build libevent + # - run: + # name: Build libevent + # shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail + # command: | + # if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then + # echo "libevent already compiled, skipping" + # exit 0 + # fi + + # mkdir -p /cygdrive/c/Users/circleci/src + # mkdir -p /cygdrive/c/Users/circleci/build + + # # Get source + # curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import + # cd /cygdrive/c/Users/circleci/src + # git clone https://github.com/libevent/libevent.git + # cd libevent + # git tag -v $LIBEVENT_TAG + # if [ $? -ne 0 ]; then + # echo "libevent tag doesn't verify" + # exit -1 + # fi + # git checkout $LIBEVENT_TAG + + # # Build + # ./autogen.sh + # ./configure \ + # --prefix=/opt/libevent-build \ + # --with-pic + # make -j$(nproc) + # make install + # cp -r /opt/libevent-build /cygdrive/c/Users/circleci/build/libevent + # - save_cache: + # # NOTE: make sure to change when upgrading libevent + # key: "build-win64-libevent-2.1.12-{{ .Environment.CACHE_VERSION }}" + # paths: + # - C:\Users\circleci\build\libevent + + # Build tor - run: - name: Build libevent and tor + name: Build tor shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail command: | if [[ -d "/cygdrive/c/Users/circleci/build/tor" ]]; then @@ -132,29 +171,9 @@ jobs: fi mkdir -p /cygdrive/c/Users/circleci/src - mkdir -p /cygdrive/c/Users/circleci/build/tor + mkdir -p /cygdrive/c/Users/circleci/build - # Get libevent source - curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import - cd /cygdrive/c/Users/circleci/src - git clone https://github.com/libevent/libevent.git - cd libevent - git tag -v $LIBEVENT_TAG - if [ $? -ne 0 ]; then - echo "libevent tag doesn't verify" - exit -1 - fi - git checkout $LIBEVENT_TAG - - # Build libevent - ./autogen.sh - ./configure \ - --prefix=/opt/libevent-build \ - --with-pic - make -j$(nproc) - make install - - # Get tor source + # Get source curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import cd /cygdrive/c/Users/circleci/src git clone https://git.torproject.org/tor.git @@ -166,15 +185,15 @@ jobs: fi git checkout $TOR_TAG - # Build tor + # Build ./autogen.sh ./configure \ - --prefix=/cygdrive/c/Users/circleci/build/tor \ + --prefix=/opt/tor-build \ --enable-static-libevent \ - --with-libevent-dir=/opt/libevent-build \ --disable-asciidoc make -j$(nproc) make install + cp -r /opt/tor-build /cygdrive/c/Users/circleci/build/tor - save_cache: # NOTE: make sure to change when upgrading tor key: "build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }}" @@ -194,14 +213,14 @@ jobs: # # Install poetry dependencies # - restore_cache: - # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }} + # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} # - run: # name: Install poetry dependencies # command: | # cd C:\Users\circleci\project\desktop # poetry install # - save_cache: - # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }} + # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} # paths: # - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs From 3d827805d2862f859ca869e680498fef345ead80 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 13:12:14 -0700 Subject: [PATCH 083/154] Do not enable static libevent (hopefully this tor binary will work on other systems) --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7d6b5a37..abbfbc8e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -189,7 +189,6 @@ jobs: ./autogen.sh ./configure \ --prefix=/opt/tor-build \ - --enable-static-libevent \ --disable-asciidoc make -j$(nproc) make install From 8e8d772b47a8acb3279b14859a9687b42218e040 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 13:47:42 -0700 Subject: [PATCH 084/154] Add zlib-devel dependency --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index abbfbc8e..f97648be 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -119,7 +119,8 @@ jobs: exit -1 fi - ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ -P libssl-devel,libevent-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg + ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ \ + -P libssl-devel,libevent-devel,zlib-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg # # Build libevent # - run: From 87b8cb9c87632d7ce323c8ffd987098e8cddb02e Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 14:17:16 -0700 Subject: [PATCH 085/154] Add compiling obfs4proxy, snowflake, and meek --- .circleci/config.yml | 124 +++++++++++++++++++++++-------------------- 1 file changed, 67 insertions(+), 57 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f97648be..8ecd2843 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -200,6 +200,72 @@ jobs: paths: - C:\Users\circleci\build\tor + - run: + name: Copy tor binaries into project + shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail + # TODO: update this to actually copy the files + command: find /cygdrive/c/Users/circleci/build/tor + + - restore_cache: + # NOTE: make sure to change when uprading obfs4proxy + key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }}" + - run: + name: Build obfs4proxy + command: | + if (Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe -PathType -leaf -eq $true) { + Write-Output "obfs4proxy already built" + } else { + # curl https://keys.openpgp.org/vks/v1/by-fingerprint/9EB1A490C73CC5D44DFB3E47BFBD1C7B8A6EC81A | gpg --import + mkdir C:\Users\circleci\src\obfs4proxy + cd C:\Users\circleci\src\obfs4proxy + git clone https://gitlab.com/yawning/obfs4 + cd obfs4 + git checkout obfs4proxy-0.0.13 + go build .\obfs4proxy + Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + } + - save_cache: + # NOTE: make sure to change when uprading obfs4proxy + key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }}" + paths: + - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + + - restore_cache: + # NOTE: make sure to change when uprading snowflake + key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }}" + - run: + name: Build snowflake-client + command: | + if (Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe -PathType -leaf -eq $true) { + Write-Output "snowflake already built" + } else { + go install git.torproject.org/pluggable-transports/snowflake.git/client@v2.1.0 + Move-Item -Path C:\Users\circleci\go\bin\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + } + - save_cache: + # NOTE: make sure to change when uprading snowflake + key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }}" + paths: + - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + + - restore_cache: + # NOTE: make sure to change when uprading meek + key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }}" + - run: + name: Build meek-client + command: | + if (Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe -PathType -leaf -eq $true) { + Write-Output "snowflake already built" + } else { + go install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 + Move-Item -Path C:\Users\circleci\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe + } + - save_cache: + # NOTE: make sure to change when uprading meek + key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }}" + paths: + - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe + # # Install python # - run: # name: Install Python 3.9.12 @@ -224,66 +290,10 @@ jobs: # paths: # - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs - # - restore_cache: - # key: download-tor-win32-0.4.7.7 - # - run: - # name: Download tor (Windows Expert Bundle) - # command: | - # mkdir ~\Downloads\tor - # cd ~\Downloads\tor - # $URL = "https://www.torproject.org/dist/torbrowser/11.0.13/tor-win32-0.4.7.7.zip" - # $Filename = "tor-win32-0.4.7.7.zip" - # $ExpectedHash = "6af0088460d61a5e60971aea94562a3bed2fb597da6b8ac77948a5f6424dfa29" - # Invoke-WebRequest -Uri $URL -OutFile $Filename - # $FileHash = Get-FileHash $Filename - # if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } - # mkdir ~\Downloads\tor\tor-win32 - # Expand-Archive -LiteralPath tor-win32-0.4.7.7.zip -DestinationPath ~\Downloads\tor\tor-win32 - # - save_cache: - # key: download-tor-win32-0.4.7.7 - # paths: - # - ~\Downloads\tor\tor-win32 # - run: # name: Copy tor binary into app # command: cp ~\Downloads\tor\tor-win32\* ~\project\desktop\onionshare\resources\tor - # - restore_cache: - # key: build-win64-obfs4proxy-v0.0.13 - # - run: - # name: Build obfs4proxy - # command: | - # mkdir ~\Downloads\obfs4proxy - # cd Downloads\obfs4proxy - # git clone https://gitlab.com/yawning/obfs4 - # cd obfs4 - # git checkout obfs4proxy-0.0.13 - # go build .\obfs4proxy - # Move-Item -Path .\obfs4proxy.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe - # - save_cache: - # key: build-win64-obfs4proxy-v0.0.13 - # paths: - # - ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe - # - restore_cache: - # key: build-win64-snowflake-v2.1.0 - # - run: - # name: Build snowflake-client - # command: | - # go install git.torproject.org/pluggable-transports/snowflake.git/client@v2.1.0 - # Move-Item -Path ~\go\bin\client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe - # - save_cache: - # key: build-win64-snowflake-v2.1.0 - # paths: - # - ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe - # - restore_cache: - # key: build-win64-meek-v0.37.0 - # - run: - # name: Build meek-client - # command: | - # go install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 - # Move-Item -Path ~\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - # - save_cache: - # key: build-win64-meek-v0.37.0 - # paths: - # - ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe + # - run: # name: Build OnionShare # command: | From 7690ccbff4413d7deec1ad1732cfc3dcc7c53b9f Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 14:19:47 -0700 Subject: [PATCH 086/154] Always install cygwin --- .circleci/config.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ecd2843..997a62c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,11 +103,6 @@ jobs: name: Install cygwin and required packages (for building tor) shell: bash.exe --login -eo pipefail command: | - if [[ -d "/c/Users/circleci/build/tor" ]]; then - echo "libevent and tor already compiled, skipping" - exit 0 - fi - curl -o setup-x86_64.exe https://cygwin.com/setup-x86_64.exe curl -o setup-x86_64.exe.sig https://cygwin.com/setup-x86_64.exe.sig curl -o pubring.asc https://cygwin.com/key/pubring.asc From d8af8a39be62debbf2445706d8904eb16a1e0b23 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 14:28:30 -0700 Subject: [PATCH 087/154] Copy tor binaries into the project --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 997a62c0..4a981253 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -197,9 +197,10 @@ jobs: - run: name: Copy tor binaries into project - shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail - # TODO: update this to actually copy the files - command: find /cygdrive/c/Users/circleci/build/tor + command: | + Copy-Item -Path C:\Useres\circleci\build\tor\bin\tor.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ + Copy-Item -Path C:\Useres\circleci\build\tor\share\tor\geoip -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ + Copy-Item -Path C:\Useres\circleci\build\tor\share\tor\geoip6 -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ - restore_cache: # NOTE: make sure to change when uprading obfs4proxy From 50ed30a84f4cf3f5429a2a0616762e6e243963f0 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 14:30:23 -0700 Subject: [PATCH 088/154] Fix Test-Path if statements --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a981253..13d1c97b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -208,7 +208,7 @@ jobs: - run: name: Build obfs4proxy command: | - if (Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe -PathType -leaf -eq $true) { + if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe) -eq $True) { Write-Output "obfs4proxy already built" } else { # curl https://keys.openpgp.org/vks/v1/by-fingerprint/9EB1A490C73CC5D44DFB3E47BFBD1C7B8A6EC81A | gpg --import @@ -232,7 +232,7 @@ jobs: - run: name: Build snowflake-client command: | - if (Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe -PathType -leaf -eq $true) { + if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe) -eq $True) { Write-Output "snowflake already built" } else { go install git.torproject.org/pluggable-transports/snowflake.git/client@v2.1.0 @@ -250,7 +250,7 @@ jobs: - run: name: Build meek-client command: | - if (Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe -PathType -leaf -eq $true) { + if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe) -eq $True) { Write-Output "snowflake already built" } else { go install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 From 35ff38780973973dd3849c78fef62f5c9d942160 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 14:41:23 -0700 Subject: [PATCH 089/154] Fix copying tor binaries into project --- .circleci/config.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13d1c97b..988cb328 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,9 +98,8 @@ jobs: # NOTE: make sure to change when upgrading tor key: "build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }}" - # Install cygwin and required packages - run: - name: Install cygwin and required packages (for building tor) + name: Install cygwin and required packages shell: bash.exe --login -eo pipefail command: | curl -o setup-x86_64.exe https://cygwin.com/setup-x86_64.exe @@ -156,7 +155,6 @@ jobs: # paths: # - C:\Users\circleci\build\libevent - # Build tor - run: name: Build tor shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail @@ -198,9 +196,10 @@ jobs: - run: name: Copy tor binaries into project command: | - Copy-Item -Path C:\Useres\circleci\build\tor\bin\tor.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ - Copy-Item -Path C:\Useres\circleci\build\tor\share\tor\geoip -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ - Copy-Item -Path C:\Useres\circleci\build\tor\share\tor\geoip6 -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ + New-Item -ItemType Directory -Force -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ + Copy-Item -Path C:\Users\circleci\build\tor\bin\tor.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ + Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ + Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip6 -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ - restore_cache: # NOTE: make sure to change when uprading obfs4proxy From 092715585b4545201cab6252d9349821264b4c6c Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 14:58:25 -0700 Subject: [PATCH 090/154] Build snowflake and meek by git cloning their source first --- .circleci/config.yml | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 988cb328..a730e6bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -196,10 +196,10 @@ jobs: - run: name: Copy tor binaries into project command: | - New-Item -ItemType Directory -Force -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ - Copy-Item -Path C:\Users\circleci\build\tor\bin\tor.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ - Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ - Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip6 -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\ + New-Item -ItemType Directory -Force -Path C:\Users\circleci\project\desktop\onionshare\resources\tor + Copy-Item -Path C:\Users\circleci\build\tor\bin\tor.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor + Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor + Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip6 -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor - restore_cache: # NOTE: make sure to change when uprading obfs4proxy @@ -211,19 +211,20 @@ jobs: Write-Output "obfs4proxy already built" } else { # curl https://keys.openpgp.org/vks/v1/by-fingerprint/9EB1A490C73CC5D44DFB3E47BFBD1C7B8A6EC81A | gpg --import - mkdir C:\Users\circleci\src\obfs4proxy + New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\obfs4proxy cd C:\Users\circleci\src\obfs4proxy git clone https://gitlab.com/yawning/obfs4 cd obfs4 + # TODO: verify signature git checkout obfs4proxy-0.0.13 go build .\obfs4proxy - Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe } - save_cache: # NOTE: make sure to change when uprading obfs4proxy key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }}" paths: - - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe + - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe - restore_cache: # NOTE: make sure to change when uprading snowflake @@ -231,17 +232,23 @@ jobs: - run: name: Build snowflake-client command: | - if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe) -eq $True) { + if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe) -eq $True) { Write-Output "snowflake already built" } else { - go install git.torproject.org/pluggable-transports/snowflake.git/client@v2.1.0 - Move-Item -Path C:\Users\circleci\go\bin\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\obfs4proxy + cd C:\Users\circleci\src\obfs4proxy + git clone https://git.torproject.org/pluggable-transports/snowflake.git + cd snowflake + # TODO: verify signature + git checkout v2.2.0 + go build .\client + Move-Item -Path .\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe } - save_cache: # NOTE: make sure to change when uprading snowflake key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }}" paths: - - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe + - C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe - restore_cache: # NOTE: make sure to change when uprading meek @@ -252,8 +259,14 @@ jobs: if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe) -eq $True) { Write-Output "snowflake already built" } else { - go install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 - Move-Item -Path C:\Users\circleci\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe + New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\meek + cd C:\Users\circleci\src\meek + git clone https://git.torproject.org/pluggable-transports/meek.git + cd meek + # TODO: verify signature + git checkout v0.37.0 + go build .\meek-client + Move-Item -Path .\meek-client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe } - save_cache: # NOTE: make sure to change when uprading meek From ac9bc039bb9415675ddf037dfbbe1be142dd862c Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 15:23:55 -0700 Subject: [PATCH 091/154] Fix get_tor_paths() to point to new Windows paths, and actually build onionshare for win64 --- .circleci/config.yml | 81 +++++++++++++++++------------------- cli/onionshare_cli/common.py | 14 +++---- 2 files changed, 45 insertions(+), 50 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a730e6bd..cf1725c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -202,7 +202,7 @@ jobs: Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip6 -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor - restore_cache: - # NOTE: make sure to change when uprading obfs4proxy + # NOTE: make sure to change when upgrading obfs4proxy key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }}" - run: name: Build obfs4proxy @@ -221,13 +221,13 @@ jobs: Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe } - save_cache: - # NOTE: make sure to change when uprading obfs4proxy + # NOTE: make sure to change when upgrading obfs4proxy key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }}" paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe - restore_cache: - # NOTE: make sure to change when uprading snowflake + # NOTE: make sure to change when upgrading snowflake key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }}" - run: name: Build snowflake-client @@ -245,13 +245,13 @@ jobs: Move-Item -Path .\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe } - save_cache: - # NOTE: make sure to change when uprading snowflake + # NOTE: make sure to change when upgrading snowflake key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }}" paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe - restore_cache: - # NOTE: make sure to change when uprading meek + # NOTE: make sure to change when upgrading meek key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }}" - run: name: Build meek-client @@ -269,52 +269,47 @@ jobs: Move-Item -Path .\meek-client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe } - save_cache: - # NOTE: make sure to change when uprading meek + # NOTE: make sure to change when upgrading meek key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }}" paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - # # Install python - # - run: - # name: Install Python 3.9.12 - # command: | - # choco install python3 --version=3.9.12 + - run: + name: Install Python 3.9.12 + command: | + choco install python3 --version=3.9.12 - # # Install poetry - # - run: - # name: Install poetry - # command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - + - run: + name: Install poetry + command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - - # # Install poetry dependencies - # - restore_cache: - # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - # - run: - # name: Install poetry dependencies - # command: | - # cd C:\Users\circleci\project\desktop - # poetry install - # - save_cache: - # key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - # paths: - # - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs + - restore_cache: + key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Install poetry dependencies + command: | + cd C:\Users\circleci\project\desktop + poetry install + - save_cache: + key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs - # - run: - # name: Copy tor binary into app - # command: cp ~\Downloads\tor\tor-win32\* ~\project\desktop\onionshare\resources\tor + - run: + name: Build OnionShare + command: | + cd ~\project\desktop + poetry run python .\setup-freeze.py build + poetry run python .\scripts\build-windows.py cleanup-build - # - run: - # name: Build OnionShare - # command: | - # cd ~\project\desktop - # poetry run python .\setup-freeze.py build - # poetry run python .\scripts\build-windows.py cleanup-build - # - run: - # name: Compress - # command: | - # mv ~\project\desktop\build\exe.win-amd64-3.9\ ~\onionshare-win64 - # Compress-Archive -LiteralPath ~\onionshare-win64 -DestinationPath ~\onionshare-win64.zip - # - store_artifacts: - # path: ~\onionshare-win64.zip + - run: + name: Compress + command: | + mv ~\project\desktop\build\exe.win-amd64-3.9\ ~\onionshare-win64 + Compress-Archive -LiteralPath ~\onionshare-win64 -DestinationPath ~\onionshare-win64.zip + + - store_artifacts: + path: ~\onionshare-win64.zip build-win32: executor: diff --git a/cli/onionshare_cli/common.py b/cli/onionshare_cli/common.py index ceec654d..da5204cf 100644 --- a/cli/onionshare_cli/common.py +++ b/cli/onionshare_cli/common.py @@ -332,21 +332,21 @@ class Common: # In Windows, the Tor binaries are in the onionshare package, not the onionshare_cli package base_path = self.get_resource_path("tor") base_path = base_path.replace("onionshare_cli", "onionshare") - tor_path = os.path.join(base_path, "Tor", "tor.exe") + tor_path = os.path.join(base_path, "tor", "tor.exe") # If tor.exe isn't there, mayber we're running from the source tree if not os.path.exists(tor_path): base_path = os.path.join(os.getcwd(), "onionshare", "resources", "tor") - tor_path = os.path.join(base_path, "Tor", "tor.exe") + tor_path = os.path.join(base_path, "tor.exe") if not os.path.exists(tor_path): raise CannotFindTor() - obfs4proxy_file_path = os.path.join(base_path, "Tor", "obfs4proxy.exe") - snowflake_file_path = os.path.join(base_path, "Tor", "snowflake-client.exe") - meek_client_file_path = os.path.join(base_path, "Tor", "meek-client.exe") - tor_geo_ip_file_path = os.path.join(base_path, "Data", "Tor", "geoip") - tor_geo_ipv6_file_path = os.path.join(base_path, "Data", "Tor", "geoip6") + obfs4proxy_file_path = os.path.join(base_path, "tor", "obfs4proxy.exe") + snowflake_file_path = os.path.join(base_path, "tor", "snowflake-client.exe") + meek_client_file_path = os.path.join(base_path, "tor", "meek-client.exe") + tor_geo_ip_file_path = os.path.join(base_path, "tor", "geoip") + tor_geo_ipv6_file_path = os.path.join(base_path, "tor", "geoip6") elif self.platform == "Darwin": # Let's see if we have tor binaries in the onionshare GUI package From f9e2bc215d573688506730fc09907e8e3e6f3cd0 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 15:33:30 -0700 Subject: [PATCH 092/154] Fix skipping tasks for restoring obfs4proxy and meek-client --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf1725c0..cb459dac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -207,7 +207,7 @@ jobs: - run: name: Build obfs4proxy command: | - if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe) -eq $True) { + if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe) -eq $True) { Write-Output "obfs4proxy already built" } else { # curl https://keys.openpgp.org/vks/v1/by-fingerprint/9EB1A490C73CC5D44DFB3E47BFBD1C7B8A6EC81A | gpg --import @@ -256,7 +256,7 @@ jobs: - run: name: Build meek-client command: | - if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe) -eq $True) { + if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe) -eq $True) { Write-Output "snowflake already built" } else { New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\meek From c23621d841e5365d9f98163ca635ed02abdfaa89 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 15:43:25 -0700 Subject: [PATCH 093/154] Fix cache keys --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb459dac..ced2b00b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,7 +96,7 @@ jobs: - restore_cache: # NOTE: make sure to change when upgrading tor - key: "build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }}" + key: build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - run: name: Install cygwin and required packages @@ -189,7 +189,7 @@ jobs: cp -r /opt/tor-build /cygdrive/c/Users/circleci/build/tor - save_cache: # NOTE: make sure to change when upgrading tor - key: "build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }}" + key: build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\build\tor @@ -203,7 +203,7 @@ jobs: - restore_cache: # NOTE: make sure to change when upgrading obfs4proxy - key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }}" + key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} - run: name: Build obfs4proxy command: | @@ -222,13 +222,13 @@ jobs: } - save_cache: # NOTE: make sure to change when upgrading obfs4proxy - key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }}" + key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe - restore_cache: # NOTE: make sure to change when upgrading snowflake - key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }}" + key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} - run: name: Build snowflake-client command: | @@ -246,13 +246,13 @@ jobs: } - save_cache: # NOTE: make sure to change when upgrading snowflake - key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }}" + key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe - restore_cache: # NOTE: make sure to change when upgrading meek - key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }}" + key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} - run: name: Build meek-client command: | @@ -270,7 +270,7 @@ jobs: } - save_cache: # NOTE: make sure to change when upgrading meek - key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }}" + key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe From 2490f45343d62048ca68a9ad6ef1048890cdb905 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 15:59:19 -0700 Subject: [PATCH 094/154] Build both win64 and win32, and update get_tor_paths() to display verbose output about the path its looking for --- .circleci/config.yml | 253 +++++++++++++++++++++++++---------- cli/onionshare_cli/common.py | 9 +- 2 files changed, 186 insertions(+), 76 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ced2b00b..a59ae14f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,10 +13,10 @@ workflows: requires: - test-cli # - test-gui - # - build-win32: - # requires: - # - test-cli - # # - test-gui + - build-win32: + requires: + - test-cli + # - test-gui # - build-macos: # requires: # - test-cli @@ -87,17 +87,17 @@ jobs: name: win/default shell: powershell.exe environment: - # NOTE: make sure to change when upgrading libevent - LIBEVENT_TAG: "release-2.1.12-stable" # NOTE: make sure to change when upgrading tor - TOR_TAG: "tor-0.4.7.7" + TOR_TAG: tor-0.4.7.7 + # NOTE: make sure to change when upgrading obfs4proxy + OBFS4PROXY_TAG: obfs4proxy-0.0.13 + # NOTE: make sure to change when upgrading snowflake + SNOWFLAKE_TAG: v2.2.0 + # NOTE: make sure to change when upgrading meek + MEEK_TAG: v0.37.0 steps: - checkout - - restore_cache: - # NOTE: make sure to change when upgrading tor - key: build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - - run: name: Install cygwin and required packages shell: bash.exe --login -eo pipefail @@ -155,6 +155,9 @@ jobs: # paths: # - C:\Users\circleci\build\libevent + - restore_cache: + # NOTE: make sure to change when upgrading tor + key: build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - run: name: Build tor shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail @@ -216,7 +219,7 @@ jobs: git clone https://gitlab.com/yawning/obfs4 cd obfs4 # TODO: verify signature - git checkout obfs4proxy-0.0.13 + git checkout $OBFS4PROXY_TAG go build .\obfs4proxy Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe } @@ -240,7 +243,7 @@ jobs: git clone https://git.torproject.org/pluggable-transports/snowflake.git cd snowflake # TODO: verify signature - git checkout v2.2.0 + git checkout $SNOWFLAKE_TAG go build .\client Move-Item -Path .\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe } @@ -264,7 +267,7 @@ jobs: git clone https://git.torproject.org/pluggable-transports/meek.git cd meek # TODO: verify signature - git checkout v0.37.0 + git checkout $MEEK_TAG go build .\meek-client Move-Item -Path .\meek-client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe } @@ -315,97 +318,199 @@ jobs: executor: name: win/default shell: powershell.exe + environment: + # NOTE: make sure to change when upgrading tor + TOR_TAG: tor-0.4.7.7 + # NOTE: make sure to change when upgrading obfs4proxy + OBFS4PROXY_TAG: obfs4proxy-0.0.13 + # NOTE: make sure to change when upgrading snowflake + SNOWFLAKE_TAG: v2.2.0 + # NOTE: make sure to change when upgrading meek + MEEK_TAG: v0.37.0 steps: - checkout + - run: - name: Install Python 3.9.12 (32-bit) + name: Install cygwin (32-bit) and required packages + shell: bash.exe --login -eo pipefail command: | - choco install python3 --params "/InstallDir32:C:\Python-32bit" --version=3.9.12 + curl -o setup-x86.exe https://cygwin.com/setup-x86.exe + curl -o setup-x86.exe.sig https://cygwin.com/setup-x86.exe.sig + curl -o pubring.asc https://cygwin.com/key/pubring.asc + + gpg --import pubring.asc + gpg --verify setup-x86.exe.sig setup-x86.exe + if [ $? -ne 0 ]; then + echo "cygwin signature doesn't verify" + exit -1 + fi + + ./setup-x86.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ \ + -P libssl-devel,libevent-devel,zlib-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg + + - restore_cache: + # NOTE: make sure to change when upgrading tor + key: build-win32-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} + - run: + name: Build tor + shell: C:\\cygwin32\\bin\\bash.exe --login -eo pipefail + command: | + if [[ -d "/cygdrive/c/Users/circleci/build/tor" ]]; then + echo "tor already compiled, skipping" + exit 0 + fi + + mkdir -p /cygdrive/c/Users/circleci/src + mkdir -p /cygdrive/c/Users/circleci/build + + # Get source + curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import + cd /cygdrive/c/Users/circleci/src + git clone https://git.torproject.org/tor.git + cd tor + git tag -v $TOR_TAG + if [ $? -ne 0 ]; then + echo "tor tag doesn't verify" + exit -1 + fi + git checkout $TOR_TAG + + # Build + ./autogen.sh + ./configure \ + --prefix=/opt/tor-build \ + --disable-asciidoc + make -j$(nproc) + make install + cp -r /opt/tor-build /cygdrive/c/Users/circleci/build/tor + - save_cache: + # NOTE: make sure to change when upgrading tor + key: build-win32-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\build\tor + + - run: + name: Copy tor binaries into project + command: | + New-Item -ItemType Directory -Force -Path C:\Users\circleci\project\desktop\onionshare\resources\tor + Copy-Item -Path C:\Users\circleci\build\tor\bin\tor.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor + Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor + Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip6 -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor + - run: name: Install golang (32-bit) command: | cd ~\Downloads Invoke-WebRequest -Uri https://go.dev/dl/go1.18.windows-386.msi -OutFile go1.18.windows-386.msi msiexec.exe /i go1.18.windows-386.msi /quiet /L*V go-install.log + + - restore_cache: + # NOTE: make sure to change when upgrading obfs4proxy + key: build-win32-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} + - run: + name: Build obfs4proxy (32-bit) + command: | + if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe) -eq $True) { + Write-Output "obfs4proxy already built" + } else { + # curl https://keys.openpgp.org/vks/v1/by-fingerprint/9EB1A490C73CC5D44DFB3E47BFBD1C7B8A6EC81A | gpg --import + New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\obfs4proxy + cd C:\Users\circleci\src\obfs4proxy + git clone https://gitlab.com/yawning/obfs4 + cd obfs4 + # TODO: verify signature + git checkout $OBFS4PROXY_TAG + "C:\\Program Files (x86)\\Go\\bin\\go" build .\obfs4proxy + Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe + } + - save_cache: + # NOTE: make sure to change when upgrading obfs4proxy + key: build-win32-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe + + - restore_cache: + # NOTE: make sure to change when upgrading snowflake + key: build-win32-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} + - run: + name: Build snowflake-client (32-bit) + command: | + if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe) -eq $True) { + Write-Output "snowflake already built" + } else { + New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\obfs4proxy + cd C:\Users\circleci\src\obfs4proxy + git clone https://git.torproject.org/pluggable-transports/snowflake.git + cd snowflake + # TODO: verify signature + git checkout $SNOWFLAKE_TAG + "C:\\Program Files (x86)\\Go\\bin\\go" build .\client + Move-Item -Path .\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe + } + - save_cache: + # NOTE: make sure to change when upgrading snowflake + key: build-win32-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe + + - restore_cache: + # NOTE: make sure to change when upgrading meek + key: build-win32-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} + - run: + name: Build meek-client (32-bit) + command: | + if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe) -eq $True) { + Write-Output "snowflake already built" + } else { + New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\meek + cd C:\Users\circleci\src\meek + git clone https://git.torproject.org/pluggable-transports/meek.git + cd meek + # TODO: verify signature + git checkout $MEEK_TAG + "C:\\Program Files (x86)\\Go\\bin\\go" build .\meek-client + Move-Item -Path .\meek-client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe + } + - save_cache: + # NOTE: make sure to change when upgrading meek + key: build-win32-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe + + - run: + name: Install Python 3.9.12 (32-bit) + command: | + choco install python3 --params "/InstallDir32:C:\Python-32bit" --version=3.9.12 + - run: name: Install poetry (32-bit) command: C:\Python-32bit\Scripts\pip install poetry + + - restore_cache: + key: build-win32-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - run: name: Install poetry dependencies command: | cd ~\project\desktop C:\Python-32bit\Scripts\poetry install - - restore_cache: - key: download-tor-win32-0.4.7.7 - - run: - name: Download tor (Windows Expert Bundle) - command: | - mkdir ~\Downloads\tor - cd ~\Downloads\tor - $URL = "https://www.torproject.org/dist/torbrowser/11.0.13/tor-win32-0.4.7.7.zip" - $Filename = "tor-win32-0.4.7.7.zip" - $ExpectedHash = "6af0088460d61a5e60971aea94562a3bed2fb597da6b8ac77948a5f6424dfa29" - Invoke-WebRequest -Uri $URL -OutFile $Filename - $FileHash = Get-FileHash $Filename - if($FileHash.Hash -ne $ExpectedHash) { throw "Invalid hash" } - mkdir ~\Downloads\tor\tor-win32 - Expand-Archive -LiteralPath tor-win32-0.4.7.7.zip -DestinationPath ~\Downloads\tor\tor-win32 - save_cache: - key: download-tor-win32-0.4.7.7 + key: build-win32-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} paths: - - ~\Downloads\tor\tor-win32 - - run: - name: Copy tor binary into app - command: cp ~\Downloads\tor\tor-win32\* ~\project\desktop\onionshare\resources\tor - - restore_cache: - key: build-win32-obfs4proxy-v0.0.13 - - run: - name: Build obfs4proxy - command: | - mkdir ~\Downloads\obfs4proxy - cd Downloads\obfs4proxy - git clone https://gitlab.com/yawning/obfs4 - git checkout obfs4proxy-0.0.13 - cd obfs4 - "C:\\Program Files (x86)\\Go\\bin\\go" build .\obfs4proxy - Move-Item -Path .\obfs4proxy.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe - - save_cache: - key: build-win32-obfs4proxy-v0.0.13 - paths: - - ~\project\desktop\onionshare\resources\tor\Tor\obfs4proxy.exe - - restore_cache: - key: build-win32-snowflake-v2.1.0 - - run: - name: Build snowflake-client - command: | - "C:\\Program Files (x86)\\Go\\bin\\go" install git.torproject.org/pluggable-transports/snowflake.git/client@v2.1.0 - Move-Item -Path ~\go\bin\client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe - - save_cache: - key: build-win32-snowflake-v2.1.0 - paths: - - ~\project\desktop\onionshare\resources\tor\Tor\snowflake-client.exe - - restore_cache: - key: build-win32-meek-v0.37.0 - - run: - name: Build meek-client - command: | - "C:\\Program Files (x86)\\Go\\bin\\go" install git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0 - Move-Item -Path ~\go\bin\meek-client.exe -Destination ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - - save_cache: - key: build-win32-meek-v0.37.0 - paths: - - ~\project\desktop\onionshare\resources\tor\Tor\meek-client.exe + - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs + - run: name: Build OnionShare command: | cd ~\project\desktop C:\Python-32bit\Scripts\poetry run python .\setup-freeze.py build C:\Python-32bit\Scripts\poetry run python .\scripts\build-windows.py cleanup-build + - run: name: Compress command: | mv ~\project\desktop\build\exe.win32-3.9\ ~\onionshare-win32 Compress-Archive -LiteralPath ~\onionshare-win32 -DestinationPath ~\onionshare-win32.zip + - store_artifacts: path: ~\onionshare-win32.zip diff --git a/cli/onionshare_cli/common.py b/cli/onionshare_cli/common.py index da5204cf..ca0acc69 100644 --- a/cli/onionshare_cli/common.py +++ b/cli/onionshare_cli/common.py @@ -336,10 +336,16 @@ class Common: # If tor.exe isn't there, mayber we're running from the source tree if not os.path.exists(tor_path): + self.log( + "Common", "get_tor_paths", f"Cannot find tor.exe at {tor_path}" + ) base_path = os.path.join(os.getcwd(), "onionshare", "resources", "tor") tor_path = os.path.join(base_path, "tor.exe") if not os.path.exists(tor_path): + self.log( + "Common", "get_tor_paths", f"Cannot find tor.exe at {tor_path}" + ) raise CannotFindTor() obfs4proxy_file_path = os.path.join(base_path, "tor", "obfs4proxy.exe") @@ -499,7 +505,7 @@ class Common: if valid_bridges: return valid_bridges else: - return False + return False def is_flatpak(self): """ @@ -513,7 +519,6 @@ class Common: """ return os.environ.get("SNAP_INSTANCE_NAME") == "onionshare" - @staticmethod def random_string(num_bytes, output_len=None): """ From 7b65e419ff21761b13b946d9b9526bc1332c5a1d Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 16:09:00 -0700 Subject: [PATCH 095/154] Try changing keys for windows poetry cache --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a59ae14f..5b447917 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -287,14 +287,14 @@ jobs: command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - - restore_cache: - key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + key: build-win64-desktop-poetry-deps-{{ checksum "project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - run: name: Install poetry dependencies command: | cd C:\Users\circleci\project\desktop poetry install - save_cache: - key: build-win64-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + key: build-win64-desktop-poetry-deps-{{ checksum "project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs @@ -487,14 +487,14 @@ jobs: command: C:\Python-32bit\Scripts\pip install poetry - restore_cache: - key: build-win32-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + key: build-win32-desktop-poetry-deps-{{ checksum "project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - run: name: Install poetry dependencies command: | cd ~\project\desktop C:\Python-32bit\Scripts\poetry install - save_cache: - key: build-win32-desktop-poetry-deps-{{ checksum "C:\Users\circleci\project\desktop\poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + key: build-win32-desktop-poetry-deps-{{ checksum "project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs From 67262fa810e443c7b7b5edbd8ed4d8de3a687584 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 16:12:38 -0700 Subject: [PATCH 096/154] Fix cygwin 32-bit path --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b447917..a4885155 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -353,7 +353,7 @@ jobs: key: build-win32-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - run: name: Build tor - shell: C:\\cygwin32\\bin\\bash.exe --login -eo pipefail + shell: C:\\cygwin\\bin\\bash.exe --login -eo pipefail command: | if [[ -d "/cygdrive/c/Users/circleci/build/tor" ]]; then echo "tor already compiled, skipping" From d2c55a0a8cd8e66e8948f2d32647548d7c99201f Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 17:02:21 -0700 Subject: [PATCH 097/154] Try fixing windows poetry cache key --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a4885155..8a314ece 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -287,14 +287,14 @@ jobs: command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - - restore_cache: - key: build-win64-desktop-poetry-deps-{{ checksum "project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - run: name: Install poetry dependencies command: | cd C:\Users\circleci\project\desktop poetry install - save_cache: - key: build-win64-desktop-poetry-deps-{{ checksum "project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs @@ -487,14 +487,14 @@ jobs: command: C:\Python-32bit\Scripts\pip install poetry - restore_cache: - key: build-win32-desktop-poetry-deps-{{ checksum "project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - run: name: Install poetry dependencies command: | cd ~\project\desktop C:\Python-32bit\Scripts\poetry install - save_cache: - key: build-win32-desktop-poetry-deps-{{ checksum "project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs From f7dd3380f82b263984617cae7b82e1f63185b92e Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 17:07:22 -0700 Subject: [PATCH 098/154] Fix tor path again --- cli/onionshare_cli/common.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/onionshare_cli/common.py b/cli/onionshare_cli/common.py index ca0acc69..67d89571 100644 --- a/cli/onionshare_cli/common.py +++ b/cli/onionshare_cli/common.py @@ -332,7 +332,7 @@ class Common: # In Windows, the Tor binaries are in the onionshare package, not the onionshare_cli package base_path = self.get_resource_path("tor") base_path = base_path.replace("onionshare_cli", "onionshare") - tor_path = os.path.join(base_path, "tor", "tor.exe") + tor_path = os.path.join(base_path, "tor.exe") # If tor.exe isn't there, mayber we're running from the source tree if not os.path.exists(tor_path): @@ -348,11 +348,11 @@ class Common: ) raise CannotFindTor() - obfs4proxy_file_path = os.path.join(base_path, "tor", "obfs4proxy.exe") - snowflake_file_path = os.path.join(base_path, "tor", "snowflake-client.exe") - meek_client_file_path = os.path.join(base_path, "tor", "meek-client.exe") - tor_geo_ip_file_path = os.path.join(base_path, "tor", "geoip") - tor_geo_ipv6_file_path = os.path.join(base_path, "tor", "geoip6") + obfs4proxy_file_path = os.path.join(base_path, "obfs4proxy.exe") + snowflake_file_path = os.path.join(base_path, "snowflake-client.exe") + meek_client_file_path = os.path.join(base_path, "meek-client.exe") + tor_geo_ip_file_path = os.path.join(base_path, "geoip") + tor_geo_ipv6_file_path = os.path.join(base_path, "geoip6") elif self.platform == "Darwin": # Let's see if we have tor binaries in the onionshare GUI package From 972d5832434150222273943d643945b43e336fce Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 17:24:46 -0700 Subject: [PATCH 099/154] Fix 32-bit go path --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8a314ece..dd817bbf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -420,7 +420,7 @@ jobs: cd obfs4 # TODO: verify signature git checkout $OBFS4PROXY_TAG - "C:\\Program Files (x86)\\Go\\bin\\go" build .\obfs4proxy + "C:\Program Files (x86)\Go\bin\go" build .\obfs4proxy Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe } - save_cache: @@ -444,7 +444,7 @@ jobs: cd snowflake # TODO: verify signature git checkout $SNOWFLAKE_TAG - "C:\\Program Files (x86)\\Go\\bin\\go" build .\client + "C:\Program Files (x86)\Go\bin\go" build .\client Move-Item -Path .\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe } - save_cache: @@ -468,7 +468,7 @@ jobs: cd meek # TODO: verify signature git checkout $MEEK_TAG - "C:\\Program Files (x86)\\Go\\bin\\go" build .\meek-client + "C:\Program Files (x86)\Go\bin\go" build .\meek-client Move-Item -Path .\meek-client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe } - save_cache: From 4f38dcbdf5d7f395a8e676d1985881aa29323359 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 29 May 2022 18:22:32 -0700 Subject: [PATCH 100/154] Compile libevent again --- .circleci/config.yml | 106 ++++++++++++++++++++++++++++++------------- 1 file changed, 75 insertions(+), 31 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dd817bbf..79c4c642 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,10 +13,10 @@ workflows: requires: - test-cli # - test-gui - - build-win32: - requires: - - test-cli - # - test-gui + # - build-win32: + # requires: + # - test-cli + # # - test-gui # - build-macos: # requires: # - test-cli @@ -87,13 +87,15 @@ jobs: name: win/default shell: powershell.exe environment: - # NOTE: make sure to change when upgrading tor + # NOTE: change when upgrading libevent + LIBEVENT_TAG: release-2.1.12-stable + # NOTE: change when upgrading tor TOR_TAG: tor-0.4.7.7 - # NOTE: make sure to change when upgrading obfs4proxy + # NOTE: change when upgrading obfs4proxy OBFS4PROXY_TAG: obfs4proxy-0.0.13 - # NOTE: make sure to change when upgrading snowflake + # NOTE: change when upgrading snowflake SNOWFLAKE_TAG: v2.2.0 - # NOTE: make sure to change when upgrading meek + # NOTE: change when upgrading meek MEEK_TAG: v0.37.0 steps: - checkout @@ -114,7 +116,7 @@ jobs: fi ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ \ - -P libssl-devel,libevent-devel,zlib-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg + -P libssl-devel,zlib-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg # # Build libevent # - run: @@ -150,13 +152,53 @@ jobs: # make install # cp -r /opt/libevent-build /cygdrive/c/Users/circleci/build/libevent # - save_cache: - # # NOTE: make sure to change when upgrading libevent + # # NOTE: change when upgrading libevent # key: "build-win64-libevent-2.1.12-{{ .Environment.CACHE_VERSION }}" # paths: # - C:\Users\circleci\build\libevent - restore_cache: - # NOTE: make sure to change when upgrading tor + # NOTE: change when upgrading libevent + key: build-win64-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + - run: + name: Build tor + shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail + command: | + if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then + echo "libevent already compiled, skipping" + exit 0 + fi + + mkdir -p /cygdrive/c/Users/circleci/src + mkdir -p /cygdrive/c/Users/circleci/build + + # Get source + curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import + cd /cygdrive/c/Users/circleci/src + git clone https://github.com/libevent/libevent.git + cd libevent + git tag -v $LIBEVENT_TAG + if [ $? -ne 0 ]; then + echo "libevent tag doesn't verify" + exit -1 + fi + git checkout $LIBEVENT_TAG + + # Build + ./autogen.sh + ./configure \ + --prefix=/cygdrive/c/Users/circleci/build/libevent \ + --with-pic + make -j$(nproc) + make install + - save_cache: + # NOTE: change when upgrading libevent + key: build-win64-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\build\libevent + + - restore_cache: + # NOTE: change when upgrading tor key: build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - run: name: Build tor @@ -186,12 +228,14 @@ jobs: ./autogen.sh ./configure \ --prefix=/opt/tor-build \ - --disable-asciidoc + --disable-asciidoc \ + --enable-static-libevent \ + --with-libevent-dir=/cygdrive/c/Users/circleci/build/libevent make -j$(nproc) make install cp -r /opt/tor-build /cygdrive/c/Users/circleci/build/tor - save_cache: - # NOTE: make sure to change when upgrading tor + # NOTE: change when upgrading tor key: build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\build\tor @@ -205,7 +249,7 @@ jobs: Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip6 -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor - restore_cache: - # NOTE: make sure to change when upgrading obfs4proxy + # NOTE: change when upgrading obfs4proxy key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} - run: name: Build obfs4proxy @@ -224,13 +268,13 @@ jobs: Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe } - save_cache: - # NOTE: make sure to change when upgrading obfs4proxy + # NOTE: change when upgrading obfs4proxy key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe - restore_cache: - # NOTE: make sure to change when upgrading snowflake + # NOTE: change when upgrading snowflake key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} - run: name: Build snowflake-client @@ -248,13 +292,13 @@ jobs: Move-Item -Path .\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe } - save_cache: - # NOTE: make sure to change when upgrading snowflake + # NOTE: change when upgrading snowflake key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe - restore_cache: - # NOTE: make sure to change when upgrading meek + # NOTE: change when upgrading meek key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} - run: name: Build meek-client @@ -272,7 +316,7 @@ jobs: Move-Item -Path .\meek-client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe } - save_cache: - # NOTE: make sure to change when upgrading meek + # NOTE: change when upgrading meek key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe @@ -319,13 +363,13 @@ jobs: name: win/default shell: powershell.exe environment: - # NOTE: make sure to change when upgrading tor + # NOTE: change when upgrading tor TOR_TAG: tor-0.4.7.7 - # NOTE: make sure to change when upgrading obfs4proxy + # NOTE: change when upgrading obfs4proxy OBFS4PROXY_TAG: obfs4proxy-0.0.13 - # NOTE: make sure to change when upgrading snowflake + # NOTE: change when upgrading snowflake SNOWFLAKE_TAG: v2.2.0 - # NOTE: make sure to change when upgrading meek + # NOTE: change when upgrading meek MEEK_TAG: v0.37.0 steps: - checkout @@ -349,7 +393,7 @@ jobs: -P libssl-devel,libevent-devel,zlib-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg - restore_cache: - # NOTE: make sure to change when upgrading tor + # NOTE: change when upgrading tor key: build-win32-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - run: name: Build tor @@ -384,7 +428,7 @@ jobs: make install cp -r /opt/tor-build /cygdrive/c/Users/circleci/build/tor - save_cache: - # NOTE: make sure to change when upgrading tor + # NOTE: change when upgrading tor key: build-win32-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\build\tor @@ -405,7 +449,7 @@ jobs: msiexec.exe /i go1.18.windows-386.msi /quiet /L*V go-install.log - restore_cache: - # NOTE: make sure to change when upgrading obfs4proxy + # NOTE: change when upgrading obfs4proxy key: build-win32-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} - run: name: Build obfs4proxy (32-bit) @@ -424,13 +468,13 @@ jobs: Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe } - save_cache: - # NOTE: make sure to change when upgrading obfs4proxy + # NOTE: change when upgrading obfs4proxy key: build-win32-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe - restore_cache: - # NOTE: make sure to change when upgrading snowflake + # NOTE: change when upgrading snowflake key: build-win32-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} - run: name: Build snowflake-client (32-bit) @@ -448,13 +492,13 @@ jobs: Move-Item -Path .\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe } - save_cache: - # NOTE: make sure to change when upgrading snowflake + # NOTE: change when upgrading snowflake key: build-win32-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe - restore_cache: - # NOTE: make sure to change when upgrading meek + # NOTE: change when upgrading meek key: build-win32-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} - run: name: Build meek-client (32-bit) @@ -472,7 +516,7 @@ jobs: Move-Item -Path .\meek-client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe } - save_cache: - # NOTE: make sure to change when upgrading meek + # NOTE: change when upgrading meek key: build-win32-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe From 2677f3330945b16417e573807950ecc06bc21d73 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Wed, 1 Jun 2022 20:40:41 -0700 Subject: [PATCH 101/154] Delete commented out libevent build, and install tor directly into build/tor --- .circleci/config.yml | 44 ++------------------------------------------ 1 file changed, 2 insertions(+), 42 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79c4c642..17cd8b95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,50 +118,11 @@ jobs: ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ \ -P libssl-devel,zlib-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg - # # Build libevent - # - run: - # name: Build libevent - # shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail - # command: | - # if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then - # echo "libevent already compiled, skipping" - # exit 0 - # fi - - # mkdir -p /cygdrive/c/Users/circleci/src - # mkdir -p /cygdrive/c/Users/circleci/build - - # # Get source - # curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import - # cd /cygdrive/c/Users/circleci/src - # git clone https://github.com/libevent/libevent.git - # cd libevent - # git tag -v $LIBEVENT_TAG - # if [ $? -ne 0 ]; then - # echo "libevent tag doesn't verify" - # exit -1 - # fi - # git checkout $LIBEVENT_TAG - - # # Build - # ./autogen.sh - # ./configure \ - # --prefix=/opt/libevent-build \ - # --with-pic - # make -j$(nproc) - # make install - # cp -r /opt/libevent-build /cygdrive/c/Users/circleci/build/libevent - # - save_cache: - # # NOTE: change when upgrading libevent - # key: "build-win64-libevent-2.1.12-{{ .Environment.CACHE_VERSION }}" - # paths: - # - C:\Users\circleci\build\libevent - - restore_cache: # NOTE: change when upgrading libevent key: build-win64-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - run: - name: Build tor + name: Build libevent shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail command: | if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then @@ -227,13 +188,12 @@ jobs: # Build ./autogen.sh ./configure \ - --prefix=/opt/tor-build \ + --prefix=/cygdrive/c/Users/circleci/build/tor \ --disable-asciidoc \ --enable-static-libevent \ --with-libevent-dir=/cygdrive/c/Users/circleci/build/libevent make -j$(nproc) make install - cp -r /opt/tor-build /cygdrive/c/Users/circleci/build/tor - save_cache: # NOTE: change when upgrading tor key: build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} From 34221947f8f1829302c41633a4a87b16c29a225d Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 2 Jun 2022 19:09:15 -0700 Subject: [PATCH 102/154] Try cross-compiling tor for windows in linux --- .circleci/config.yml | 217 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 215 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 17cd8b95..a680ee82 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,10 +9,13 @@ workflows: jobs: - test-cli # - test-gui - - build-win64: + - build-tor-windows: requires: - test-cli - # - test-gui + # - build-win64: + # requires: + # - test-cli + # # - test-gui # - build-win32: # requires: # - test-cli @@ -82,6 +85,216 @@ jobs: cd ~/project/desktop QT_DEBUG_PLUGINS=1 xvfb-run poetry run pytest -v ./tests/test_gui_*.py + # Based off of https://github.com/ahf/tor-win32 + build-tor-windows: + docker: + - image: cimg/python:3.9 + environment: + # NOTE: change when upgrading openssl + OPENSSL_VERSION: "3.0.3" + # NOTE: change when upgrading libevent + LIBEVENT_VERSION: "2.1.12-stable" + # NOTE: change when upgrading tor + TOR_TAG: tor-0.4.7.7 + steps: + - checkout + - run: + name: Install build dependencies + command: | + sudo apt-get update + sudo apt-get install -y autoconf automake libtool + sudo apt-get install -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + + - run: + name: Create folders + command: | + mkdir ~/build + mkdir ~/build/src + mkdir ~/build/dist + mkdir ~/build/prefix-x32 + mkdir ~/build/prefix-x64 + + - restore_cache: + # NOTE: change when upgrading openssl + key: build-tor-windows-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} + - run: + name: Download and build openssl + command: | + cd ~/build/dist + + # download + wget https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz + echo "ee0078adcef1de5f003c62c80cc96527721609c6f3bb42b7795df31f8b558c0b openssl-$OPENSSL_VERSION.tar.gz" | sha256sum --check --status + if [ $? -ne 0 ]; then + echo "openssl checksum failed" + exit -1 + fi + + # extract + cd ~/build/ + tar zxfv ~/build/dist/openssl-$OPENSSL_VERSION.tar.gz -C ~/build/src/ + mv ~/build/src/openssl-$OPENSSL_VERSION ~/build/src/openssl-$OPENSSL_VERSION-x32 + cp -r ~/build/src/openssl-$OPENSSL_VERSION-x32 ~/build/src/openssl-$OPENSSL_VERSION-x64 + + # build 32-bit + export MINGW=mingw + export HOST=i686-w64-mingw32 + cd ~/build/src/openssl-$OPENSSL_VERSION-x32 + ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/home/circleci/build/prefix-x32 + make -j$(nproc) + + # build 64-bit + export MINGW=mingw64 + export HOST=x86_64-w64-mingw32 + cd ~/build/src/openssl-$OPENSSL_VERSION-x64 + ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/home/circleci/build/prefix-x64 + make -j$(nproc) + - save_cache: + # NOTE: change when upgrading openssl + key: build-tor-windows-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} + paths: + - /home/circleci/build/dist/openssl-3.0.3.tar.gz + - /home/circleci/build/src/openssl-3.0.3-x32 + - /home/circleci/build/src/openssl-3.0.3-x64 + + - run: + name: make install openssl + command: | + cd ~/build/src/openssl-$OPENSSL_VERSION-x32 + make install + cd ~/build/src/openssl-$OPENSSL_VERSION-x64 + make install + + - restore_cache: + # NOTE: change when upgrading libevent + key: build-tor-windows-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + - run: + name: Download and build libevent + command: | + cd ~/build/dist + + # download + wget https://github.com/libevent/libevent/releases/download/release-$LIBEVENT_VERSION/libevent-$LIBEVENT_VERSION.tar.gz + echo "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb libevent-$LIBEVENT_VERSION.tar.gz | sha256sum --check --status + if [ $? -ne 0 ]; then + echo "libevent checksum failed" + exit -1 + fi + + # extract + cd ~/build/ + tar zxfv ~/build/dist/libevent-$LIBEVENT_VERSION.tar.gz -C ~/build/src/ + mv ~/build/src/libevent-$LIBEVENT_VERSION ~/build/src/libevent-$LIBEVENT_VERSION-x32 + cp -r ~/build/src/libevent-$LIBEVENT_VERSION-x32 ~/build/src/libevent-$LIBEVENT_VERSION-x64 + + # build 32-bit + export MINGW=mingw + export HOST=i686-w64-mingw32 + cd ~/build/src/libevent-$LIBEVENT_VERSION-x32 + ./configure --host=$HOST --prefix=--prefix=/home/circleci/build/prefix-x32 --disable-openssl + make -j$(nproc) + + # build 64-bit + export MINGW=mingw64 + export HOST=x86_64-w64-mingw32 + cd ~/build/src/libevent-$LIBEVENT_VERSION-x64 + ./configure --host=$HOST --prefix=--prefix=/home/circleci/build/prefix-x32 --disable-openssl + make -j$(nproc) + - save_cache: + # NOTE: change when upgrading libevent + key: build-tor-windows-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + paths: + - /home/circleci/build/dist/libevent-2.1.12-stable.tar.gz + - /home/circleci/build/src/libevent-2.1.12-stable-x32 + - /home/circleci/build/src/libevent-2.1.12-stable-x64 + + - run: + name: make install libevent + command: | + cd ~/build/src/libevent-$LIBEVENT_VERSION-x32 + make install + cd ~/build/src/libevent-$LIBEVENT_VERSION-x64 + make install + + - restore_cache: + # NOTE: change when upgrading tor + key: build-tor-windows-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} + - run: + name: Download and build tor + command: | + + + # git clone + curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import + cd ~/build/src + git clone https://git.torproject.org/tor.git + cd tor + git tag -v $TOR_TAG + if [ $? -ne 0 ]; then + echo "tor tag doesn't verify" + exit -1 + fi + git checkout $TOR_TAG + + cd ~/build + mv ~/build/src/tor ~/build/src/tor-x32 + cp -r ~/build/src/tor-x32 ~/build/src/tor-x64 + + # build 32-bit + export MINGW=mingw + export HOST=i686-w64-mingw32 + cd ~/build/src/tor-x32 + ./configure --host=$(HOST) \ + --disable-asciidoc \ + --disable-zstd \ + --disable-lzma \ + --enable-static-libevent \ + --with-libevent-dir=$(PREFIX) \ + --enable-static-openssl \ + --with-openssl-dir=$(PREFIX) \ + --disable-tool-name-check \ + --enable-fatal-warnings \ + --prefix=$(PREFIX) + make -j$(nproc) + + # build 64-bit + export MINGW=mingw64 + export HOST=x86_64-w64-mingw32 + cd ~/build/src/tor-x64 + ./configure --host=$(HOST) \ + --disable-asciidoc \ + --disable-zstd \ + --disable-lzma \ + --enable-static-libevent \ + --with-libevent-dir=$(PREFIX) \ + --enable-static-openssl \ + --with-openssl-dir=$(PREFIX) \ + --disable-tool-name-check \ + --enable-fatal-warnings \ + --prefix=$(PREFIX) + make -j$(nproc) + make -j$(nproc) + - save_cache: + # NOTE: change when upgrading libevent + key: build-tor-windows-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} + paths: + - /home/circleci/build/src/tor-x32 + - /home/circleci/build/src/tor-x64 + + - run: + name: make install tor + command: | + cd ~/build/src/tor-x32 + make install + cd ~/build/src/tor-x64 + make install + + - persist_to_workspace: + root: ~/build/prefix + paths: + - prefix-x32 + - prefix-x64 + build-win64: executor: name: win/default From bed48c57fe56fb3d90909cd9d372f0dc2bdb347f Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 2 Jun 2022 19:36:10 -0700 Subject: [PATCH 103/154] make install, and then cache the final installed version --- .circleci/config.yml | 102 +++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a680ee82..15ded148 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,15 +111,24 @@ jobs: mkdir ~/build mkdir ~/build/src mkdir ~/build/dist - mkdir ~/build/prefix-x32 - mkdir ~/build/prefix-x64 + mkdir ~/build/prefix-openssl-x32 + mkdir ~/build/prefix-openssl-x64 + mkdir ~/build/prefix-libevent-x32 + mkdir ~/build/prefix-libevent-x64 + mkdir ~/build/prefix-tor-x32 + mkdir ~/build/prefix-tor-x64 - restore_cache: # NOTE: change when upgrading openssl key: build-tor-windows-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} - run: - name: Download and build openssl + name: Build openssl command: | + if [[ -d "~/build/prefix-openssl-x32/bin" ]]; then + echo "openssl already compiled, skipping" + exit 0 + fi + cd ~/build/dist # download @@ -140,37 +149,35 @@ jobs: export MINGW=mingw export HOST=i686-w64-mingw32 cd ~/build/src/openssl-$OPENSSL_VERSION-x32 - ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/home/circleci/build/prefix-x32 + ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/home/circleci/build/prefix-openssl-x32 make -j$(nproc) + make install # build 64-bit export MINGW=mingw64 export HOST=x86_64-w64-mingw32 cd ~/build/src/openssl-$OPENSSL_VERSION-x64 - ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/home/circleci/build/prefix-x64 + ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/home/circleci/build/prefix-openssl-x64 make -j$(nproc) + make install - save_cache: # NOTE: change when upgrading openssl key: build-tor-windows-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/dist/openssl-3.0.3.tar.gz - - /home/circleci/build/src/openssl-3.0.3-x32 - - /home/circleci/build/src/openssl-3.0.3-x64 - - - run: - name: make install openssl - command: | - cd ~/build/src/openssl-$OPENSSL_VERSION-x32 - make install - cd ~/build/src/openssl-$OPENSSL_VERSION-x64 - make install + - /home/circleci/build/prefix-openssl-x32 + - /home/circleci/build/prefix-openssl-x64 - restore_cache: # NOTE: change when upgrading libevent key: build-tor-windows-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - run: - name: Download and build libevent + name: Build libevent command: | + if [[ -d "~/build/prefix-libevent-x32/lib" ]]; then + echo "libevent already compiled, skipping" + exit 0 + fi + cd ~/build/dist # download @@ -191,38 +198,34 @@ jobs: export MINGW=mingw export HOST=i686-w64-mingw32 cd ~/build/src/libevent-$LIBEVENT_VERSION-x32 - ./configure --host=$HOST --prefix=--prefix=/home/circleci/build/prefix-x32 --disable-openssl + ./configure --host=$HOST --prefix=/home/circleci/build/prefix-libevent-x32 --disable-openssl make -j$(nproc) + make install # build 64-bit export MINGW=mingw64 export HOST=x86_64-w64-mingw32 cd ~/build/src/libevent-$LIBEVENT_VERSION-x64 - ./configure --host=$HOST --prefix=--prefix=/home/circleci/build/prefix-x32 --disable-openssl + ./configure --host=$HOST --prefix=/home/circleci/build/prefix-libevent-x64 --disable-openssl make -j$(nproc) + make install - save_cache: # NOTE: change when upgrading libevent key: build-tor-windows-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/dist/libevent-2.1.12-stable.tar.gz - - /home/circleci/build/src/libevent-2.1.12-stable-x32 - - /home/circleci/build/src/libevent-2.1.12-stable-x64 - - - run: - name: make install libevent - command: | - cd ~/build/src/libevent-$LIBEVENT_VERSION-x32 - make install - cd ~/build/src/libevent-$LIBEVENT_VERSION-x64 - make install + - /home/circleci/build/prefix-libevent-x32 + - /home/circleci/build/prefix-libevent-x64 - restore_cache: # NOTE: change when upgrading tor key: build-tor-windows-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - run: - name: Download and build tor + name: Build tor command: | - + if [[ -d "~/build/prefix-tor-x32/bin" ]]; then + echo "tor already compiled, skipping" + exit 0 + fi # git clone curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import @@ -244,56 +247,49 @@ jobs: export MINGW=mingw export HOST=i686-w64-mingw32 cd ~/build/src/tor-x32 - ./configure --host=$(HOST) \ + ./configure --host=$HOST \ --disable-asciidoc \ --disable-zstd \ --disable-lzma \ --enable-static-libevent \ - --with-libevent-dir=$(PREFIX) \ + --with-libevent-dir=/home/circleci/build/prefix-libevent-x32 \ --enable-static-openssl \ - --with-openssl-dir=$(PREFIX) \ + --with-openssl-dir=/home/circleci/build/prefix-openssl-x32 \ --disable-tool-name-check \ --enable-fatal-warnings \ - --prefix=$(PREFIX) + --prefix=/home/circleci/build/prefix-tor-x32 make -j$(nproc) + make install # build 64-bit export MINGW=mingw64 export HOST=x86_64-w64-mingw32 cd ~/build/src/tor-x64 - ./configure --host=$(HOST) \ + ./configure --host=$HOST \ --disable-asciidoc \ --disable-zstd \ --disable-lzma \ --enable-static-libevent \ - --with-libevent-dir=$(PREFIX) \ + --with-libevent-dir=/home/circleci/build/prefix-libevent-x64 \ --enable-static-openssl \ - --with-openssl-dir=$(PREFIX) \ + --with-openssl-dir=/home/circleci/build/prefix-openssl-x64 \ --disable-tool-name-check \ --enable-fatal-warnings \ - --prefix=$(PREFIX) - make -j$(nproc) + --prefix=/home/circleci/build/prefix-tor-x64 make -j$(nproc) + make install - save_cache: # NOTE: change when upgrading libevent key: build-tor-windows-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/src/tor-x32 - - /home/circleci/build/src/tor-x64 - - - run: - name: make install tor - command: | - cd ~/build/src/tor-x32 - make install - cd ~/build/src/tor-x64 - make install + - /home/circleci/build/prefix-tor-x32 + - /home/circleci/build/prefix-tor-x64 - persist_to_workspace: root: ~/build/prefix paths: - - prefix-x32 - - prefix-x64 + - prefix-tor-x32 + - prefix-tor-x64 build-win64: executor: From 6a4449e9a892e21551e9fecaf20262de8223d0b3 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 2 Jun 2022 20:22:04 -0700 Subject: [PATCH 104/154] Syntax error in libevent checksum verification --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 15ded148..0ca0d955 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -182,7 +182,7 @@ jobs: # download wget https://github.com/libevent/libevent/releases/download/release-$LIBEVENT_VERSION/libevent-$LIBEVENT_VERSION.tar.gz - echo "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb libevent-$LIBEVENT_VERSION.tar.gz | sha256sum --check --status + echo "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb libevent-$LIBEVENT_VERSION.tar.gz" | sha256sum --check --status if [ $? -ne 0 ]; then echo "libevent checksum failed" exit -1 @@ -279,7 +279,7 @@ jobs: make -j$(nproc) make install - save_cache: - # NOTE: change when upgrading libevent + # NOTE: change when upgrading tor key: build-tor-windows-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} paths: - /home/circleci/build/prefix-tor-x32 From 6a925d98e303d9a664214a10e994d880069949e0 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 2 Jun 2022 20:28:28 -0700 Subject: [PATCH 105/154] Fix testing if builds are already complete --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ca0d955..1d347646 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -124,7 +124,7 @@ jobs: - run: name: Build openssl command: | - if [[ -d "~/build/prefix-openssl-x32/bin" ]]; then + if [[ -f "/home/circleci/build/prefix-openssl-x32//bin/openssl.exe" ]]; then echo "openssl already compiled, skipping" exit 0 fi @@ -173,7 +173,7 @@ jobs: - run: name: Build libevent command: | - if [[ -d "~/build/prefix-libevent-x32/lib" ]]; then + if [[ -d "/home/circleci/build/prefix-libevent-x32/lib" ]]; then echo "libevent already compiled, skipping" exit 0 fi @@ -222,7 +222,7 @@ jobs: - run: name: Build tor command: | - if [[ -d "~/build/prefix-tor-x32/bin" ]]; then + if [[ -f "/home/circleci/build/prefix-tor-x32/bin/tor.exe" ]]; then echo "tor already compiled, skipping" exit 0 fi From 69fe52848269adaff4750e2b2843e498994e942a Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 3 Jun 2022 09:51:21 -0700 Subject: [PATCH 106/154] Install dev dependencies from ubuntu jammy --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d347646..eebf2c57 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,6 @@ jobs: docker: - image: cimg/python:3.9 steps: - - checkout - run: name: Install dependencies command: | @@ -101,9 +100,11 @@ jobs: - run: name: Install build dependencies command: | + # libevent requires newer version of automake than is available in ubuntu focal, so install from jammy instead + sudo bash -c 'echo deb http://archive.ubuntu.com/ubuntu/ jammy main restricted >> /etc/apt/sources.list' sudo apt-get update - sudo apt-get install -y autoconf automake libtool - sudo apt-get install -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + sudo apt-get install -y -t jammy autoconf automake libtool + sudo apt-get install -y -t jammy gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev - run: name: Create folders From c2456c106b3afadfbbc1e8a23e6f8d1a9d5b288e Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 3 Jun 2022 09:52:29 -0700 Subject: [PATCH 107/154] Oops, checkout test-cli --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eebf2c57..e530eee2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,6 +30,7 @@ jobs: docker: - image: cimg/python:3.9 steps: + - checkout - run: name: Install dependencies command: | @@ -96,7 +97,6 @@ jobs: # NOTE: change when upgrading tor TOR_TAG: tor-0.4.7.7 steps: - - checkout - run: name: Install build dependencies command: | From f918be9e71aa81e89396a15fa090b6f0128186af Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 3 Jun 2022 10:03:45 -0700 Subject: [PATCH 108/154] Try install build deps from impish instead of jammy --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e530eee2..f8f1118a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,11 +100,11 @@ jobs: - run: name: Install build dependencies command: | - # libevent requires newer version of automake than is available in ubuntu focal, so install from jammy instead - sudo bash -c 'echo deb http://archive.ubuntu.com/ubuntu/ jammy main restricted >> /etc/apt/sources.list' + # libevent requires newer version of automake than is available in ubuntu focal, so install from impish instead + sudo bash -c 'echo deb http://archive.ubuntu.com/ubuntu/ impish main restricted >> /etc/apt/sources.list' sudo apt-get update - sudo apt-get install -y -t jammy autoconf automake libtool - sudo apt-get install -y -t jammy gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + sudo apt-get install -y -t impish autoconf automake libtool + sudo apt-get install -y -t impish gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev - run: name: Create folders From 5ffa86be09dbf872db452adedfbf3f864ee39fc2 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Fri, 3 Jun 2022 10:12:21 -0700 Subject: [PATCH 109/154] Switch libevent to building from git, and try it all in focal --- .circleci/config.yml | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f8f1118a..ad589f20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,18 +93,16 @@ jobs: # NOTE: change when upgrading openssl OPENSSL_VERSION: "3.0.3" # NOTE: change when upgrading libevent - LIBEVENT_VERSION: "2.1.12-stable" + LIBEVENT_TAG: release-2.1.12-stable # NOTE: change when upgrading tor TOR_TAG: tor-0.4.7.7 steps: - run: name: Install build dependencies command: | - # libevent requires newer version of automake than is available in ubuntu focal, so install from impish instead - sudo bash -c 'echo deb http://archive.ubuntu.com/ubuntu/ impish main restricted >> /etc/apt/sources.list' sudo apt-get update - sudo apt-get install -y -t impish autoconf automake libtool - sudo apt-get install -y -t impish gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + sudo apt-get install -y autoconf automake libtool + sudo apt-get install -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev - run: name: Create folders @@ -179,26 +177,27 @@ jobs: exit 0 fi - cd ~/build/dist - - # download - wget https://github.com/libevent/libevent/releases/download/release-$LIBEVENT_VERSION/libevent-$LIBEVENT_VERSION.tar.gz - echo "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb libevent-$LIBEVENT_VERSION.tar.gz" | sha256sum --check --status + # git clone + curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import + cd ~/build/src + git clone https://github.com/libevent/libevent.git + cd libevent + git tag -v $LIBEVENT_TAG if [ $? -ne 0 ]; then - echo "libevent checksum failed" + echo "libevent tag doesn't verify" exit -1 fi + git checkout $LIBEVENT_TAG - # extract - cd ~/build/ - tar zxfv ~/build/dist/libevent-$LIBEVENT_VERSION.tar.gz -C ~/build/src/ - mv ~/build/src/libevent-$LIBEVENT_VERSION ~/build/src/libevent-$LIBEVENT_VERSION-x32 - cp -r ~/build/src/libevent-$LIBEVENT_VERSION-x32 ~/build/src/libevent-$LIBEVENT_VERSION-x64 + cd ~/build + mv ~/build/src/libevent ~/build/src/libevent-x32 + cp -r ~/build/src/libevent-x32 ~/build/src/libevent-x64 # build 32-bit export MINGW=mingw export HOST=i686-w64-mingw32 - cd ~/build/src/libevent-$LIBEVENT_VERSION-x32 + cd ~/build/src/libevent-x32 + ./autogen.sh ./configure --host=$HOST --prefix=/home/circleci/build/prefix-libevent-x32 --disable-openssl make -j$(nproc) make install @@ -206,7 +205,8 @@ jobs: # build 64-bit export MINGW=mingw64 export HOST=x86_64-w64-mingw32 - cd ~/build/src/libevent-$LIBEVENT_VERSION-x64 + cd ~/build/src/libevent-x64 + ./autogen.sh ./configure --host=$HOST --prefix=/home/circleci/build/prefix-libevent-x64 --disable-openssl make -j$(nproc) make install @@ -248,6 +248,7 @@ jobs: export MINGW=mingw export HOST=i686-w64-mingw32 cd ~/build/src/tor-x32 + ./autogen.sh ./configure --host=$HOST \ --disable-asciidoc \ --disable-zstd \ @@ -265,6 +266,7 @@ jobs: # build 64-bit export MINGW=mingw64 export HOST=x86_64-w64-mingw32 + ./autogen.sh cd ~/build/src/tor-x64 ./configure --host=$HOST \ --disable-asciidoc \ From f1618e326da4cfc26c8568cdc5a21763cccba510 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sat, 4 Jun 2022 20:52:41 -0700 Subject: [PATCH 110/154] Build 32-bit and 64-bit dependencies in separate tasks --- .circleci/config.yml | 232 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 202 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ad589f20..b42b010d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -119,12 +119,12 @@ jobs: - restore_cache: # NOTE: change when upgrading openssl - key: build-tor-windows-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} + key: build-tor-windows-download-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} - run: - name: Build openssl + name: Download openssl command: | - if [[ -f "/home/circleci/build/prefix-openssl-x32//bin/openssl.exe" ]]; then - echo "openssl already compiled, skipping" + if [[ -f "/home/circleci/build/dist/openssl-$OPENSSL_VERSION.tar.gz" ]]; then + echo "openssl already downloaded, skipping" exit 0 fi @@ -137,20 +137,110 @@ jobs: echo "openssl checksum failed" exit -1 fi + - save_cache: + # NOTE: change when upgrading openssl + key: build-tor-windows-download-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} + paths: + - /home/circleci/build/dist/openssl-3.0.3.tar.gz + + - restore_cache: + # NOTE: change when upgrading libevent + key: build-tor-windows-download-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + - run: + name: Download libevent + command: | + if [[ -d "/home/circleci/build/dist/libevent" ]]; then + echo "livevent already downloaded, skipping" + exit 0 + fi + + # git clone + curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import + cd ~/build/dist + git clone https://github.com/libevent/libevent.git + cd libevent + git tag -v $LIBEVENT_TAG + if [ $? -ne 0 ]; then + echo "libevent tag doesn't verify" + exit -1 + fi + git checkout $LIBEVENT_TAG + - save_cache: + # NOTE: change when upgrading libevent + key: build-tor-windows-download-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + paths: + - /home/circleci/build/dist/libevent + + - restore_cache: + # NOTE: change when upgrading tor + key: build-tor-windows-download-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + - run: + name: Download tor + command: | + if [[ -d "/home/circleci/build/dist/tor" ]]; then + echo "tor already downloaded, skipping" + exit 0 + fi + + # git clone + curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import + cd ~/build/dist + git clone https://git.torproject.org/tor.git + cd tor + git tag -v $TOR_TAG + if [ $? -ne 0 ]; then + echo "tor tag doesn't verify" + exit -1 + fi + git checkout $TOR_TAG + - save_cache: + # NOTE: change when upgrading tor + key: build-tor-windows-download-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + paths: + - /home/circleci/build/dist/tor + + - restore_cache: + # NOTE: change when upgrading openssl + key: build-tor-windows-openssl-x32-3.0.3-{{ .Environment.CACHE_VERSION }} + - run: + name: Build openssl (32-bit) + command: | + if [[ -f "/home/circleci/build/prefix-openssl-x32/bin/openssl.exe" ]]; then + echo "openssl (32-bit) already compiled, skipping" + exit 0 + fi # extract - cd ~/build/ tar zxfv ~/build/dist/openssl-$OPENSSL_VERSION.tar.gz -C ~/build/src/ - mv ~/build/src/openssl-$OPENSSL_VERSION ~/build/src/openssl-$OPENSSL_VERSION-x32 - cp -r ~/build/src/openssl-$OPENSSL_VERSION-x32 ~/build/src/openssl-$OPENSSL_VERSION-x64 + mv ~/build/src/openssl-$OPENSSL_VERSION ~/build/src/openssl-x32 # build 32-bit export MINGW=mingw export HOST=i686-w64-mingw32 - cd ~/build/src/openssl-$OPENSSL_VERSION-x32 + cd ~/build/src/openssl-x32 ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/home/circleci/build/prefix-openssl-x32 make -j$(nproc) make install + - save_cache: + # NOTE: change when upgrading openssl + key: build-tor-windows-openssl-x32-3.0.3-{{ .Environment.CACHE_VERSION }} + paths: + - /home/circleci/build/prefix-openssl-x32 + + - restore_cache: + # NOTE: change when upgrading openssl + key: build-tor-windows-openssl-x64-3.0.3-{{ .Environment.CACHE_VERSION }} + - run: + name: Build openssl (64-bit) + command: | + if [[ -f "/home/circleci/build/prefix-openssl-x64/bin/openssl.exe" ]]; then + echo "openssl (64-bit) already compiled, skipping" + exit 0 + fi + + # extract + tar zxfv ~/build/dist/openssl-$OPENSSL_VERSION.tar.gz -C ~/build/src/ + mv ~/build/src/openssl-$OPENSSL_VERSION ~/build/src/openssl-x64 # build 64-bit export MINGW=mingw64 @@ -161,37 +251,22 @@ jobs: make install - save_cache: # NOTE: change when upgrading openssl - key: build-tor-windows-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} + key: build-tor-windows-openssl-x64-3.0.3-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/prefix-openssl-x32 - /home/circleci/build/prefix-openssl-x64 - restore_cache: # NOTE: change when upgrading libevent - key: build-tor-windows-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + key: build-tor-windows-libevent-x32-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - run: - name: Build libevent + name: Build libevent (32-bit) command: | if [[ -d "/home/circleci/build/prefix-libevent-x32/lib" ]]; then - echo "libevent already compiled, skipping" + echo "libevent (32-bit) already compiled, skipping" exit 0 fi - # git clone - curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import - cd ~/build/src - git clone https://github.com/libevent/libevent.git - cd libevent - git tag -v $LIBEVENT_TAG - if [ $? -ne 0 ]; then - echo "libevent tag doesn't verify" - exit -1 - fi - git checkout $LIBEVENT_TAG - - cd ~/build - mv ~/build/src/libevent ~/build/src/libevent-x32 - cp -r ~/build/src/libevent-x32 ~/build/src/libevent-x64 + cp -r ~/build/dist/libevent ~/build/src/libevent-x32 # build 32-bit export MINGW=mingw @@ -201,6 +276,24 @@ jobs: ./configure --host=$HOST --prefix=/home/circleci/build/prefix-libevent-x32 --disable-openssl make -j$(nproc) make install + - save_cache: + # NOTE: change when upgrading libevent + key: build-tor-windows-libevent-x32-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + paths: + - /home/circleci/build/prefix-libevent-x32 + + - restore_cache: + # NOTE: change when upgrading libevent + key: build-tor-windows-libevent-x64-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + - run: + name: Build libevent (64-bit) + command: | + if [[ -d "/home/circleci/build/prefix-libevent-x64/lib" ]]; then + echo "libevent (64-bit) already compiled, skipping" + exit 0 + fi + + cp -r ~/build/dist/libevent ~/build/src/libevent-x64 # build 64-bit export MINGW=mingw64 @@ -212,11 +305,90 @@ jobs: make install - save_cache: # NOTE: change when upgrading libevent - key: build-tor-windows-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + key: build-tor-windows-libevent-x64-2.1.12-stable-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/prefix-libevent-x32 - /home/circleci/build/prefix-libevent-x64 + - restore_cache: + # NOTE: change when upgrading tor + key: build-tor-windows-tor-x32-0.4.7.7-{{ .Environment.CACHE_VERSION }} + - run: + name: Build tor (32-bit) + command: | + if [[ -f "/home/circleci/build/prefix-tor-x32/bin/tor.exe" ]]; then + echo "tor (32-bit) already compiled, skipping" + exit 0 + fi + + cp -r ~/build/dist/tor ~/build/src/tor-x32 + + # build 32-bit + export MINGW=mingw + export HOST=i686-w64-mingw32 + cd ~/build/src/tor-x32 + ./autogen.sh + ./configure --host=$HOST \ + --disable-asciidoc \ + --disable-zstd \ + --disable-lzma \ + --enable-static-libevent \ + --with-libevent-dir=/home/circleci/build/prefix-libevent-x32 \ + --enable-static-openssl \ + --with-openssl-dir=/home/circleci/build/prefix-openssl-x32 \ + --disable-tool-name-check \ + --enable-fatal-warnings \ + --prefix=/home/circleci/build/prefix-tor-x32 + make -j$(nproc) + make install + - save_cache: + # NOTE: change when upgrading tor + key: build-tor-windows-tor-x32-0.4.7.7-{{ .Environment.CACHE_VERSION }} + paths: + - /home/circleci/build/prefix-tor-x32 + + - restore_cache: + # NOTE: change when upgrading tor + key: build-tor-windows-tor-x64-0.4.7.7-{{ .Environment.CACHE_VERSION }} + - run: + name: Build tor (64-bit) + command: | + if [[ -f "/home/circleci/build/prefix-tor-x64/bin/tor.exe" ]]; then + echo "tor (64-bit) already compiled, skipping" + exit 0 + fi + + cp -r ~/build/dist/tor ~/build/src/tor-x64 + + # build 64-bit + export MINGW=mingw64 + export HOST=x86_64-w64-mingw32 + ./autogen.sh + cd ~/build/src/tor-x64 + ./configure --host=$HOST \ + --disable-asciidoc \ + --disable-zstd \ + --disable-lzma \ + --enable-static-libevent \ + --with-libevent-dir=/home/circleci/build/prefix-libevent-x64 \ + --enable-static-openssl \ + --with-openssl-dir=/home/circleci/build/prefix-openssl-x64 \ + --disable-tool-name-check \ + --enable-fatal-warnings \ + --prefix=/home/circleci/build/prefix-tor-x64 + make -j$(nproc) + make install + - save_cache: + # NOTE: change when upgrading tor + key: build-tor-windows-tor-x64-0.4.7.7-{{ .Environment.CACHE_VERSION }} + paths: + - /home/circleci/build/prefix-tor-x64 + + - persist_to_workspace: + root: ~/build/prefix + paths: + - prefix-tor-x32 + - prefix-tor-x64 + - restore_cache: # NOTE: change when upgrading tor key: build-tor-windows-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} From e7cb821f0372fddab10a63c35a9b3671b8e09788 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sat, 4 Jun 2022 21:09:02 -0700 Subject: [PATCH 111/154] Fix openssl-x64 path --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b42b010d..a6f9ef18 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -245,7 +245,7 @@ jobs: # build 64-bit export MINGW=mingw64 export HOST=x86_64-w64-mingw32 - cd ~/build/src/openssl-$OPENSSL_VERSION-x64 + cd ~/build/src/openssl-x64 ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/home/circleci/build/prefix-openssl-x64 make -j$(nproc) make install From bc889a98ed352922da35de1119e559b571034109 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sat, 4 Jun 2022 21:11:18 -0700 Subject: [PATCH 112/154] Fix download-tor cache key --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6f9ef18..5468f64a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -173,7 +173,7 @@ jobs: - restore_cache: # NOTE: change when upgrading tor - key: build-tor-windows-download-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + key: build-tor-windows-download-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - run: name: Download tor command: | @@ -195,7 +195,7 @@ jobs: git checkout $TOR_TAG - save_cache: # NOTE: change when upgrading tor - key: build-tor-windows-download-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} + key: build-tor-windows-download-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} paths: - /home/circleci/build/dist/tor From 5f4e595f444e692644487bcf530bd9d90d4db01d Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sat, 4 Jun 2022 21:26:11 -0700 Subject: [PATCH 113/154] Install build tools from ubuntu jammy --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5468f64a..ba5dc345 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,9 +100,10 @@ jobs: - run: name: Install build dependencies command: | + sudo bash -c 'echo "deb http://archive.ubuntu.com/ubuntu/ jammy main restricted" >> /etc/apt/sources.list' sudo apt-get update - sudo apt-get install -y autoconf automake libtool - sudo apt-get install -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + sudo apt-get install -t jammy -y build-essential autoconf automake libtool + sudo apt-get install -t jammy -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev - run: name: Create folders From 19d2f8d3b9513c474d38b805f822d220866cd594 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sat, 4 Jun 2022 21:29:33 -0700 Subject: [PATCH 114/154] Add universe repo too, to install everything from jammy --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba5dc345..bba49724 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,7 +100,7 @@ jobs: - run: name: Install build dependencies command: | - sudo bash -c 'echo "deb http://archive.ubuntu.com/ubuntu/ jammy main restricted" >> /etc/apt/sources.list' + sudo bash -c 'echo "deb http://archive.ubuntu.com/ubuntu/ jammy main restricted universe" >> /etc/apt/sources.list' sudo apt-get update sudo apt-get install -t jammy -y build-essential autoconf automake libtool sudo apt-get install -t jammy -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev From dedec13d2d781c4c7036f789f7cf3e1340561abf Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sat, 4 Jun 2022 21:42:24 -0700 Subject: [PATCH 115/154] Try with resource_class: medium+, which has more RAM --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index bba49724..e6edf875 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,6 +89,7 @@ jobs: build-tor-windows: docker: - image: cimg/python:3.9 + resource_class: medium+ environment: # NOTE: change when upgrading openssl OPENSSL_VERSION: "3.0.3" From 5e71c90828a9dfe7bbc4dc3d7bef629fe190fc7b Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sat, 4 Jun 2022 21:51:20 -0700 Subject: [PATCH 116/154] Increase RAM --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e6edf875..26882fb0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,7 +89,7 @@ jobs: build-tor-windows: docker: - image: cimg/python:3.9 - resource_class: medium+ + resource_class: large environment: # NOTE: change when upgrading openssl OPENSSL_VERSION: "3.0.3" From 4fce717cdb4f7f2af4c074883fc586681123ef50 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 5 Jun 2022 15:14:41 -0700 Subject: [PATCH 117/154] Try building in Debian instead of Ubuntu --- .circleci/config.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 26882fb0..018ceadd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,8 +88,8 @@ jobs: # Based off of https://github.com/ahf/tor-win32 build-tor-windows: docker: - - image: cimg/python:3.9 - resource_class: large + - image: debian/bullseye + resource_class: medium+ environment: # NOTE: change when upgrading openssl OPENSSL_VERSION: "3.0.3" @@ -101,10 +101,8 @@ jobs: - run: name: Install build dependencies command: | - sudo bash -c 'echo "deb http://archive.ubuntu.com/ubuntu/ jammy main restricted universe" >> /etc/apt/sources.list' - sudo apt-get update - sudo apt-get install -t jammy -y build-essential autoconf automake libtool - sudo apt-get install -t jammy -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + apt-get update + apt-get install -t jammy -y build-essential autoconf automake libtool gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev - run: name: Create folders @@ -339,8 +337,8 @@ jobs: --with-openssl-dir=/home/circleci/build/prefix-openssl-x32 \ --disable-tool-name-check \ --enable-fatal-warnings \ - --prefix=/home/circleci/build/prefix-tor-x32 - make -j$(nproc) + --prefix=/home/circleci/build/prefix-tor-x32 && \ + make -j$(nproc) && \ make install - save_cache: # NOTE: change when upgrading tor @@ -376,8 +374,8 @@ jobs: --with-openssl-dir=/home/circleci/build/prefix-openssl-x64 \ --disable-tool-name-check \ --enable-fatal-warnings \ - --prefix=/home/circleci/build/prefix-tor-x64 - make -j$(nproc) + --prefix=/home/circleci/build/prefix-tor-x64 && \ + make -j$(nproc) && \ make install - save_cache: # NOTE: change when upgrading tor From 19abe55d9dc2af04007b75005b13750271e642dc Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 5 Jun 2022 15:47:58 -0700 Subject: [PATCH 118/154] debian:bullseye --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 018ceadd..9b891e9b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,7 +88,7 @@ jobs: # Based off of https://github.com/ahf/tor-win32 build-tor-windows: docker: - - image: debian/bullseye + - image: debian:bullseye resource_class: medium+ environment: # NOTE: change when upgrading openssl From 6555c00456316cc59c52833d968112ff25b16dde Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 5 Jun 2022 15:50:17 -0700 Subject: [PATCH 119/154] Without jammy --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b891e9b..626cfa19 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,7 +102,7 @@ jobs: name: Install build dependencies command: | apt-get update - apt-get install -t jammy -y build-essential autoconf automake libtool gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + apt-get install -y build-essential autoconf automake libtool gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev - run: name: Create folders From 55554f3061a82fe6e6c817aea0539cbb6a97cd83 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 5 Jun 2022 15:52:15 -0700 Subject: [PATCH 120/154] Install wget --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 626cfa19..55da7ab0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,7 +102,7 @@ jobs: name: Install build dependencies command: | apt-get update - apt-get install -y build-essential autoconf automake libtool gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + apt-get install -y wget build-essential autoconf automake libtool gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev - run: name: Create folders From 7271deeef9e34b7be7801e00d9e0970418df8444 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 5 Jun 2022 15:54:05 -0700 Subject: [PATCH 121/154] Install curl --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55da7ab0..7665124a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,7 +102,7 @@ jobs: name: Install build dependencies command: | apt-get update - apt-get install -y wget build-essential autoconf automake libtool gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + apt-get install -y wget curl build-essential autoconf automake libtool gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev - run: name: Create folders From 6fb21066820a6a54d146d325e7cb9e96eab44cb4 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 5 Jun 2022 15:55:57 -0700 Subject: [PATCH 122/154] Install git --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7665124a..c4fa455c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,7 +102,7 @@ jobs: name: Install build dependencies command: | apt-get update - apt-get install -y wget curl build-essential autoconf automake libtool gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + apt-get install -y git wget curl build-essential autoconf automake libtool gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev - run: name: Create folders From e8e8e3c30a2df29daf119c69ccc1fe3c60e399ff Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 5 Jun 2022 16:38:01 -0700 Subject: [PATCH 123/154] Change paths since we are building as root in debian now --- .circleci/config.yml | 139 ++++++++++++++++++++----------------------- 1 file changed, 66 insertions(+), 73 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4fa455c..6a1c67f4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -107,15 +107,8 @@ jobs: - run: name: Create folders command: | - mkdir ~/build - mkdir ~/build/src - mkdir ~/build/dist - mkdir ~/build/prefix-openssl-x32 - mkdir ~/build/prefix-openssl-x64 - mkdir ~/build/prefix-libevent-x32 - mkdir ~/build/prefix-libevent-x64 - mkdir ~/build/prefix-tor-x32 - mkdir ~/build/prefix-tor-x64 + mkdir -p /build + mkdir -p /build/{src,dist,prefix-openssl-x32,prefix-openssl-x64,prefix-libevent-x32,prefix-libevent-x64,prefix-tor-x32,prefix-tor-x64} - restore_cache: # NOTE: change when upgrading openssl @@ -123,12 +116,12 @@ jobs: - run: name: Download openssl command: | - if [[ -f "/home/circleci/build/dist/openssl-$OPENSSL_VERSION.tar.gz" ]]; then + if [[ -f "/build/dist/openssl-$OPENSSL_VERSION.tar.gz" ]]; then echo "openssl already downloaded, skipping" exit 0 fi - cd ~/build/dist + cd /build/dist # download wget https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz @@ -141,7 +134,7 @@ jobs: # NOTE: change when upgrading openssl key: build-tor-windows-download-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/dist/openssl-3.0.3.tar.gz + - /build/dist/openssl-3.0.3.tar.gz - restore_cache: # NOTE: change when upgrading libevent @@ -149,14 +142,14 @@ jobs: - run: name: Download libevent command: | - if [[ -d "/home/circleci/build/dist/libevent" ]]; then + if [[ -d "/build/dist/libevent" ]]; then echo "livevent already downloaded, skipping" exit 0 fi # git clone curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import - cd ~/build/dist + cd /build/dist git clone https://github.com/libevent/libevent.git cd libevent git tag -v $LIBEVENT_TAG @@ -169,7 +162,7 @@ jobs: # NOTE: change when upgrading libevent key: build-tor-windows-download-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/dist/libevent + - /build/dist/libevent - restore_cache: # NOTE: change when upgrading tor @@ -177,14 +170,14 @@ jobs: - run: name: Download tor command: | - if [[ -d "/home/circleci/build/dist/tor" ]]; then + if [[ -d "/build/dist/tor" ]]; then echo "tor already downloaded, skipping" exit 0 fi # git clone curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import - cd ~/build/dist + cd /build/dist git clone https://git.torproject.org/tor.git cd tor git tag -v $TOR_TAG @@ -197,7 +190,7 @@ jobs: # NOTE: change when upgrading tor key: build-tor-windows-download-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/dist/tor + - /build/dist/tor - restore_cache: # NOTE: change when upgrading openssl @@ -205,27 +198,27 @@ jobs: - run: name: Build openssl (32-bit) command: | - if [[ -f "/home/circleci/build/prefix-openssl-x32/bin/openssl.exe" ]]; then + if [[ -f "/build/prefix-openssl-x32/bin/openssl.exe" ]]; then echo "openssl (32-bit) already compiled, skipping" exit 0 fi # extract - tar zxfv ~/build/dist/openssl-$OPENSSL_VERSION.tar.gz -C ~/build/src/ - mv ~/build/src/openssl-$OPENSSL_VERSION ~/build/src/openssl-x32 + tar zxfv /build/dist/openssl-$OPENSSL_VERSION.tar.gz -C /build/src/ + mv /build/src/openssl-$OPENSSL_VERSION /build/src/openssl-x32 # build 32-bit export MINGW=mingw export HOST=i686-w64-mingw32 - cd ~/build/src/openssl-x32 - ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/home/circleci/build/prefix-openssl-x32 + cd /build/src/openssl-x32 + ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/build/prefix-openssl-x32 make -j$(nproc) make install - save_cache: # NOTE: change when upgrading openssl key: build-tor-windows-openssl-x32-3.0.3-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/prefix-openssl-x32 + - /build/prefix-openssl-x32 - restore_cache: # NOTE: change when upgrading openssl @@ -233,27 +226,27 @@ jobs: - run: name: Build openssl (64-bit) command: | - if [[ -f "/home/circleci/build/prefix-openssl-x64/bin/openssl.exe" ]]; then + if [[ -f "/build/prefix-openssl-x64/bin/openssl.exe" ]]; then echo "openssl (64-bit) already compiled, skipping" exit 0 fi # extract - tar zxfv ~/build/dist/openssl-$OPENSSL_VERSION.tar.gz -C ~/build/src/ - mv ~/build/src/openssl-$OPENSSL_VERSION ~/build/src/openssl-x64 + tar zxfv /build/dist/openssl-$OPENSSL_VERSION.tar.gz -C /build/src/ + mv /build/src/openssl-$OPENSSL_VERSION /build/src/openssl-x64 # build 64-bit export MINGW=mingw64 export HOST=x86_64-w64-mingw32 - cd ~/build/src/openssl-x64 - ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/home/circleci/build/prefix-openssl-x64 + cd /build/src/openssl-x64 + ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/build/prefix-openssl-x64 make -j$(nproc) make install - save_cache: # NOTE: change when upgrading openssl key: build-tor-windows-openssl-x64-3.0.3-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/prefix-openssl-x64 + - /build/prefix-openssl-x64 - restore_cache: # NOTE: change when upgrading libevent @@ -261,26 +254,26 @@ jobs: - run: name: Build libevent (32-bit) command: | - if [[ -d "/home/circleci/build/prefix-libevent-x32/lib" ]]; then + if [[ -d "/build/prefix-libevent-x32/lib" ]]; then echo "libevent (32-bit) already compiled, skipping" exit 0 fi - cp -r ~/build/dist/libevent ~/build/src/libevent-x32 + cp -r /build/dist/libevent /build/src/libevent-x32 # build 32-bit export MINGW=mingw export HOST=i686-w64-mingw32 - cd ~/build/src/libevent-x32 + cd /build/src/libevent-x32 ./autogen.sh - ./configure --host=$HOST --prefix=/home/circleci/build/prefix-libevent-x32 --disable-openssl + ./configure --host=$HOST --prefix=/build/prefix-libevent-x32 --disable-openssl make -j$(nproc) make install - save_cache: # NOTE: change when upgrading libevent key: build-tor-windows-libevent-x32-2.1.12-stable-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/prefix-libevent-x32 + - /build/prefix-libevent-x32 - restore_cache: # NOTE: change when upgrading libevent @@ -288,26 +281,26 @@ jobs: - run: name: Build libevent (64-bit) command: | - if [[ -d "/home/circleci/build/prefix-libevent-x64/lib" ]]; then + if [[ -d "/build/prefix-libevent-x64/lib" ]]; then echo "libevent (64-bit) already compiled, skipping" exit 0 fi - cp -r ~/build/dist/libevent ~/build/src/libevent-x64 + cp -r /build/dist/libevent /build/src/libevent-x64 # build 64-bit export MINGW=mingw64 export HOST=x86_64-w64-mingw32 - cd ~/build/src/libevent-x64 + cd /build/src/libevent-x64 ./autogen.sh - ./configure --host=$HOST --prefix=/home/circleci/build/prefix-libevent-x64 --disable-openssl + ./configure --host=$HOST --prefix=/build/prefix-libevent-x64 --disable-openssl make -j$(nproc) make install - save_cache: # NOTE: change when upgrading libevent key: build-tor-windows-libevent-x64-2.1.12-stable-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/prefix-libevent-x64 + - /build/prefix-libevent-x64 - restore_cache: # NOTE: change when upgrading tor @@ -315,36 +308,36 @@ jobs: - run: name: Build tor (32-bit) command: | - if [[ -f "/home/circleci/build/prefix-tor-x32/bin/tor.exe" ]]; then + if [[ -f "/build/prefix-tor-x32/bin/tor.exe" ]]; then echo "tor (32-bit) already compiled, skipping" exit 0 fi - cp -r ~/build/dist/tor ~/build/src/tor-x32 + cp -r /build/dist/tor /build/src/tor-x32 # build 32-bit export MINGW=mingw export HOST=i686-w64-mingw32 - cd ~/build/src/tor-x32 + cd /build/src/tor-x32 ./autogen.sh ./configure --host=$HOST \ --disable-asciidoc \ --disable-zstd \ --disable-lzma \ --enable-static-libevent \ - --with-libevent-dir=/home/circleci/build/prefix-libevent-x32 \ + --with-libevent-dir=/build/prefix-libevent-x32 \ --enable-static-openssl \ - --with-openssl-dir=/home/circleci/build/prefix-openssl-x32 \ + --with-openssl-dir=/build/prefix-openssl-x32 \ --disable-tool-name-check \ --enable-fatal-warnings \ - --prefix=/home/circleci/build/prefix-tor-x32 && \ + --prefix=/build/prefix-tor-x32 && \ make -j$(nproc) && \ make install - save_cache: # NOTE: change when upgrading tor key: build-tor-windows-tor-x32-0.4.7.7-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/prefix-tor-x32 + - /build/prefix-tor-x32 - restore_cache: # NOTE: change when upgrading tor @@ -352,39 +345,39 @@ jobs: - run: name: Build tor (64-bit) command: | - if [[ -f "/home/circleci/build/prefix-tor-x64/bin/tor.exe" ]]; then + if [[ -f "/build/prefix-tor-x64/bin/tor.exe" ]]; then echo "tor (64-bit) already compiled, skipping" exit 0 fi - cp -r ~/build/dist/tor ~/build/src/tor-x64 + cp -r /build/dist/tor /build/src/tor-x64 # build 64-bit export MINGW=mingw64 export HOST=x86_64-w64-mingw32 ./autogen.sh - cd ~/build/src/tor-x64 + cd /build/src/tor-x64 ./configure --host=$HOST \ --disable-asciidoc \ --disable-zstd \ --disable-lzma \ --enable-static-libevent \ - --with-libevent-dir=/home/circleci/build/prefix-libevent-x64 \ + --with-libevent-dir=/build/prefix-libevent-x64 \ --enable-static-openssl \ - --with-openssl-dir=/home/circleci/build/prefix-openssl-x64 \ + --with-openssl-dir=/build/prefix-openssl-x64 \ --disable-tool-name-check \ --enable-fatal-warnings \ - --prefix=/home/circleci/build/prefix-tor-x64 && \ + --prefix=/build/prefix-tor-x64 && \ make -j$(nproc) && \ make install - save_cache: # NOTE: change when upgrading tor key: build-tor-windows-tor-x64-0.4.7.7-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/prefix-tor-x64 + - /build/prefix-tor-x64 - persist_to_workspace: - root: ~/build/prefix + root: /build/prefix paths: - prefix-tor-x32 - prefix-tor-x64 @@ -395,14 +388,14 @@ jobs: - run: name: Build tor command: | - if [[ -f "/home/circleci/build/prefix-tor-x32/bin/tor.exe" ]]; then + if [[ -f "/build/prefix-tor-x32/bin/tor.exe" ]]; then echo "tor already compiled, skipping" exit 0 fi # git clone curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import - cd ~/build/src + cd /build/src git clone https://git.torproject.org/tor.git cd tor git tag -v $TOR_TAG @@ -412,26 +405,26 @@ jobs: fi git checkout $TOR_TAG - cd ~/build - mv ~/build/src/tor ~/build/src/tor-x32 - cp -r ~/build/src/tor-x32 ~/build/src/tor-x64 + cd /build + mv /build/src/tor /build/src/tor-x32 + cp -r /build/src/tor-x32 /build/src/tor-x64 # build 32-bit export MINGW=mingw export HOST=i686-w64-mingw32 - cd ~/build/src/tor-x32 + cd /build/src/tor-x32 ./autogen.sh ./configure --host=$HOST \ --disable-asciidoc \ --disable-zstd \ --disable-lzma \ --enable-static-libevent \ - --with-libevent-dir=/home/circleci/build/prefix-libevent-x32 \ + --with-libevent-dir=/build/prefix-libevent-x32 \ --enable-static-openssl \ - --with-openssl-dir=/home/circleci/build/prefix-openssl-x32 \ + --with-openssl-dir=/build/prefix-openssl-x32 \ --disable-tool-name-check \ --enable-fatal-warnings \ - --prefix=/home/circleci/build/prefix-tor-x32 + --prefix=/build/prefix-tor-x32 make -j$(nproc) make install @@ -439,32 +432,32 @@ jobs: export MINGW=mingw64 export HOST=x86_64-w64-mingw32 ./autogen.sh - cd ~/build/src/tor-x64 + cd /build/src/tor-x64 ./configure --host=$HOST \ --disable-asciidoc \ --disable-zstd \ --disable-lzma \ --enable-static-libevent \ - --with-libevent-dir=/home/circleci/build/prefix-libevent-x64 \ + --with-libevent-dir=/build/prefix-libevent-x64 \ --enable-static-openssl \ - --with-openssl-dir=/home/circleci/build/prefix-openssl-x64 \ + --with-openssl-dir=/build/prefix-openssl-x64 \ --disable-tool-name-check \ --enable-fatal-warnings \ - --prefix=/home/circleci/build/prefix-tor-x64 + --prefix=/build/prefix-tor-x64 make -j$(nproc) make install - save_cache: # NOTE: change when upgrading tor key: build-tor-windows-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} paths: - - /home/circleci/build/prefix-tor-x32 - - /home/circleci/build/prefix-tor-x64 + - /build/prefix-tor-x32 + - /build/prefix-tor-x64 - persist_to_workspace: - root: ~/build/prefix + root: / paths: - - prefix-tor-x32 - - prefix-tor-x64 + - /root/build/prefix-tor-x32 + - /root/build/prefix-tor-x64 build-win64: executor: From a04715b8057da838967b57398a7d254f20e2ffaa Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 5 Jun 2022 16:40:29 -0700 Subject: [PATCH 124/154] Fix persiste_to_workspace paths --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a1c67f4..44b26dbe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -454,10 +454,10 @@ jobs: - /build/prefix-tor-x64 - persist_to_workspace: - root: / + root: /build paths: - - /root/build/prefix-tor-x32 - - /root/build/prefix-tor-x64 + - prefix-tor-x32 + - prefix-tor-x64 build-win64: executor: From e2a601ad751c6479f19104260ca667a8845821bb Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 16 Jun 2022 20:03:23 -0700 Subject: [PATCH 125/154] Rip out code to build tor, and just get it from Tor Browser again --- .circleci/config.yml | 598 +++---------------------------------------- 1 file changed, 32 insertions(+), 566 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 44b26dbe..c8a71409 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,18 +8,15 @@ workflows: ci: jobs: - test-cli - # - test-gui - - build-tor-windows: + - test-gui + - build-win64: requires: - test-cli - # - build-win64: - # requires: - # - test-cli - # # - test-gui - # - build-win32: - # requires: - # - test-cli - # # - test-gui + # - test-gui + - build-win32: + requires: + - test-cli + # - test-gui # - build-macos: # requires: # - test-cli @@ -85,389 +82,11 @@ jobs: cd ~/project/desktop QT_DEBUG_PLUGINS=1 xvfb-run poetry run pytest -v ./tests/test_gui_*.py - # Based off of https://github.com/ahf/tor-win32 - build-tor-windows: - docker: - - image: debian:bullseye - resource_class: medium+ - environment: - # NOTE: change when upgrading openssl - OPENSSL_VERSION: "3.0.3" - # NOTE: change when upgrading libevent - LIBEVENT_TAG: release-2.1.12-stable - # NOTE: change when upgrading tor - TOR_TAG: tor-0.4.7.7 - steps: - - run: - name: Install build dependencies - command: | - apt-get update - apt-get install -y git wget curl build-essential autoconf automake libtool gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev - - - run: - name: Create folders - command: | - mkdir -p /build - mkdir -p /build/{src,dist,prefix-openssl-x32,prefix-openssl-x64,prefix-libevent-x32,prefix-libevent-x64,prefix-tor-x32,prefix-tor-x64} - - - restore_cache: - # NOTE: change when upgrading openssl - key: build-tor-windows-download-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} - - run: - name: Download openssl - command: | - if [[ -f "/build/dist/openssl-$OPENSSL_VERSION.tar.gz" ]]; then - echo "openssl already downloaded, skipping" - exit 0 - fi - - cd /build/dist - - # download - wget https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz - echo "ee0078adcef1de5f003c62c80cc96527721609c6f3bb42b7795df31f8b558c0b openssl-$OPENSSL_VERSION.tar.gz" | sha256sum --check --status - if [ $? -ne 0 ]; then - echo "openssl checksum failed" - exit -1 - fi - - save_cache: - # NOTE: change when upgrading openssl - key: build-tor-windows-download-openssl-3.0.3-{{ .Environment.CACHE_VERSION }} - paths: - - /build/dist/openssl-3.0.3.tar.gz - - - restore_cache: - # NOTE: change when upgrading libevent - key: build-tor-windows-download-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - - run: - name: Download libevent - command: | - if [[ -d "/build/dist/libevent" ]]; then - echo "livevent already downloaded, skipping" - exit 0 - fi - - # git clone - curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import - cd /build/dist - git clone https://github.com/libevent/libevent.git - cd libevent - git tag -v $LIBEVENT_TAG - if [ $? -ne 0 ]; then - echo "libevent tag doesn't verify" - exit -1 - fi - git checkout $LIBEVENT_TAG - - save_cache: - # NOTE: change when upgrading libevent - key: build-tor-windows-download-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - paths: - - /build/dist/libevent - - - restore_cache: - # NOTE: change when upgrading tor - key: build-tor-windows-download-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - - run: - name: Download tor - command: | - if [[ -d "/build/dist/tor" ]]; then - echo "tor already downloaded, skipping" - exit 0 - fi - - # git clone - curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import - cd /build/dist - git clone https://git.torproject.org/tor.git - cd tor - git tag -v $TOR_TAG - if [ $? -ne 0 ]; then - echo "tor tag doesn't verify" - exit -1 - fi - git checkout $TOR_TAG - - save_cache: - # NOTE: change when upgrading tor - key: build-tor-windows-download-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - paths: - - /build/dist/tor - - - restore_cache: - # NOTE: change when upgrading openssl - key: build-tor-windows-openssl-x32-3.0.3-{{ .Environment.CACHE_VERSION }} - - run: - name: Build openssl (32-bit) - command: | - if [[ -f "/build/prefix-openssl-x32/bin/openssl.exe" ]]; then - echo "openssl (32-bit) already compiled, skipping" - exit 0 - fi - - # extract - tar zxfv /build/dist/openssl-$OPENSSL_VERSION.tar.gz -C /build/src/ - mv /build/src/openssl-$OPENSSL_VERSION /build/src/openssl-x32 - - # build 32-bit - export MINGW=mingw - export HOST=i686-w64-mingw32 - cd /build/src/openssl-x32 - ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/build/prefix-openssl-x32 - make -j$(nproc) - make install - - save_cache: - # NOTE: change when upgrading openssl - key: build-tor-windows-openssl-x32-3.0.3-{{ .Environment.CACHE_VERSION }} - paths: - - /build/prefix-openssl-x32 - - - restore_cache: - # NOTE: change when upgrading openssl - key: build-tor-windows-openssl-x64-3.0.3-{{ .Environment.CACHE_VERSION }} - - run: - name: Build openssl (64-bit) - command: | - if [[ -f "/build/prefix-openssl-x64/bin/openssl.exe" ]]; then - echo "openssl (64-bit) already compiled, skipping" - exit 0 - fi - - # extract - tar zxfv /build/dist/openssl-$OPENSSL_VERSION.tar.gz -C /build/src/ - mv /build/src/openssl-$OPENSSL_VERSION /build/src/openssl-x64 - - # build 64-bit - export MINGW=mingw64 - export HOST=x86_64-w64-mingw32 - cd /build/src/openssl-x64 - ./Configure $MINGW shared --cross-compile-prefix=$HOST- --prefix=/build/prefix-openssl-x64 - make -j$(nproc) - make install - - save_cache: - # NOTE: change when upgrading openssl - key: build-tor-windows-openssl-x64-3.0.3-{{ .Environment.CACHE_VERSION }} - paths: - - /build/prefix-openssl-x64 - - - restore_cache: - # NOTE: change when upgrading libevent - key: build-tor-windows-libevent-x32-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - - run: - name: Build libevent (32-bit) - command: | - if [[ -d "/build/prefix-libevent-x32/lib" ]]; then - echo "libevent (32-bit) already compiled, skipping" - exit 0 - fi - - cp -r /build/dist/libevent /build/src/libevent-x32 - - # build 32-bit - export MINGW=mingw - export HOST=i686-w64-mingw32 - cd /build/src/libevent-x32 - ./autogen.sh - ./configure --host=$HOST --prefix=/build/prefix-libevent-x32 --disable-openssl - make -j$(nproc) - make install - - save_cache: - # NOTE: change when upgrading libevent - key: build-tor-windows-libevent-x32-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - paths: - - /build/prefix-libevent-x32 - - - restore_cache: - # NOTE: change when upgrading libevent - key: build-tor-windows-libevent-x64-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - - run: - name: Build libevent (64-bit) - command: | - if [[ -d "/build/prefix-libevent-x64/lib" ]]; then - echo "libevent (64-bit) already compiled, skipping" - exit 0 - fi - - cp -r /build/dist/libevent /build/src/libevent-x64 - - # build 64-bit - export MINGW=mingw64 - export HOST=x86_64-w64-mingw32 - cd /build/src/libevent-x64 - ./autogen.sh - ./configure --host=$HOST --prefix=/build/prefix-libevent-x64 --disable-openssl - make -j$(nproc) - make install - - save_cache: - # NOTE: change when upgrading libevent - key: build-tor-windows-libevent-x64-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - paths: - - /build/prefix-libevent-x64 - - - restore_cache: - # NOTE: change when upgrading tor - key: build-tor-windows-tor-x32-0.4.7.7-{{ .Environment.CACHE_VERSION }} - - run: - name: Build tor (32-bit) - command: | - if [[ -f "/build/prefix-tor-x32/bin/tor.exe" ]]; then - echo "tor (32-bit) already compiled, skipping" - exit 0 - fi - - cp -r /build/dist/tor /build/src/tor-x32 - - # build 32-bit - export MINGW=mingw - export HOST=i686-w64-mingw32 - cd /build/src/tor-x32 - ./autogen.sh - ./configure --host=$HOST \ - --disable-asciidoc \ - --disable-zstd \ - --disable-lzma \ - --enable-static-libevent \ - --with-libevent-dir=/build/prefix-libevent-x32 \ - --enable-static-openssl \ - --with-openssl-dir=/build/prefix-openssl-x32 \ - --disable-tool-name-check \ - --enable-fatal-warnings \ - --prefix=/build/prefix-tor-x32 && \ - make -j$(nproc) && \ - make install - - save_cache: - # NOTE: change when upgrading tor - key: build-tor-windows-tor-x32-0.4.7.7-{{ .Environment.CACHE_VERSION }} - paths: - - /build/prefix-tor-x32 - - - restore_cache: - # NOTE: change when upgrading tor - key: build-tor-windows-tor-x64-0.4.7.7-{{ .Environment.CACHE_VERSION }} - - run: - name: Build tor (64-bit) - command: | - if [[ -f "/build/prefix-tor-x64/bin/tor.exe" ]]; then - echo "tor (64-bit) already compiled, skipping" - exit 0 - fi - - cp -r /build/dist/tor /build/src/tor-x64 - - # build 64-bit - export MINGW=mingw64 - export HOST=x86_64-w64-mingw32 - ./autogen.sh - cd /build/src/tor-x64 - ./configure --host=$HOST \ - --disable-asciidoc \ - --disable-zstd \ - --disable-lzma \ - --enable-static-libevent \ - --with-libevent-dir=/build/prefix-libevent-x64 \ - --enable-static-openssl \ - --with-openssl-dir=/build/prefix-openssl-x64 \ - --disable-tool-name-check \ - --enable-fatal-warnings \ - --prefix=/build/prefix-tor-x64 && \ - make -j$(nproc) && \ - make install - - save_cache: - # NOTE: change when upgrading tor - key: build-tor-windows-tor-x64-0.4.7.7-{{ .Environment.CACHE_VERSION }} - paths: - - /build/prefix-tor-x64 - - - persist_to_workspace: - root: /build/prefix - paths: - - prefix-tor-x32 - - prefix-tor-x64 - - - restore_cache: - # NOTE: change when upgrading tor - key: build-tor-windows-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - - run: - name: Build tor - command: | - if [[ -f "/build/prefix-tor-x32/bin/tor.exe" ]]; then - echo "tor already compiled, skipping" - exit 0 - fi - - # git clone - curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import - cd /build/src - git clone https://git.torproject.org/tor.git - cd tor - git tag -v $TOR_TAG - if [ $? -ne 0 ]; then - echo "tor tag doesn't verify" - exit -1 - fi - git checkout $TOR_TAG - - cd /build - mv /build/src/tor /build/src/tor-x32 - cp -r /build/src/tor-x32 /build/src/tor-x64 - - # build 32-bit - export MINGW=mingw - export HOST=i686-w64-mingw32 - cd /build/src/tor-x32 - ./autogen.sh - ./configure --host=$HOST \ - --disable-asciidoc \ - --disable-zstd \ - --disable-lzma \ - --enable-static-libevent \ - --with-libevent-dir=/build/prefix-libevent-x32 \ - --enable-static-openssl \ - --with-openssl-dir=/build/prefix-openssl-x32 \ - --disable-tool-name-check \ - --enable-fatal-warnings \ - --prefix=/build/prefix-tor-x32 - make -j$(nproc) - make install - - # build 64-bit - export MINGW=mingw64 - export HOST=x86_64-w64-mingw32 - ./autogen.sh - cd /build/src/tor-x64 - ./configure --host=$HOST \ - --disable-asciidoc \ - --disable-zstd \ - --disable-lzma \ - --enable-static-libevent \ - --with-libevent-dir=/build/prefix-libevent-x64 \ - --enable-static-openssl \ - --with-openssl-dir=/build/prefix-openssl-x64 \ - --disable-tool-name-check \ - --enable-fatal-warnings \ - --prefix=/build/prefix-tor-x64 - make -j$(nproc) - make install - - save_cache: - # NOTE: change when upgrading tor - key: build-tor-windows-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - paths: - - /build/prefix-tor-x32 - - /build/prefix-tor-x64 - - - persist_to_workspace: - root: /build - paths: - - prefix-tor-x32 - - prefix-tor-x64 - build-win64: executor: name: win/default shell: powershell.exe environment: - # NOTE: change when upgrading libevent - LIBEVENT_TAG: release-2.1.12-stable - # NOTE: change when upgrading tor - TOR_TAG: tor-0.4.7.7 # NOTE: change when upgrading obfs4proxy OBFS4PROXY_TAG: obfs4proxy-0.0.13 # NOTE: change when upgrading snowflake @@ -477,115 +96,7 @@ jobs: steps: - checkout - - run: - name: Install cygwin and required packages - shell: bash.exe --login -eo pipefail - command: | - curl -o setup-x86_64.exe https://cygwin.com/setup-x86_64.exe - curl -o setup-x86_64.exe.sig https://cygwin.com/setup-x86_64.exe.sig - curl -o pubring.asc https://cygwin.com/key/pubring.asc - - gpg --import pubring.asc - gpg --verify setup-x86_64.exe.sig setup-x86_64.exe - if [ $? -ne 0 ]; then - echo "cygwin signature doesn't verify" - exit -1 - fi - - ./setup-x86_64.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ \ - -P libssl-devel,zlib-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg - - - restore_cache: - # NOTE: change when upgrading libevent - key: build-win64-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - - run: - name: Build libevent - shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail - command: | - if [[ -d "/cygdrive/c/Users/circleci/build/libevent" ]]; then - echo "libevent already compiled, skipping" - exit 0 - fi - - mkdir -p /cygdrive/c/Users/circleci/src - mkdir -p /cygdrive/c/Users/circleci/build - - # Get source - curl "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9e3ac83a27974b84d1b3401db86086848ef8686d" | gpg --import - cd /cygdrive/c/Users/circleci/src - git clone https://github.com/libevent/libevent.git - cd libevent - git tag -v $LIBEVENT_TAG - if [ $? -ne 0 ]; then - echo "libevent tag doesn't verify" - exit -1 - fi - git checkout $LIBEVENT_TAG - - # Build - ./autogen.sh - ./configure \ - --prefix=/cygdrive/c/Users/circleci/build/libevent \ - --with-pic - make -j$(nproc) - make install - - save_cache: - # NOTE: change when upgrading libevent - key: build-win64-libevent-2.1.12-stable-{{ .Environment.CACHE_VERSION }} - paths: - - C:\Users\circleci\build\libevent - - - restore_cache: - # NOTE: change when upgrading tor - key: build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - - run: - name: Build tor - shell: C:\\cygwin64\\bin\\bash.exe --login -eo pipefail - command: | - if [[ -d "/cygdrive/c/Users/circleci/build/tor" ]]; then - echo "tor already compiled, skipping" - exit 0 - fi - - mkdir -p /cygdrive/c/Users/circleci/src - mkdir -p /cygdrive/c/Users/circleci/build - - # Get source - curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import - cd /cygdrive/c/Users/circleci/src - git clone https://git.torproject.org/tor.git - cd tor - git tag -v $TOR_TAG - if [ $? -ne 0 ]; then - echo "tor tag doesn't verify" - exit -1 - fi - git checkout $TOR_TAG - - # Build - ./autogen.sh - ./configure \ - --prefix=/cygdrive/c/Users/circleci/build/tor \ - --disable-asciidoc \ - --enable-static-libevent \ - --with-libevent-dir=/cygdrive/c/Users/circleci/build/libevent - make -j$(nproc) - make install - - save_cache: - # NOTE: change when upgrading tor - key: build-win64-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - paths: - - C:\Users\circleci\build\tor - - - run: - name: Copy tor binaries into project - command: | - New-Item -ItemType Directory -Force -Path C:\Users\circleci\project\desktop\onionshare\resources\tor - Copy-Item -Path C:\Users\circleci\build\tor\bin\tor.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor - Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor - Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip6 -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor - - - restore_cache: + - restore_cache: # NOTE: change when upgrading obfs4proxy key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} - run: @@ -679,6 +190,18 @@ jobs: paths: - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs + - restore_cache: + key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Get tor binaries from Tor Browser (32-bit) + command: | + cd desktop + poetry run .\scripts\get-tor.py + - save_cache: + key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe + - run: name: Build OnionShare command: | @@ -700,8 +223,6 @@ jobs: name: win/default shell: powershell.exe environment: - # NOTE: change when upgrading tor - TOR_TAG: tor-0.4.7.7 # NOTE: change when upgrading obfs4proxy OBFS4PROXY_TAG: obfs4proxy-0.0.13 # NOTE: change when upgrading snowflake @@ -711,73 +232,6 @@ jobs: steps: - checkout - - run: - name: Install cygwin (32-bit) and required packages - shell: bash.exe --login -eo pipefail - command: | - curl -o setup-x86.exe https://cygwin.com/setup-x86.exe - curl -o setup-x86.exe.sig https://cygwin.com/setup-x86.exe.sig - curl -o pubring.asc https://cygwin.com/key/pubring.asc - - gpg --import pubring.asc - gpg --verify setup-x86.exe.sig setup-x86.exe - if [ $? -ne 0 ]; then - echo "cygwin signature doesn't verify" - exit -1 - fi - - ./setup-x86.exe -q -s https://mirrors.kernel.org/sourceware/cygwin/ \ - -P libssl-devel,libevent-devel,zlib-devel,git,gcc-core,make,autoconf,automake,libtool,gnupg - - - restore_cache: - # NOTE: change when upgrading tor - key: build-win32-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - - run: - name: Build tor - shell: C:\\cygwin\\bin\\bash.exe --login -eo pipefail - command: | - if [[ -d "/cygdrive/c/Users/circleci/build/tor" ]]; then - echo "tor already compiled, skipping" - exit 0 - fi - - mkdir -p /cygdrive/c/Users/circleci/src - mkdir -p /cygdrive/c/Users/circleci/build - - # Get source - curl https://keys.openpgp.org/vks/v1/by-fingerprint/B74417EDDF22AC9F9E90F49142E86A2A11F48D36 | gpg --import - cd /cygdrive/c/Users/circleci/src - git clone https://git.torproject.org/tor.git - cd tor - git tag -v $TOR_TAG - if [ $? -ne 0 ]; then - echo "tor tag doesn't verify" - exit -1 - fi - git checkout $TOR_TAG - - # Build - ./autogen.sh - ./configure \ - --prefix=/opt/tor-build \ - --disable-asciidoc - make -j$(nproc) - make install - cp -r /opt/tor-build /cygdrive/c/Users/circleci/build/tor - - save_cache: - # NOTE: change when upgrading tor - key: build-win32-tor-0.4.7.7-{{ .Environment.CACHE_VERSION }} - paths: - - C:\Users\circleci\build\tor - - - run: - name: Copy tor binaries into project - command: | - New-Item -ItemType Directory -Force -Path C:\Users\circleci\project\desktop\onionshare\resources\tor - Copy-Item -Path C:\Users\circleci\build\tor\bin\tor.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor - Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor - Copy-Item -Path C:\Users\circleci\build\tor\share\tor\geoip6 -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor - - run: name: Install golang (32-bit) command: | @@ -879,6 +333,18 @@ jobs: paths: - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs + - restore_cache: + key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Get tor binaries from Tor Browser (32-bit) + command: | + cd desktop + poetry run .\scripts\get-tor.py + - save_cache: + key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe + - run: name: Build OnionShare command: | From 5c24453ddc05d2b87db9af62823e61cd0285159a Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 16 Jun 2022 20:04:42 -0700 Subject: [PATCH 126/154] Syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c8a71409..1fe526d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,7 +96,7 @@ jobs: steps: - checkout - - restore_cache: + - restore_cache: # NOTE: change when upgrading obfs4proxy key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} - run: From 549b0eac72f041059058e51b1b91126e442ef12e Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 16 Jun 2022 21:05:11 -0700 Subject: [PATCH 127/154] Update get-tor.py to use Tor Browser 11.0.4, and improve it. Make sure we run get-tor.py in CI before compiling pluggable transports, so the correct folder will exist --- .circleci/config.yml | 136 +++++++++--------- .../resources/torrc_template-obfs4 | 1 - desktop/onionshare/gui_common.py | 16 ++- desktop/scripts/get-tor.py | 60 ++++---- 4 files changed, 112 insertions(+), 101 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1fe526d1..15d059c7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,6 +96,39 @@ jobs: steps: - checkout + - run: + name: Install Python 3.9.12 + command: | + choco install python3 --version=3.9.12 + + - run: + name: Install poetry + command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - + + - restore_cache: + key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Install poetry dependencies + command: | + cd C:\Users\circleci\project\desktop + poetry install + - save_cache: + key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs + + - restore_cache: + key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Get tor binaries from Tor Browser (32-bit) + command: | + cd desktop + poetry run python .\scripts\get-tor.py + - save_cache: + key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\project\desktop\build\tor + - restore_cache: # NOTE: change when upgrading obfs4proxy key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} @@ -167,40 +200,7 @@ jobs: # NOTE: change when upgrading meek key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} paths: - - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - - - run: - name: Install Python 3.9.12 - command: | - choco install python3 --version=3.9.12 - - - run: - name: Install poetry - command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - - - - restore_cache: - key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - - run: - name: Install poetry dependencies - command: | - cd C:\Users\circleci\project\desktop - poetry install - - save_cache: - key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - paths: - - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs - - - restore_cache: - key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} - - run: - name: Get tor binaries from Tor Browser (32-bit) - command: | - cd desktop - poetry run .\scripts\get-tor.py - - save_cache: - key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} - paths: - - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe + - C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe - run: name: Build OnionShare @@ -232,6 +232,39 @@ jobs: steps: - checkout + - run: + name: Install Python 3.9.12 (32-bit) + command: | + choco install python3 --params "/InstallDir32:C:\Python-32bit" --version=3.9.12 + + - run: + name: Install poetry (32-bit) + command: C:\Python-32bit\Scripts\pip install poetry + + - restore_cache: + key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Install poetry dependencies + command: | + cd ~\project\desktop + C:\Python-32bit\Scripts\poetry install + - save_cache: + key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs + + - restore_cache: + key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Get tor binaries from Tor Browser (32-bit) + command: | + cd desktop + C:\Python-32bit\Scripts\poetry run python run .\scripts\get-tor.py + - save_cache: + key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + paths: + - C:\Users\circleci\project\desktop\build\tor + - run: name: Install golang (32-bit) command: | @@ -310,40 +343,7 @@ jobs: # NOTE: change when upgrading meek key: build-win32-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} paths: - - C:\Users\circleci\project\desktop\onionshare\resources\tor\Tor\meek-client.exe - - - run: - name: Install Python 3.9.12 (32-bit) - command: | - choco install python3 --params "/InstallDir32:C:\Python-32bit" --version=3.9.12 - - - run: - name: Install poetry (32-bit) - command: C:\Python-32bit\Scripts\pip install poetry - - - restore_cache: - key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - - run: - name: Install poetry dependencies - command: | - cd ~\project\desktop - C:\Python-32bit\Scripts\poetry install - - save_cache: - key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - paths: - - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs - - - restore_cache: - key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} - - run: - name: Get tor binaries from Tor Browser (32-bit) - command: | - cd desktop - poetry run .\scripts\get-tor.py - - save_cache: - key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} - paths: - - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe + - C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe - run: name: Build OnionShare diff --git a/cli/onionshare_cli/resources/torrc_template-obfs4 b/cli/onionshare_cli/resources/torrc_template-obfs4 index adf343de..c6091b87 100644 --- a/cli/onionshare_cli/resources/torrc_template-obfs4 +++ b/cli/onionshare_cli/resources/torrc_template-obfs4 @@ -1,5 +1,4 @@ # Enable built-in obfs4 bridge -Bridge obfs4 144.217.20.138:80 FB70B257C162BF1038CA669D568D76F5B7F0BABB cert=vYIV5MgrghGQvZPIi1tJwnzorMgqgmlKaB77Y3Z9Q/v94wZBOAXkW+fdx4aSxLVnKO+xNw iat-mode=0 Bridge obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=0 Bridge obfs4 192.95.36.142:443 CDF2E852BF539B82BD10E27E9115A31734E378C2 cert=qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ iat-mode=1 Bridge obfs4 193.11.166.194:27015 2D82C2E354D531A68469ADF7F878FA6060C6BACA cert=4TLQPJrTSaDffMK7Nbao6LC7G9OW/NHkUwIdjLSS3KYf0Nv4/nQiiI8dY2TcsQx01NniOg iat-mode=0 diff --git a/desktop/onionshare/gui_common.py b/desktop/onionshare/gui_common.py index c717fd22..76233016 100644 --- a/desktop/onionshare/gui_common.py +++ b/desktop/onionshare/gui_common.py @@ -495,12 +495,12 @@ class GuiCommon: if self.common.platform == "Windows": base_path = self.get_resource_path("tor") - tor_path = os.path.join(base_path, "Tor", "tor.exe") - obfs4proxy_file_path = os.path.join(base_path, "Tor", "obfs4proxy.exe") - snowflake_file_path = os.path.join(base_path, "Tor", "snowflake-client.exe") - meek_client_file_path = os.path.join(base_path, "Tor", "meek-client.exe") - tor_geo_ip_file_path = os.path.join(base_path, "Data", "Tor", "geoip") - tor_geo_ipv6_file_path = os.path.join(base_path, "Data", "Tor", "geoip6") + tor_path = os.path.join(base_path, "tor.exe") + obfs4proxy_file_path = os.path.join(base_path, "obfs4proxy.exe") + snowflake_file_path = os.path.join(base_path, "snowflake-client.exe") + meek_client_file_path = os.path.join(base_path, "meek-client.exe") + tor_geo_ip_file_path = os.path.join(base_path, "geoip") + tor_geo_ipv6_file_path = os.path.join(base_path, "geoip6") elif self.common.platform == "Darwin": base_path = self.get_resource_path("tor") tor_path = os.path.join(base_path, "tor") @@ -601,7 +601,9 @@ class ToggleCheckbox(QtWidgets.QCheckBox): x = ( rect.width() - rect.x() - self.w + 20 ) # 20 is the padding between text and toggle - y = self.height() / 2 - self.h / 2 + 16 # 16 is the padding top for the checkbox + y = ( + self.height() / 2 - self.h / 2 + 16 + ) # 16 is the padding top for the checkbox self.toggleRect = QtCore.QRect(x, y, self.w, self.h) painter.setBrush(QtGui.QColor(self.bg_color)) painter.drawRoundedRect(x, y, self.w, self.h, self.h / 2, self.h / 2) diff --git a/desktop/scripts/get-tor.py b/desktop/scripts/get-tor.py index 9d5b42da..29c5a1e4 100644 --- a/desktop/scripts/get-tor.py +++ b/desktop/scripts/get-tor.py @@ -2,32 +2,24 @@ import platform import inspect import os +from re import M import sys import hashlib import shutil import subprocess import requests -# Windows -exe_url = ( - "https://dist.torproject.org/torbrowser/11.0.9/torbrowser-install-11.0.9_en-US.exe" -) -exe_filename = "torbrowser-install-11.0.9_en-US.exe" -expected_exe_sha256 = "e938433028b6ffb5d312db6268b19e419626b071f08209684c8e5b9f3d3df2bc" +torbrowser_version = "11.0.14" +expected_exe_sha256 = "c14b979c81310ad039985e047dbb5b8058662bb3105b9022f7b9e0d18a29d0d6" +expected_dmg_sha256 = "558ae5ab188f62feb04c6b2e7f43eae2361e8ec1718e0f4f927801411d911e22" +expected_txz_sha256 = "b606924fdf8237e697cf95c229189da5875c190875d729769655c7b67aeb9aa6" -# macOS -dmg_url = ( - "https://dist.torproject.org/torbrowser/11.0.9/TorBrowser-11.0.9-osx64_en-US.dmg" -) -dmg_filename = "TorBrowser-11.0.9-osx64_en-US.dmg" -expected_dmg_sha256 = "e34629a178a92983924a5a89c7a988285d2d27f21832413a7f7e33af7871c8d6" - -# Linux -tarball_url = "https://dist.torproject.org/torbrowser/11.0.9/tor-browser-linux64-11.0.9_en-US.tar.xz" -tarball_filename = "tor-browser-linux64-11.0.9_en-US.tar.xz" -expected_tarball_sha256 = ( - "baa5ccafb5c68f1c46f9ae983b9b0a0419f66d41e0483ba5aacb3462fa0a8032" -) +exe_url = f"https://dist.torproject.org/torbrowser/{torbrowser_version}/torbrowser-install-{torbrowser_version}_en-US.exe" +exe_filename = f"torbrowser-install-{torbrowser_version}_en-US.exe" +dmg_url = f"https://dist.torproject.org/torbrowser/{torbrowser_version}/TorBrowser-{torbrowser_version}-osx64_en-US.dmg" +dmg_filename = f"TorBrowser-{torbrowser_version}-osx64_en-US.dmg" +tarball_url = f"https://dist.torproject.org/torbrowser/{torbrowser_version}/tor-browser-linux64-{torbrowser_version}_en-US.tar.xz" +tarball_filename = f"tor-browser-linux64-{torbrowser_version}_en-US.tar.xz" # Common paths @@ -53,6 +45,7 @@ def get_tor_windows(): open(exe_path, "wb").write(r.content) exe_sha256 = hashlib.sha256(r.content).hexdigest() else: + print("Already downloaded: {}".format(exe_path)) exe_data = open(exe_path, "rb").read() exe_sha256 = hashlib.sha256(exe_data).hexdigest() @@ -89,10 +82,27 @@ def get_tor_windows(): if os.path.exists(dist_path): shutil.rmtree(dist_path) os.makedirs(dist_path) - shutil.copytree(os.path.join(working_path, "Tor"), os.path.join(dist_path, "Tor")) - shutil.copytree( - os.path.join(working_path, "Data"), os.path.join(dist_path, "Data", "Tor") - ) + for filename in [ + "libcrypto-1_1.dll", + "libevent-2-1-7.dll", + "libevent_core-2-1-7.dll", + "libevent_extra-2-1-7.dll", + "libgcc_s_dw2-1.dll", + "libssl-1_1.dll", + "libssp-0.dll", + "libwinpthread-1.dll", + "tor.exe", + "zlib1.dll", + ]: + shutil.copyfile( + os.path.join(working_path, "Tor", filename), + os.path.join(dist_path, filename), + ) + for filename in ["geoip", "geoip6"]: + shutil.copyfile( + os.path.join(working_path, "Data", filename), + os.path.join(dist_path, filename), + ) # Fetch the built-in bridges update_tor_bridges() @@ -195,9 +205,9 @@ def get_tor_linux(): tarball_sha256 = hashlib.sha256(tarball_data).hexdigest() # Compare the hash - if tarball_sha256 != expected_tarball_sha256: + if tarball_sha256 != expected_txz_sha256: print("ERROR! The sha256 doesn't match:") - print("expected: {}".format(expected_tarball_sha256)) + print("expected: {}".format(expected_txz_sha256)) print(" actual: {}".format(tarball_sha256)) sys.exit(-1) From e7067a1c6d574bade6978893e1318d914f4c5122 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Thu, 16 Jun 2022 22:48:02 -0700 Subject: [PATCH 128/154] Typo in win32 get-tor task --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 15d059c7..a93cbde4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -259,7 +259,7 @@ jobs: name: Get tor binaries from Tor Browser (32-bit) command: | cd desktop - C:\Python-32bit\Scripts\poetry run python run .\scripts\get-tor.py + C:\Python-32bit\Scripts\poetry run python .\scripts\get-tor.py - save_cache: key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} paths: From 7526141024fef0b0e45403ffd2415a581c5b2f63 Mon Sep 17 00:00:00 2001 From: Saptak S Date: Fri, 17 Jun 2022 14:41:49 +0530 Subject: [PATCH 129/154] Adds call operator for calling go build --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a93cbde4..45519b87 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -288,7 +288,7 @@ jobs: cd obfs4 # TODO: verify signature git checkout $OBFS4PROXY_TAG - "C:\Program Files (x86)\Go\bin\go" build .\obfs4proxy + & "C:\Program Files (x86)\Go\bin\go" build .\obfs4proxy Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe } - save_cache: @@ -312,7 +312,7 @@ jobs: cd snowflake # TODO: verify signature git checkout $SNOWFLAKE_TAG - "C:\Program Files (x86)\Go\bin\go" build .\client + & "C:\Program Files (x86)\Go\bin\go" build .\client Move-Item -Path .\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe } - save_cache: @@ -336,7 +336,7 @@ jobs: cd meek # TODO: verify signature git checkout $MEEK_TAG - "C:\Program Files (x86)\Go\bin\go" build .\meek-client + & "C:\Program Files (x86)\Go\bin\go" build .\meek-client Move-Item -Path .\meek-client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe } - save_cache: From 8ff1a0b29459a48bd15de099860061e874592965 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 14:57:37 -0400 Subject: [PATCH 130/154] Make scripts to build PT binaries, and run those in CI --- .circleci/config.yml | 240 +++++------------------- desktop/README.md | 20 +- desktop/scripts/build-meek-client.py | 80 -------- desktop/scripts/build-pt-meek.ps1 | 9 + desktop/scripts/build-pt-meek.sh | 9 + desktop/scripts/build-pt-obfs4proxy.ps1 | 9 + desktop/scripts/build-pt-obfs4proxy.sh | 9 + desktop/scripts/build-pt-snowflake.ps1 | 9 + desktop/scripts/build-pt-snowflake.sh | 9 + desktop/scripts/get-tor.py | 129 +++++++------ 10 files changed, 187 insertions(+), 336 deletions(-) delete mode 100755 desktop/scripts/build-meek-client.py create mode 100644 desktop/scripts/build-pt-meek.ps1 create mode 100755 desktop/scripts/build-pt-meek.sh create mode 100644 desktop/scripts/build-pt-obfs4proxy.ps1 create mode 100755 desktop/scripts/build-pt-obfs4proxy.sh create mode 100644 desktop/scripts/build-pt-snowflake.ps1 create mode 100755 desktop/scripts/build-pt-snowflake.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 45519b87..5c0e36a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,10 +17,10 @@ workflows: requires: - test-cli # - test-gui - # - build-macos: - # requires: - # - test-cli - # # - test-gui + - build-macos: + requires: + - test-cli + # - test-gui jobs: test-cli: @@ -86,13 +86,6 @@ jobs: executor: name: win/default shell: powershell.exe - environment: - # NOTE: change when upgrading obfs4proxy - OBFS4PROXY_TAG: obfs4proxy-0.0.13 - # NOTE: change when upgrading snowflake - SNOWFLAKE_TAG: v2.2.0 - # NOTE: change when upgrading meek - MEEK_TAG: v0.37.0 steps: - checkout @@ -120,85 +113,60 @@ jobs: - restore_cache: key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} - run: - name: Get tor binaries from Tor Browser (32-bit) + name: Get tor binaries from Tor Browser (64-bit) command: | cd desktop - poetry run python .\scripts\get-tor.py + poetry run python .\scripts\get-tor.py win64 - save_cache: key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\build\tor - restore_cache: - # NOTE: change when upgrading obfs4proxy - key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} + key: build-win64-obfs4proxy-{{ checksum "~/project/desktop/scripts/build-pt-obfs4proxy.ps1" }}-{{ .Environment.CACHE_VERSION }} - run: name: Build obfs4proxy command: | if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe) -eq $True) { Write-Output "obfs4proxy already built" } else { - # curl https://keys.openpgp.org/vks/v1/by-fingerprint/9EB1A490C73CC5D44DFB3E47BFBD1C7B8A6EC81A | gpg --import - New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\obfs4proxy - cd C:\Users\circleci\src\obfs4proxy - git clone https://gitlab.com/yawning/obfs4 - cd obfs4 - # TODO: verify signature - git checkout $OBFS4PROXY_TAG - go build .\obfs4proxy - Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe + cd C:\Users\circleci\project\desktop + .\scripts\build-pt-obfs4proxy.ps1 } - save_cache: - # NOTE: change when upgrading obfs4proxy - key: build-win64-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} + key: build-win64-obfs4proxy-{{ checksum "~/project/desktop/scripts/build-pt-obfs4proxy.ps1" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe - restore_cache: - # NOTE: change when upgrading snowflake - key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} + key: build-win64-snowflake-{{ checksum "~/project/desktop/scripts/build-pt-snowflake.ps1" }}-{{ .Environment.CACHE_VERSION }} - run: name: Build snowflake-client command: | if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe) -eq $True) { Write-Output "snowflake already built" } else { - New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\obfs4proxy - cd C:\Users\circleci\src\obfs4proxy - git clone https://git.torproject.org/pluggable-transports/snowflake.git - cd snowflake - # TODO: verify signature - git checkout $SNOWFLAKE_TAG - go build .\client - Move-Item -Path .\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe + cd C:\Users\circleci\project\desktop + .\scripts\build-pt-snowflake.ps1 } - save_cache: - # NOTE: change when upgrading snowflake - key: build-win64-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} + key: build-win64-snowflake-{{ checksum "~/project/desktop/scripts/build-pt-snowflake.ps1" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe - restore_cache: - # NOTE: change when upgrading meek - key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} + key: build-win64-meek-{{ checksum "~/project/desktop/scripts/build-pt-meek.ps1" }}-{{ .Environment.CACHE_VERSION }} - run: name: Build meek-client command: | if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe) -eq $True) { Write-Output "snowflake already built" } else { - New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\meek - cd C:\Users\circleci\src\meek - git clone https://git.torproject.org/pluggable-transports/meek.git - cd meek - # TODO: verify signature - git checkout $MEEK_TAG - go build .\meek-client - Move-Item -Path .\meek-client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe + cd C:\Users\circleci\project\desktop + .\scripts\build-pt-meek.ps1 } - save_cache: - # NOTE: change when upgrading meek - key: build-win64-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} + key: build-win64-meek-{{ checksum "~/project/desktop/scripts/build-pt-meek.ps1" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe @@ -222,13 +190,6 @@ jobs: executor: name: win/default shell: powershell.exe - environment: - # NOTE: change when upgrading obfs4proxy - OBFS4PROXY_TAG: obfs4proxy-0.0.13 - # NOTE: change when upgrading snowflake - SNOWFLAKE_TAG: v2.2.0 - # NOTE: change when upgrading meek - MEEK_TAG: v0.37.0 steps: - checkout @@ -259,7 +220,7 @@ jobs: name: Get tor binaries from Tor Browser (32-bit) command: | cd desktop - C:\Python-32bit\Scripts\poetry run python .\scripts\get-tor.py + C:\Python-32bit\Scripts\poetry run python .\scripts\get-tor.py win32 - save_cache: key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} paths: @@ -273,75 +234,53 @@ jobs: msiexec.exe /i go1.18.windows-386.msi /quiet /L*V go-install.log - restore_cache: - # NOTE: change when upgrading obfs4proxy - key: build-win32-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} + key: build-win32-obfs4proxy-{{ checksum "~/project/desktop/scripts/build-pt-obfs4proxy.ps1" }}-{{ .Environment.CACHE_VERSION }} - run: - name: Build obfs4proxy (32-bit) + name: Build obfs4proxy command: | if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe) -eq $True) { Write-Output "obfs4proxy already built" } else { - # curl https://keys.openpgp.org/vks/v1/by-fingerprint/9EB1A490C73CC5D44DFB3E47BFBD1C7B8A6EC81A | gpg --import - New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\obfs4proxy - cd C:\Users\circleci\src\obfs4proxy - git clone https://gitlab.com/yawning/obfs4 - cd obfs4 - # TODO: verify signature - git checkout $OBFS4PROXY_TAG - & "C:\Program Files (x86)\Go\bin\go" build .\obfs4proxy - Move-Item -Path .\obfs4proxy.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe + $env:PATH = "C:\Program Files (x86)\Go\bin\go;$env:PATH" + cd C:\Users\circleci\project\desktop + .\scripts\build-pt-obfs4proxy.ps1 } - save_cache: - # NOTE: change when upgrading obfs4proxy - key: build-win32-obfs4proxy-v0.0.13-{{ .Environment.CACHE_VERSION }} + key: build-win32-obfs4proxy-{{ checksum "~/project/desktop/scripts/build-pt-obfs4proxy.ps1" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe - restore_cache: - # NOTE: change when upgrading snowflake - key: build-win32-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} + key: build-win32-snowflake-{{ checksum "~/project/desktop/scripts/build-pt-snowflake.ps1" }}-{{ .Environment.CACHE_VERSION }} - run: - name: Build snowflake-client (32-bit) + name: Build snowflake-client command: | if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe) -eq $True) { Write-Output "snowflake already built" } else { - New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\obfs4proxy - cd C:\Users\circleci\src\obfs4proxy - git clone https://git.torproject.org/pluggable-transports/snowflake.git - cd snowflake - # TODO: verify signature - git checkout $SNOWFLAKE_TAG - & "C:\Program Files (x86)\Go\bin\go" build .\client - Move-Item -Path .\client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe + $env:PATH = "C:\Program Files (x86)\Go\bin\go;$env:PATH" + cd C:\Users\circleci\project\desktop + .\scripts\build-pt-snowflake.ps1 } - save_cache: - # NOTE: change when upgrading snowflake - key: build-win32-snowflake-v2.1.0-{{ .Environment.CACHE_VERSION }} + key: build-win32-snowflake-{{ checksum "~/project/desktop/scripts/build-pt-snowflake.ps1" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe - restore_cache: - # NOTE: change when upgrading meek - key: build-win32-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} + key: build-win32-meek-{{ checksum "~/project/desktop/scripts/build-pt-meek.ps1" }}-{{ .Environment.CACHE_VERSION }} - run: - name: Build meek-client (32-bit) + name: Build meek-client command: | if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe) -eq $True) { Write-Output "snowflake already built" } else { - New-Item -ItemType Directory -Force -Path C:\Users\circleci\src\meek - cd C:\Users\circleci\src\meek - git clone https://git.torproject.org/pluggable-transports/meek.git - cd meek - # TODO: verify signature - git checkout $MEEK_TAG - & "C:\Program Files (x86)\Go\bin\go" build .\meek-client - Move-Item -Path .\meek-client.exe -Destination C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe + $env:PATH = "C:\Program Files (x86)\Go\bin\go;$env:PATH" + cd C:\Users\circleci\project\desktop + .\scripts\build-pt-meek.ps1 } - save_cache: - # NOTE: change when upgrading meek - key: build-win32-meek-v0.37.0-{{ .Environment.CACHE_VERSION }} + key: build-win32-meek-{{ checksum "~/project/desktop/scripts/build-pt-meek.ps1" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe @@ -373,120 +312,25 @@ jobs: command: | brew install wget brew install go - - restore_cache: - key: build-libevent-2.1.12-tor-0.4.6.10 - - run: - name: Build libevent and tor - command: | - mkdir -p $BINARY_DIR/tor - # Download and verify libevent - cd ~/Downloads - URL=https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz - FILENAME=libevent-2.1.12-stable.tar.gz - EXPECTED_SHA256=92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb - wget $URL - SHA256=$(shasum -a 256 $FILENAME | cut -d" " -f1) - if [ "$SHA256" != "$EXPECTED_SHA256" ]; then echo "Failed" && exit -1; fi - tar -xvf $FILENAME - # Build libevent - cd libevent-2.1.12-stable - ./configure --disable-openssl --prefix=$BINARY_DIR/tor - make - make install - # Download and verify tor - cd ~/Downloads - URL=https://dist.torproject.org/tor-0.4.6.10.tar.gz - FILENAME=tor-0.4.6.10.tar.gz - EXPECTED_SHA256=94ccd60e04e558f33be73032bc84ea241660f92f58cfb88789bda6893739e31c - wget $URL - SHA256=$(shasum -a 256 $FILENAME | cut -d" " -f1) - if [ "$SHA256" != "$EXPECTED_SHA256" ]; then echo "Failed" && exit -1; fi - tar -xvf $FILENAME - # Build tor - cd tor-0.4.6.10 - ./configure --prefix=$BINARY_DIR/tor - make - make install - - save_cache: - key: build-libevent-2.1.12-tor-0.4.6.10 - paths: - - /Users/distiller/bin/tor - - restore_cache: - key: build-macos-obfs4proxy-v0.0.13 - - run: - name: Build obfs4proxy - command: | - # Get source code - mkdir ~/Downloads/obfs4proxy - cd ~/Downloads/obfs4proxy - git clone https://gitlab.com/yawning/obfs4 - cd obfs4 - git checkout obfs4proxy-0.0.13 - # Build - go build -o $BINARY_DIR/obfs4proxy ./obfs4proxy - - save_cache: - key: build-macos-obfs4proxy-v0.0.13 - paths: - - /Users/distiller/bin/obfs4proxy - - restore_cache: - key: build-macos-snowflake-v2.1.0 - - run: - name: Build snowflake-client - command: | - # Get source code - mkdir ~/Downloads/snowflake-client - cd ~/Downloads/snowflake-client - git clone https://git.torproject.org/pluggable-transports/snowflake.git - cd snowflake - git checkout v2.1.0 - # Build - go build -o $BINARY_DIR/snowflake-client ./client - - save_cache: - key: build-macos-snowflake-v2.1.0 - paths: - - /Users/distiller/bin/snowflake-client - - restore_cache: - key: build-macos-meek-v0.37.0 - - run: - name: Build meek-client - command: | - # Get source code - mkdir ~/Downloads/meek-client - cd ~/Downloads/meek-client - git clone https://git.torproject.org/pluggable-transports/meek.git - cd meek - git checkout v0.37.0 - # Build - go build -o $BINARY_DIR/meek-client ./meek-client - - save_cache: - key: build-macos-meek-v0.37.0 - paths: - - /Users/distiller/bin/meek-client - - run: - name: Copy binaries into app - command: | - export DEST=~\project\desktop\onionshare\resources\tor - cp $BINARY_DIR/tor/bin/tor $DEST - cp $BINARY_DIR/tor/lib/libevent-2.1.7.dylib $DEST - cp $BINARY_DIR/tor/share/tor/geoip* $DEST - cp $BINARY_DIR/obfs4proxy $DEST - cp $BINARY_DIR/snowflake-client $DEST - cp $BINARY_DIR/meek-client $DEST + - run: name: Install Python 3.9.12 command: | wget https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg -O ~/Downloads/python.pkg sudo installer -pkg ~/Downloads/python.pkg -target / + - run: name: Install poetry command: | pip3 install poetry ln -s /Library/Frameworks/Python.framework/Versions/3.9/bin/poetry /usr/local/bin + - run: name: Install poetry dependencies command: | cd ~/project/desktop poetry install + - run: name: Build OnionShare command: | @@ -494,10 +338,12 @@ jobs: poetry run python ./setup-freeze.py build poetry run python ./setup-freeze.py bdist_mac poetry run python ./scripts/build-macos.py cleanup-build + - run: name: Compress command: | cd ~/project/desktop/build zip -r ~/onionshare-macos.zip OnionShare.app + - store_artifacts: path: ~/onionshare-macos.zip diff --git a/desktop/README.md b/desktop/README.md index 31071782..5a070ee6 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -31,20 +31,32 @@ poetry install **Windows users:** Download and install 7-Zip from https://7-zip.org/download.html. [Add](https://medium.com/@kevinmarkvi/how-to-add-executables-to-your-path-in-windows-5ffa4ce61a53) `C:\Program Files (x86)\7-Zip` to your path. -Download Tor Browser and extract the binaries: +Download Tor Browser and extract the binaries for your platform. The platform must be `win32`, `win64`, `macos`, or `linux64`. ```sh -poetry run python ./scripts/get-tor.py +poetry run python ./scripts/get-tor.py [platform] ``` ### Compile dependencies Install Go. The simplest way to make sure everything works is to install Go by following [these instructions](https://golang.org/doc/install). -Download and compile `meek-client`: +Compile pluggable transports: + +**Windows users, in PowerShell:** + +```powershell +.\scripts\build-pt-obfs4proxy.ps1 +.\scripts\build-pt-snowflake.ps1 +.\scripts\build-pt-meek.ps1 +``` + +**macOS and Linux users:** ```sh -poetry run python ./scripts/build-meek-client.py +./scripts/build-pt-obfs4proxy.sh +./scripts/build-pt-snowflake.sh +./scripts/build-pt-meek.sh ``` ### Running OnionShare from the source code tree diff --git a/desktop/scripts/build-meek-client.py b/desktop/scripts/build-meek-client.py deleted file mode 100755 index 6e5d7974..00000000 --- a/desktop/scripts/build-meek-client.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -OnionShare | https://onionshare.org/ - -Copyright (C) 2014-2022 Micah Lee, et al. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -""" -This script downloads a pre-built tor binary to bundle with OnionShare. -In order to avoid a Mac gnupg dependency, I manually verify the signature -and hard-code the sha256 hash. -""" -import shutil -import os -import sys -import subprocess -import inspect -import platform - - -def main(): - # Figure out the architecture and python path - if "64 bit" in sys.version: - python_arch = "win-amd64" - else: - python_arch = "win32" - - if os.getlogin() == "circleci" and python_arch == "win32": - go_path = "C:\\Program Files (x86)\\Go\\bin\\go" - else: - go_path = shutil.which("go") - - if go_path is None: - print("Install go: https://golang.org/doc/install") - return - - subprocess.run( - [ - go_path, - "install", - "git.torproject.org/pluggable-transports/meek.git/meek-client@v0.37.0", - ] - ) - - root_path = os.path.dirname( - os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) - ) - if platform.system() == "Windows": - dist_path = os.path.join(root_path, "onionshare", "resources", "tor", "Tor") - bin_filename = "meek-client.exe" - else: - dist_path = os.path.join(root_path, "onionshare", "resources", "tor") - bin_filename = "meek-client" - - bin_path = os.path.join(os.path.expanduser("~"), "go", "bin", bin_filename) - shutil.copyfile( - os.path.join(bin_path), - os.path.join(dist_path, bin_filename), - ) - os.chmod(os.path.join(dist_path, bin_filename), 0o755) - - print(f"Installed {bin_filename} in {dist_path}") - - -if __name__ == "__main__": - main() diff --git a/desktop/scripts/build-pt-meek.ps1 b/desktop/scripts/build-pt-meek.ps1 new file mode 100644 index 00000000..58a7366a --- /dev/null +++ b/desktop/scripts/build-pt-meek.ps1 @@ -0,0 +1,9 @@ +$env:MEEK_TAG = 'v0.37.0' + +New-Item -ItemType Directory -Force -Path .\build\meek +cd .\build\meek +git clone https://git.torproject.org/pluggable-transports/meek.git +cd meek +git checkout $MEEK_TAG +go build .\meek-client +Move-Item -Path .\meek-client.exe -Destination ..\onionshare\resources\tor\meek-client.exe diff --git a/desktop/scripts/build-pt-meek.sh b/desktop/scripts/build-pt-meek.sh new file mode 100755 index 00000000..5ce91d6c --- /dev/null +++ b/desktop/scripts/build-pt-meek.sh @@ -0,0 +1,9 @@ +#!/bin/bash +MEEK_TAG=v0.37.0 + +mkdir -p ./build/meek +cd ./build/meek +git clone https://git.torproject.org/pluggable-transports/meek.git +cd meek +git checkout $MEEK_TAG +go build -o ../../../onionshare/resources/tor/meek-client ./meek-client diff --git a/desktop/scripts/build-pt-obfs4proxy.ps1 b/desktop/scripts/build-pt-obfs4proxy.ps1 new file mode 100644 index 00000000..e4688c93 --- /dev/null +++ b/desktop/scripts/build-pt-obfs4proxy.ps1 @@ -0,0 +1,9 @@ +$env:OBFS4PROXY_TAG = 'obfs4proxy-0.0.13' + +New-Item -ItemType Directory -Force -Path .\build\obfs4proxy +cd .\build\obfs4proxy +git clone https://gitlab.com/yawning/obfs4 +cd obfs4 +git checkout $OBFS4PROXY_TAG +go build .\obfs4proxy +Move-Item -Path .\obfs4proxy.exe -Destination ..\onionshare\resources\tor\obfs4proxy.exe diff --git a/desktop/scripts/build-pt-obfs4proxy.sh b/desktop/scripts/build-pt-obfs4proxy.sh new file mode 100755 index 00000000..15b317a3 --- /dev/null +++ b/desktop/scripts/build-pt-obfs4proxy.sh @@ -0,0 +1,9 @@ +#!/bin/bash +OBFS4PROXY_TAG=obfs4proxy-0.0.13 + +mkdir -p ./build/obfs4proxy +cd ./build/obfs4proxy +git clone https://gitlab.com/yawning/obfs4 || echo "already cloned" +cd obfs4 +git checkout $OBFS4PROXY_TAG +go build -o ../../../onionshare/resources/tor/obfs4proxy ./obfs4proxy \ No newline at end of file diff --git a/desktop/scripts/build-pt-snowflake.ps1 b/desktop/scripts/build-pt-snowflake.ps1 new file mode 100644 index 00000000..5d15759d --- /dev/null +++ b/desktop/scripts/build-pt-snowflake.ps1 @@ -0,0 +1,9 @@ +$env:SNOWFLAKE_TAG = 'v2.2.0' + +New-Item -ItemType Directory -Force -Path .\build\snowflake +cd .\build\snowflake +git clone https://git.torproject.org/pluggable-transports/snowflake.git +cd snowflake +git checkout $SNOWFLAKE_TAG +go build .\client +Move-Item -Path .\client.exe -Destination ..\onionshare\resources\tor\snowflake-client.exe diff --git a/desktop/scripts/build-pt-snowflake.sh b/desktop/scripts/build-pt-snowflake.sh new file mode 100755 index 00000000..24a7ce74 --- /dev/null +++ b/desktop/scripts/build-pt-snowflake.sh @@ -0,0 +1,9 @@ +#!/bin/bash +SNOWFLAKE_TAG=v2.2.0 + +mkdir -p ./build/snowflake +cd ./build/snowflake +git clone https://git.torproject.org/pluggable-transports/snowflake.git +cd snowflake +git checkout $SNOWFLAKE_TAG +go build -o ../../../onionshare/resources/tor/snowflake-client ./client diff --git a/desktop/scripts/get-tor.py b/desktop/scripts/get-tor.py index 29c5a1e4..bda4e8b8 100644 --- a/desktop/scripts/get-tor.py +++ b/desktop/scripts/get-tor.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -import platform import inspect import os from re import M @@ -8,18 +7,30 @@ import hashlib import shutil import subprocess import requests +import click torbrowser_version = "11.0.14" -expected_exe_sha256 = "c14b979c81310ad039985e047dbb5b8058662bb3105b9022f7b9e0d18a29d0d6" -expected_dmg_sha256 = "558ae5ab188f62feb04c6b2e7f43eae2361e8ec1718e0f4f927801411d911e22" -expected_txz_sha256 = "b606924fdf8237e697cf95c229189da5875c190875d729769655c7b67aeb9aa6" +expected_win32_sha256 = ( + "c14b979c81310ad039985e047dbb5b8058662bb3105b9022f7b9e0d18a29d0d6" +) +expected_win64_sha256 = ( + "ced3de06d089fbbeb8cee309971ac26983aba8eaf948fedce472d40cdd572301" +) +expected_macos_sha256 = ( + "558ae5ab188f62feb04c6b2e7f43eae2361e8ec1718e0f4f927801411d911e22" +) +expected_linux64_sha256 = ( + "b606924fdf8237e697cf95c229189da5875c190875d729769655c7b67aeb9aa6" +) -exe_url = f"https://dist.torproject.org/torbrowser/{torbrowser_version}/torbrowser-install-{torbrowser_version}_en-US.exe" -exe_filename = f"torbrowser-install-{torbrowser_version}_en-US.exe" -dmg_url = f"https://dist.torproject.org/torbrowser/{torbrowser_version}/TorBrowser-{torbrowser_version}-osx64_en-US.dmg" -dmg_filename = f"TorBrowser-{torbrowser_version}-osx64_en-US.dmg" -tarball_url = f"https://dist.torproject.org/torbrowser/{torbrowser_version}/tor-browser-linux64-{torbrowser_version}_en-US.tar.xz" -tarball_filename = f"tor-browser-linux64-{torbrowser_version}_en-US.tar.xz" +win32_url = f"https://dist.torproject.org/torbrowser/{torbrowser_version}/torbrowser-install-{torbrowser_version}_en-US.exe" +win32_filename = f"torbrowser-install-{torbrowser_version}_en-US.exe" +win64_url = f"https://dist.torproject.org/torbrowser/{torbrowser_version}/torbrowser-install-win64-{torbrowser_version}_en-US.exe" +win64_filename = f"torbrowser-install-win64-{torbrowser_version}_en-US.exe" +macos_url = f"https://dist.torproject.org/torbrowser/{torbrowser_version}/TorBrowser-{torbrowser_version}-osx64_en-US.dmg" +macos_filename = f"TorBrowser-{torbrowser_version}-osx64_en-US.dmg" +linux64_url = f"https://dist.torproject.org/torbrowser/{torbrowser_version}/tor-browser-linux64-{torbrowser_version}_en-US.tar.xz" +linux64_filename = f"tor-browser-linux64-{torbrowser_version}_en-US.tar.xz" # Common paths @@ -29,9 +40,21 @@ root_path = os.path.dirname( working_path = os.path.join(root_path, "build", "tor") -def get_tor_windows(): +def get_tor_windows(platform): + if platform == "win32": + win_url = win32_url + win_filename = win32_filename + expected_win_sha256 = expected_win32_sha256 + elif platform == "win64": + win_url = win64_url + win_filename = win64_filename + expected_win_sha256 = expected_win64_sha256 + else: + click.echo("invalid platform") + return + # Build paths - exe_path = os.path.join(working_path, exe_filename) + win_path = os.path.join(working_path, win_filename) dist_path = os.path.join(root_path, "onionshare", "resources", "tor") # Make sure the working folder exists @@ -39,21 +62,21 @@ def get_tor_windows(): os.makedirs(working_path) # Make sure Tor Browser is downloaded - if not os.path.exists(exe_path): - print("Downloading {}".format(exe_url)) - r = requests.get(exe_url) - open(exe_path, "wb").write(r.content) + if not os.path.exists(win_path): + print("Downloading {}".format(win_url)) + r = requests.get(win_url) + open(win_path, "wb").write(r.content) exe_sha256 = hashlib.sha256(r.content).hexdigest() else: - print("Already downloaded: {}".format(exe_path)) - exe_data = open(exe_path, "rb").read() - exe_sha256 = hashlib.sha256(exe_data).hexdigest() + print("Already downloaded: {}".format(win_path)) + win_data = open(win_path, "rb").read() + win_sha256 = hashlib.sha256(win_data).hexdigest() # Compare the hash - if exe_sha256 != expected_exe_sha256: + if win_sha256 != expected_win_sha256: print("ERROR! The sha256 doesn't match:") - print("expected: {}".format(expected_exe_sha256)) - print(" actual: {}".format(exe_sha256)) + print("expected: {}".format(expected_win32_sha256)) + print(" actual: {}".format(win_sha256)) sys.exit(-1) # Extract the bits we need from the exe @@ -62,7 +85,7 @@ def get_tor_windows(): "7z", "e", "-y", - exe_path, + win_path, "Browser\\TorBrowser\\Tor", "-o%s" % os.path.join(working_path, "Tor"), ] @@ -72,7 +95,7 @@ def get_tor_windows(): "7z", "e", "-y", - exe_path, + win_path, "Browser\\TorBrowser\\Data\\Tor\\geoip*", "-o%s" % os.path.join(working_path, "Data"), ] @@ -113,7 +136,7 @@ def get_tor_macos(): dmg_tor_path = os.path.join( "/Volumes", "Tor Browser", "Tor Browser.app", "Contents" ) - dmg_path = os.path.join(working_path, dmg_filename) + dmg_path = os.path.join(working_path, macos_filename) dist_path = os.path.join(root_path, "onionshare", "resources", "tor") if not os.path.exists(dist_path): os.makedirs(dist_path, exist_ok=True) @@ -124,8 +147,8 @@ def get_tor_macos(): # Make sure the zip is downloaded if not os.path.exists(dmg_path): - print("Downloading {}".format(dmg_url)) - r = requests.get(dmg_url) + print("Downloading {}".format(macos_url)) + r = requests.get(macos_url) open(dmg_path, "wb").write(r.content) dmg_sha256 = hashlib.sha256(r.content).hexdigest() else: @@ -133,9 +156,9 @@ def get_tor_macos(): dmg_sha256 = hashlib.sha256(dmg_data).hexdigest() # Compare the hash - if dmg_sha256 != expected_dmg_sha256: + if dmg_sha256 != expected_macos_sha256: print("ERROR! The sha256 doesn't match:") - print("expected: {}".format(expected_dmg_sha256)) + print("expected: {}".format(expected_macos_sha256)) print(" actual: {}".format(dmg_sha256)) sys.exit(-1) @@ -182,9 +205,9 @@ def get_tor_macos(): update_tor_bridges() -def get_tor_linux(): +def get_tor_linux64(): # Build paths - tarball_path = os.path.join(working_path, tarball_filename) + tarball_path = os.path.join(working_path, linux64_filename) dist_path = os.path.join(root_path, "onionshare", "resources", "tor") # Make sure dirs exist @@ -196,8 +219,8 @@ def get_tor_linux(): # Make sure the tarball is downloaded if not os.path.exists(tarball_path): - print("Downloading {}".format(tarball_url)) - r = requests.get(tarball_url) + print("Downloading {}".format(linux64_url)) + r = requests.get(linux64_url) open(tarball_path, "wb").write(r.content) tarball_sha256 = hashlib.sha256(r.content).hexdigest() else: @@ -205,9 +228,9 @@ def get_tor_linux(): tarball_sha256 = hashlib.sha256(tarball_data).hexdigest() # Compare the hash - if tarball_sha256 != expected_txz_sha256: + if tarball_sha256 != expected_linux64_sha256: print("ERROR! The sha256 doesn't match:") - print("expected: {}".format(expected_txz_sha256)) + print("expected: {}".format(expected_linux64_sha256)) print(" actual: {}".format(tarball_sha256)) sys.exit(-1) @@ -250,18 +273,6 @@ def get_tor_linux(): os.path.join(tarball_tor_path, "Tor", "libstdc++", "libstdc++.so.6"), os.path.join(dist_path, "libstdc++.so.6"), ) - shutil.copyfile( - os.path.join(tarball_tor_path, "Tor", "PluggableTransports", "obfs4proxy"), - os.path.join(dist_path, "obfs4proxy"), - ) - os.chmod(os.path.join(dist_path, "obfs4proxy"), 0o755) - shutil.copyfile( - os.path.join( - tarball_tor_path, "Tor", "PluggableTransports", "snowflake-client" - ), - os.path.join(dist_path, "snowflake-client"), - ) - os.chmod(os.path.join(dist_path, "snowflake-client"), 0o755) print(f"Tor binaries extracted to: {dist_path}") @@ -317,19 +328,27 @@ def update_tor_bridges(): f.write(f"Bridge {item}\n") -def main(): +@click.command() +@click.argument("platform") +def main(platform): """ Download Tor Browser and extract tor binaries """ - system = platform.system() - if system == "Windows": - get_tor_windows() - elif system == "Darwin": + valid_platforms = ["win32", "win64", "macos", "linux64"] + if platform not in valid_platforms: + click.echo(f"platform must be one of: {valid_platforms}") + return + + if platform == "win32": + get_tor_windows(platform) + elif platform == "win64": + get_tor_windows(platform) + elif platform == "macos": get_tor_macos() - elif system == "Linux": - get_tor_linux() + elif platform == "linux64": + get_tor_linux64() else: - print("Platform not supported") + click.echo("invalid platform") if __name__ == "__main__": From 038e1e9ef06418f6d82fa3714383d1312e70e350 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 15:19:21 -0400 Subject: [PATCH 131/154] Run get-tor, and the PT build scripts, in macos --- .circleci/config.yml | 74 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5c0e36a0..d415fa46 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,14 +111,14 @@ jobs: - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs - restore_cache: - key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + key: get-tor-win64-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} - run: name: Get tor binaries from Tor Browser (64-bit) command: | cd desktop poetry run python .\scripts\get-tor.py win64 - save_cache: - key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + key: get-tor-win64-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\build\tor @@ -215,14 +215,14 @@ jobs: - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs - restore_cache: - key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + key: get-tor-win32-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} - run: name: Get tor binaries from Tor Browser (32-bit) command: | cd desktop C:\Python-32bit\Scripts\poetry run python .\scripts\get-tor.py win32 - save_cache: - key: get-tor-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + key: get-tor-win32-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} paths: - C:\Users\circleci\project\desktop\build\tor @@ -325,11 +325,77 @@ jobs: pip3 install poetry ln -s /Library/Frameworks/Python.framework/Versions/3.9/bin/poetry /usr/local/bin + - save_cache: + key: build-macos-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - run: name: Install poetry dependencies command: | cd ~/project/desktop poetry install + - save_cache: + key: build-macos-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + paths: + - /Users/distiller/Library/Caches/pypoetry/virtualenvs + + - restore_cache: + key: get-tor-macos-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Get tor binaries from Tor Browser + command: | + cd desktop + poetry run python ./scripts/get-tor.py macos + - save_cache: + key: get-tor-macos-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} + paths: + - ~/project/desktop/build/tor + + - restore_cache: + key: build-macos-obfs4proxy-{{ checksum "~/project/desktop/scripts/build-pt-obfs4proxy.sh" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Build obfs4proxy + command: | + if [[ -f "~/project/desktop/onionshare/resources/tor/obfs4proxy" ]]; then + echo "obfs4proxy already built" + else + cd ~/projects/desktop + ./scripts/build-pt-obfs4proxy.sh + fi + - save_cache: + key: build-macos-obfs4proxy-{{ checksum "~/project/desktop/scripts/build-pt-obfs4proxy.sh" }}-{{ .Environment.CACHE_VERSION }} + paths: + - ~/project/desktop/onionshare/resources/tor/obfs4proxy + + - restore_cache: + key: build-macos-snowflake-{{ checksum "~/project/desktop/scripts/build-pt-snowflake.sh" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Build snowflake + command: | + if [[ -f "~/project/desktop/onionshare/resources/tor/snowflake-client" ]]; then + echo "snowflake already built" + else + cd ~/projects/desktop + ./scripts/build-pt-snowflake.sh + fi + - save_cache: + key: build-macos-snowflake-{{ checksum "~/project/desktop/scripts/build-pt-snowflake.sh" }}-{{ .Environment.CACHE_VERSION }} + paths: + - ~/project/desktop/onionshare/resources/tor/snowflake-client + + - restore_cache: + key: build-macos-meek-{{ checksum "~/project/desktop/scripts/build-pt-obfs4proxy.sh" }}-{{ .Environment.CACHE_VERSION }} + - run: + name: Build meek + command: | + if [[ -f "~/project/desktop/onionshare/resources/tor/meek-client" ]]; then + echo "meek already built" + else + cd ~/projects/desktop + ./scripts/build-pt-meek.sh + fi + - save_cache: + key: build-macos-meek-{{ checksum "~/project/desktop/scripts/build-pt-meek.sh" }}-{{ .Environment.CACHE_VERSION }} + paths: + - ~/project/desktop/onionshare/resources/tor/meek-client - run: name: Build OnionShare From 3d1b122bcafe195d7787883c10f73d3e82d0d42e Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 15:28:04 -0400 Subject: [PATCH 132/154] Fix get-tor bug --- desktop/scripts/get-tor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/desktop/scripts/get-tor.py b/desktop/scripts/get-tor.py index bda4e8b8..5df6918f 100644 --- a/desktop/scripts/get-tor.py +++ b/desktop/scripts/get-tor.py @@ -66,7 +66,7 @@ def get_tor_windows(platform): print("Downloading {}".format(win_url)) r = requests.get(win_url) open(win_path, "wb").write(r.content) - exe_sha256 = hashlib.sha256(r.content).hexdigest() + win_sha256 = hashlib.sha256(r.content).hexdigest() else: print("Already downloaded: {}".format(win_path)) win_data = open(win_path, "rb").read() From 6a5de62ba5aba8e11a2b11813cacc2ab9ee88c35 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 15:35:25 -0400 Subject: [PATCH 133/154] Oops, fix circleci config --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d415fa46..94f0761c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -325,7 +325,7 @@ jobs: pip3 install poetry ln -s /Library/Frameworks/Python.framework/Versions/3.9/bin/poetry /usr/local/bin - - save_cache: + - restore_cache: key: build-macos-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - run: name: Install poetry dependencies From e60c56d4574402d1647c4fcb47b260bd21dd250c Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 15:57:10 -0400 Subject: [PATCH 134/154] 64-bit Tor Browser binaries have different filenames than 32-bit --- .circleci/config.yml | 6 +++--- desktop/scripts/get-tor.py | 37 +++++++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 94f0761c..6e5fa62e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -357,7 +357,7 @@ jobs: if [[ -f "~/project/desktop/onionshare/resources/tor/obfs4proxy" ]]; then echo "obfs4proxy already built" else - cd ~/projects/desktop + cd ~/project/desktop ./scripts/build-pt-obfs4proxy.sh fi - save_cache: @@ -373,7 +373,7 @@ jobs: if [[ -f "~/project/desktop/onionshare/resources/tor/snowflake-client" ]]; then echo "snowflake already built" else - cd ~/projects/desktop + cd ~/project/desktop ./scripts/build-pt-snowflake.sh fi - save_cache: @@ -389,7 +389,7 @@ jobs: if [[ -f "~/project/desktop/onionshare/resources/tor/meek-client" ]]; then echo "meek already built" else - cd ~/projects/desktop + cd ~/project/desktop ./scripts/build-pt-meek.sh fi - save_cache: diff --git a/desktop/scripts/get-tor.py b/desktop/scripts/get-tor.py index 5df6918f..4ba53ce2 100644 --- a/desktop/scripts/get-tor.py +++ b/desktop/scripts/get-tor.py @@ -45,10 +45,34 @@ def get_tor_windows(platform): win_url = win32_url win_filename = win32_filename expected_win_sha256 = expected_win32_sha256 + bin_filenames = [ + "libcrypto-1_1.dll", + "libevent-2-1-7.dll", + "libevent_core-2-1-7.dll", + "libevent_extra-2-1-7.dll", + "libgcc_s_dw2-1.dll", + "libssl-1_1.dll", + "libssp-0.dll", + "libwinpthread-1.dll", + "tor.exe", + "zlib1.dll", + ] elif platform == "win64": win_url = win64_url win_filename = win64_filename expected_win_sha256 = expected_win64_sha256 + bin_filenames = [ + "libcrypto-1_1-x64.dll", + "libevent-2-1-7.dll", + "libevent_core-2-1-7.dll", + "libevent_extra-2-1-7.dll", + "libgcc_s_dw2-1.dll", + "libssl-1_1-x64.dll", + "libssp-0.dll", + "libwinpthread-1.dll", + "tor.exe", + "zlib1.dll", + ] else: click.echo("invalid platform") return @@ -105,18 +129,7 @@ def get_tor_windows(platform): if os.path.exists(dist_path): shutil.rmtree(dist_path) os.makedirs(dist_path) - for filename in [ - "libcrypto-1_1.dll", - "libevent-2-1-7.dll", - "libevent_core-2-1-7.dll", - "libevent_extra-2-1-7.dll", - "libgcc_s_dw2-1.dll", - "libssl-1_1.dll", - "libssp-0.dll", - "libwinpthread-1.dll", - "tor.exe", - "zlib1.dll", - ]: + for filename in bin_filenames: shutil.copyfile( os.path.join(working_path, "Tor", filename), os.path.join(dist_path, filename), From 609cf488ebfb35c8769a11454ec4c0a38eefa3e4 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 16:05:17 -0400 Subject: [PATCH 135/154] One more get-tor fix for win64 --- desktop/scripts/get-tor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/desktop/scripts/get-tor.py b/desktop/scripts/get-tor.py index 4ba53ce2..edd47320 100644 --- a/desktop/scripts/get-tor.py +++ b/desktop/scripts/get-tor.py @@ -66,7 +66,7 @@ def get_tor_windows(platform): "libevent-2-1-7.dll", "libevent_core-2-1-7.dll", "libevent_extra-2-1-7.dll", - "libgcc_s_dw2-1.dll", + "libgcc_s_seh-1.dll", "libssl-1_1-x64.dll", "libssp-0.dll", "libwinpthread-1.dll", From e8b388995b610ca7be4e3c185a8d3e956d1d6fa3 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 18:11:50 -0400 Subject: [PATCH 136/154] Switch to python 3.9.13 and switch how python is installed in Windows --- .circleci/config.yml | 36 ++++++++++++++++++++---------------- RELEASE.md | 38 ++++++-------------------------------- desktop/README.md | 2 +- 3 files changed, 27 insertions(+), 49 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e5fa62e..d9e8048b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,10 +17,10 @@ workflows: requires: - test-cli # - test-gui - - build-macos: - requires: - - test-cli - # - test-gui + # - build-macos: + # requires: + # - test-cli + # # - test-gui jobs: test-cli: @@ -90,9 +90,11 @@ jobs: - checkout - run: - name: Install Python 3.9.12 + name: Install Python 3.9.13 (64-bit) command: | - choco install python3 --version=3.9.12 + cd ~\Downloads + Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe -OutFile python-3.9.13-amd64.exe + .\python-3.9.13-amd64.exe /quiet PrependPath=1 - run: name: Install poetry @@ -194,13 +196,15 @@ jobs: - checkout - run: - name: Install Python 3.9.12 (32-bit) + name: Install Python 3.9.13 (32-bit) command: | - choco install python3 --params "/InstallDir32:C:\Python-32bit" --version=3.9.12 + cd ~\Downloads + Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.9.13/python-3.9.13.exe -OutFile python-3.9.13.exe + .\python-3.9.13.exe /quiet PrependPath=1 - run: - name: Install poetry (32-bit) - command: C:\Python-32bit\Scripts\pip install poetry + name: Install poetry + command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - - restore_cache: key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} @@ -208,7 +212,7 @@ jobs: name: Install poetry dependencies command: | cd ~\project\desktop - C:\Python-32bit\Scripts\poetry install + poetry install - save_cache: key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} paths: @@ -220,7 +224,7 @@ jobs: name: Get tor binaries from Tor Browser (32-bit) command: | cd desktop - C:\Python-32bit\Scripts\poetry run python .\scripts\get-tor.py win32 + poetry run python .\scripts\get-tor.py win32 - save_cache: key: get-tor-win32-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} paths: @@ -288,8 +292,8 @@ jobs: name: Build OnionShare command: | cd ~\project\desktop - C:\Python-32bit\Scripts\poetry run python .\setup-freeze.py build - C:\Python-32bit\Scripts\poetry run python .\scripts\build-windows.py cleanup-build + poetry run python .\setup-freeze.py build + poetry run python .\scripts\build-windows.py cleanup-build - run: name: Compress @@ -314,9 +318,9 @@ jobs: brew install go - run: - name: Install Python 3.9.12 + name: Install Python 3.9.13 command: | - wget https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg -O ~/Downloads/python.pkg + wget https://www.python.org/ftp/python/3.9.13/python-3.9.13-macosx10.9.pkg -O ~/Downloads/python.pkg sudo installer -pkg ~/Downloads/python.pkg -target / - run: diff --git a/RELEASE.md b/RELEASE.md index b85e82f3..eb2d9434 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -16,12 +16,6 @@ Before making a release, you must update the version in these places: If you update `flask-socketio`, ensure that you also update the [socket.io.min.js](https://github.com/micahflee/onionshare/blob/develop/cli/onionshare_cli/resources/static/js/socket.io.min.js) file to a version that is [supported](https://flask-socketio.readthedocs.io/en/latest/#version-compatibility) by the updated version of `flask-socketio`. -Use tor binaries from the latest Tor Browser: - -- [ ] `desktop/scripts/get-tor-linux.py` -- [ ] `desktop/scripts/get-tor-osx.py` -- [ ] `desktop/scripts/get-tor-windows.py` - Update the documentation: - [ ] Update all of the documentation in `docs` to cover new features, including taking new screenshots if necessary @@ -100,27 +94,18 @@ snapcraft upload --release=stable onionshare_${VERSION}_amd64.snap ## Windows -Set up the development environment described in desktop `README.md`. +Set up the packaging environment: -- To get `signtool.exe`, install the [Windows SDK](https://developer.microsoft.com/en-us/windows/downloads/windows-sdk/) and add `C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64` to your path. +- Install the [Windows SDK](https://developer.microsoft.com/en-us/windows/downloads/windows-sdk/) and add `C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64` to your path (to get `signtool.exe`). - Go to https://dotnet.microsoft.com/download/dotnet-framework and download and install .NET Framework 3.5 SP1 Runtime. I downloaded `dotnetfx35.exe`. - Go to https://wixtoolset.org/releases/ and download and install WiX toolset. I downloaded `wix311.exe`. Add `C:\Program Files (x86)\WiX Toolset v3.11\bin` to the path. -Build the Windows binaries, delete extra files, codesign, and create an MSI package: - -CircleCI will build the binary and delete extra files, basically running these: - -``` -poetry run python .\setup-freeze.py build -poetry run python .\scripts\build-windows.py cleanup-build -``` - -Find the CircleCI jobs `build-win32` and `build-win64`, switch to the artifacts tab, and download: +CircleCI will build the binaries. Find the CircleCI jobs `build-win32` and `build-win64`, switch to the artifacts tab, and download: - `onionshare-win32.zip` - `onionshare-win64.zip` -Extract these files, then run: +Extract these files, change to the `desktop` folder, and run: ``` poetry run python .\scripts\build-windows.py codesign [onionshare_win32_path] [onionshare_win64_path] @@ -134,22 +119,11 @@ This will create: ## macOS -Set up the development environment described in `README.md`. - -Then build an executable, make it a macOS app bundle, and package it in a dmg. - -CircleCI will build the binary and delete extra files, basically running these: - -```sh -poetry run python ./setup-freeze.py bdist_mac -poetry run python ./scripts/build-macos.py cleanup-build -``` - -Find the CircleCI job `build-macos`, switch to the artifacts tab, and download: +CircleCI will build the binaries. Find the CircleCI job `build-macos`, switch to the artifacts tab, and download: - `onionshare-macos.zip` -Extract this file, then run: +Extract these files, change to the `desktop` folder, and run: ```sh poetry run python ./scripts/build-macos.py codesign [app_path] diff --git a/desktop/README.md b/desktop/README.md index 5a070ee6..26dd0cf7 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -9,7 +9,7 @@ git clone https://github.com/onionshare/onionshare.git cd onionshare/desktop ``` -Make sure you have Python 3 installed. If you're using Windows or macOS, install version 3.9.12 [from python.org](https://www.python.org/downloads/release/python-3912/). For Windows, make sure to check the box to add python to the path on the first page of the installer. +Make sure you have Python 3 installed. If you're using Windows or macOS, install version 3.9.13 [from python.org](https://www.python.org/downloads/release/python-3913/). For Windows, make sure to check the box to add python to the path on the first page of the installer. Make sure you have [poetry](https://python-poetry.org/) installed: From 82a8df604d0cfc9b4b1a2437ec6e958ac0e0595f Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 18:35:47 -0400 Subject: [PATCH 137/154] Use the correct Python --- .circleci/config.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d9e8048b..d238455b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,11 +94,11 @@ jobs: command: | cd ~\Downloads Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe -OutFile python-3.9.13-amd64.exe - .\python-3.9.13-amd64.exe /quiet PrependPath=1 + .\python-3.9.13-amd64.exe /quiet InstallAllUsers=1 TargetDir=C:\Python39 /log Python39-Install.log - run: name: Install poetry - command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - + command: C:\Python39\python -m pip install poetry - restore_cache: key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} @@ -106,7 +106,7 @@ jobs: name: Install poetry dependencies command: | cd C:\Users\circleci\project\desktop - poetry install + C:\Python39\scripts\poetry install - save_cache: key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} paths: @@ -118,7 +118,7 @@ jobs: name: Get tor binaries from Tor Browser (64-bit) command: | cd desktop - poetry run python .\scripts\get-tor.py win64 + C:\Python39\Scripts\poetry run python .\scripts\get-tor.py win64 - save_cache: key: get-tor-win64-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} paths: @@ -176,8 +176,8 @@ jobs: name: Build OnionShare command: | cd ~\project\desktop - poetry run python .\setup-freeze.py build - poetry run python .\scripts\build-windows.py cleanup-build + C:\Python39\Scripts\poetry run python .\setup-freeze.py build + C:\Python39\Scripts\poetry run python .\scripts\build-windows.py cleanup-build - run: name: Compress @@ -200,11 +200,11 @@ jobs: command: | cd ~\Downloads Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.9.13/python-3.9.13.exe -OutFile python-3.9.13.exe - .\python-3.9.13.exe /quiet PrependPath=1 + .\python-3.9.13-amd64.exe /quiet InstallAllUsers=1 TargetDir=C:\Python39 /log Python39-Install.log - run: name: Install poetry - command: (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - + command: C:\Python39\python -m pip install poetry - restore_cache: key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} @@ -212,7 +212,7 @@ jobs: name: Install poetry dependencies command: | cd ~\project\desktop - poetry install + C:\Python39\Scripts\poetry install - save_cache: key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} paths: @@ -224,7 +224,7 @@ jobs: name: Get tor binaries from Tor Browser (32-bit) command: | cd desktop - poetry run python .\scripts\get-tor.py win32 + C:\Python39\Scripts\poetry run python .\scripts\get-tor.py win32 - save_cache: key: get-tor-win32-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} paths: @@ -292,8 +292,8 @@ jobs: name: Build OnionShare command: | cd ~\project\desktop - poetry run python .\setup-freeze.py build - poetry run python .\scripts\build-windows.py cleanup-build + C:\Python39\Scripts\poetry run python .\setup-freeze.py build + C:\Python39\Scripts\poetry run python .\scripts\build-windows.py cleanup-build - run: name: Compress From 8eef25f2ff3ec2768020fd0d29632eacf5c7bac2 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 18:38:38 -0400 Subject: [PATCH 138/154] Oops, install 32-bit python in build-win32 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d238455b..f9646ae9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -200,7 +200,7 @@ jobs: command: | cd ~\Downloads Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.9.13/python-3.9.13.exe -OutFile python-3.9.13.exe - .\python-3.9.13-amd64.exe /quiet InstallAllUsers=1 TargetDir=C:\Python39 /log Python39-Install.log + .\python-3.9.13.exe /quiet InstallAllUsers=1 TargetDir=C:\Python39 /log Python39-Install.log - run: name: Install poetry From 5ce4af6918a5f72d6624ec8e3b5713162730de0a Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 19:32:37 -0400 Subject: [PATCH 139/154] Try removing python install logs --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f9646ae9..02f6e9d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,7 +94,7 @@ jobs: command: | cd ~\Downloads Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe -OutFile python-3.9.13-amd64.exe - .\python-3.9.13-amd64.exe /quiet InstallAllUsers=1 TargetDir=C:\Python39 /log Python39-Install.log + .\python-3.9.13-amd64.exe /quiet InstallAllUsers=1 TargetDir=C:\Python39 - run: name: Install poetry @@ -200,7 +200,7 @@ jobs: command: | cd ~\Downloads Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.9.13/python-3.9.13.exe -OutFile python-3.9.13.exe - .\python-3.9.13.exe /quiet InstallAllUsers=1 TargetDir=C:\Python39 /log Python39-Install.log + .\python-3.9.13.exe /quiet InstallAllUsers=1 TargetDir=C:\Python39 - run: name: Install poetry From 93e129983a070ea5cb186a6e66898e3e76ab4d12 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 19:53:15 -0400 Subject: [PATCH 140/154] Try compressing macos version with tar --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 02f6e9d7..6e6b5e1b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,10 +17,10 @@ workflows: requires: - test-cli # - test-gui - # - build-macos: - # requires: - # - test-cli - # # - test-gui + - build-macos: + requires: + - test-cli + # - test-gui jobs: test-cli: @@ -413,7 +413,7 @@ jobs: name: Compress command: | cd ~/project/desktop/build - zip -r ~/onionshare-macos.zip OnionShare.app + tar -cvfx ~/onionshare-macos.tar.gz OnionShare.app - store_artifacts: - path: ~/onionshare-macos.zip + path: ~/onionshare-macos.tar.gz From e4114bf01da6223cc3d5f7f0e4970fc27dd4e645 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 22:57:13 -0400 Subject: [PATCH 141/154] Fix windows codesigning script --- desktop/scripts/build-windows.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/desktop/scripts/build-windows.py b/desktop/scripts/build-windows.py index a0cb3994..c7a6d345 100644 --- a/desktop/scripts/build-windows.py +++ b/desktop/scripts/build-windows.py @@ -590,7 +590,6 @@ def codesign(win32_path, win64_path): "onionshare", "resources", "tor", - "Tor", "meek-client.exe", ) ) @@ -601,7 +600,6 @@ def codesign(win32_path, win64_path): "onionshare", "resources", "tor", - "Tor", "obfs4proxy.exe", ) ) @@ -612,7 +610,6 @@ def codesign(win32_path, win64_path): "onionshare", "resources", "tor", - "Tor", "snowflake-client.exe", ) ) From 5cbb2b3be4f392ec501b3dba0b7dc2804f69374a Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 22:57:33 -0400 Subject: [PATCH 142/154] Fix tarball compression, and also try to fix CircleCI caches --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e6b5e1b..b12a4e6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -358,26 +358,26 @@ jobs: - run: name: Build obfs4proxy command: | - if [[ -f "~/project/desktop/onionshare/resources/tor/obfs4proxy" ]]; then + if [[ -f "/Users/distiller/project/desktop/onionshare/resources/tor/obfs4proxy" ]]; then echo "obfs4proxy already built" else - cd ~/project/desktop + cd /Users/distiller/project/desktop ./scripts/build-pt-obfs4proxy.sh fi - save_cache: key: build-macos-obfs4proxy-{{ checksum "~/project/desktop/scripts/build-pt-obfs4proxy.sh" }}-{{ .Environment.CACHE_VERSION }} paths: - - ~/project/desktop/onionshare/resources/tor/obfs4proxy + - /Users/distiller/project/desktop/onionshare/resources/tor/obfs4proxy - restore_cache: key: build-macos-snowflake-{{ checksum "~/project/desktop/scripts/build-pt-snowflake.sh" }}-{{ .Environment.CACHE_VERSION }} - run: name: Build snowflake command: | - if [[ -f "~/project/desktop/onionshare/resources/tor/snowflake-client" ]]; then + if [[ -f "/Users/distiller/project/desktop/onionshare/resources/tor/snowflake-client" ]]; then echo "snowflake already built" else - cd ~/project/desktop + cd /Users/distiller/project/desktop ./scripts/build-pt-snowflake.sh fi - save_cache: @@ -390,10 +390,10 @@ jobs: - run: name: Build meek command: | - if [[ -f "~/project/desktop/onionshare/resources/tor/meek-client" ]]; then + if [[ -f "/Users/distiller/project/desktop/onionshare/resources/tor/meek-client" ]]; then echo "meek already built" else - cd ~/project/desktop + cd /Users/distiller/project/desktop ./scripts/build-pt-meek.sh fi - save_cache: @@ -413,7 +413,7 @@ jobs: name: Compress command: | cd ~/project/desktop/build - tar -cvfx ~/onionshare-macos.tar.gz OnionShare.app + tar -czvf ~/onionshare-macos.tar.gz OnionShare.app - store_artifacts: path: ~/onionshare-macos.tar.gz From a54b6beb12436c3b042387d5f193526d7fea4767 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 23:04:28 -0400 Subject: [PATCH 143/154] Wait for Python to finish installing in Windows before finishing the task --- .circleci/config.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b12a4e6a..3f7a6b20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,6 +95,15 @@ jobs: cd ~\Downloads Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe -OutFile python-3.9.13-amd64.exe .\python-3.9.13-amd64.exe /quiet InstallAllUsers=1 TargetDir=C:\Python39 + while($true) { + if ((Test-Path -Path C:\Python39\python.exe) -eq $True) { + Write-Output "Python is installed" + break + } else { + Write-Output "Waiting for Python to finish installing ..." + Start-Sleep -Seconds 2 + } + } - run: name: Install poetry @@ -201,6 +210,15 @@ jobs: cd ~\Downloads Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.9.13/python-3.9.13.exe -OutFile python-3.9.13.exe .\python-3.9.13.exe /quiet InstallAllUsers=1 TargetDir=C:\Python39 + while($true) { + if ((Test-Path -Path C:\Python39\python.exe) -eq $True) { + Write-Output "Python is installed" + break + } else { + Write-Output "Waiting for Python to finish installing ..." + Start-Sleep -Seconds 2 + } + } - run: name: Install poetry From 1116a407ea79d68c47bb29074f52ff855bac56db Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 23:09:00 -0400 Subject: [PATCH 144/154] Speed up macOS build by skipping homebrew (downloading the Go pkg, and using curl instead of wget) --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3f7a6b20..fa7970f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -330,15 +330,15 @@ jobs: steps: - checkout - run: - name: Install Homebrew dependencies + name: Install Go 1.18.3 command: | - brew install wget - brew install go + curl https://go.dev/dl/go1.18.3.darwin-amd64.pkg --output ~/Downloads/go.pkg + sudo installer -pkg ~/Downloads/go.pkg -target / - run: name: Install Python 3.9.13 command: | - wget https://www.python.org/ftp/python/3.9.13/python-3.9.13-macosx10.9.pkg -O ~/Downloads/python.pkg + curl https://www.python.org/ftp/python/3.9.13/python-3.9.13-macosx10.9.pkg --output ~/Downloads/python.pkg sudo installer -pkg ~/Downloads/python.pkg -target / - run: From fef5e3b2ec7b57a622e41a421fca14de766554c1 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 23:18:25 -0400 Subject: [PATCH 145/154] Follow redirects when downloading with curl --- .circleci/config.yml | 4 ++-- RELEASE.md | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fa7970f8..00ac65fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -332,13 +332,13 @@ jobs: - run: name: Install Go 1.18.3 command: | - curl https://go.dev/dl/go1.18.3.darwin-amd64.pkg --output ~/Downloads/go.pkg + curl -L https://go.dev/dl/go1.18.3.darwin-amd64.pkg --output ~/Downloads/go.pkg sudo installer -pkg ~/Downloads/go.pkg -target / - run: name: Install Python 3.9.13 command: | - curl https://www.python.org/ftp/python/3.9.13/python-3.9.13-macosx10.9.pkg --output ~/Downloads/python.pkg + curl -L https://www.python.org/ftp/python/3.9.13/python-3.9.13-macosx10.9.pkg --output ~/Downloads/python.pkg sudo installer -pkg ~/Downloads/python.pkg -target / - run: diff --git a/RELEASE.md b/RELEASE.md index eb2d9434..ece4851f 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -119,6 +119,10 @@ This will create: ## macOS +Set up the packaging environment: + +- Install create-dmg: `brew install create-dmg` + CircleCI will build the binaries. Find the CircleCI job `build-macos`, switch to the artifacts tab, and download: - `onionshare-macos.zip` From 21a4f0ee01b9f8f1d9c1ca01861fb967e12a9e68 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 23:20:53 -0400 Subject: [PATCH 146/154] Try quoting paths to make Windows caches works --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 00ac65fa..560ccf0e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -138,7 +138,7 @@ jobs: - run: name: Build obfs4proxy command: | - if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe) -eq $True) { + if ((Test-Path -Path 'C:\Users\circleci\project\desktop\onionshare\resources\tor\obfs4proxy.exe') -eq $True) { Write-Output "obfs4proxy already built" } else { cd C:\Users\circleci\project\desktop @@ -154,7 +154,7 @@ jobs: - run: name: Build snowflake-client command: | - if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe) -eq $True) { + if ((Test-Path -Path 'C:\Users\circleci\project\desktop\onionshare\resources\tor\snowflake-client.exe') -eq $True) { Write-Output "snowflake already built" } else { cd C:\Users\circleci\project\desktop @@ -170,7 +170,7 @@ jobs: - run: name: Build meek-client command: | - if ((Test-Path -Path C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe) -eq $True) { + if ((Test-Path -Path 'C:\Users\circleci\project\desktop\onionshare\resources\tor\meek-client.exe') -eq $True) { Write-Output "snowflake already built" } else { cd C:\Users\circleci\project\desktop From 7173a4b9d35cd4a297107b6134471f310a7a2aba Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 23:36:08 -0400 Subject: [PATCH 147/154] Version bump to 2.6.dev1 --- CHANGELOG.md | 4 ++++ cli/onionshare_cli/resources/version.txt | 2 +- cli/pyproject.toml | 2 +- desktop/org.onionshare.OnionShare.appdata.xml | 2 +- desktop/pyproject.toml | 2 +- desktop/setup.py | 2 +- docs/source/conf.py | 2 +- snap/snapcraft.yaml | 4 ++-- 8 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce455a45..33c0efbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # OnionShare Changelog +## 2.6 + +* TODO + ## 2.5 * Security fix: Sanitize the path parameter in History item widget to be plain text diff --git a/cli/onionshare_cli/resources/version.txt b/cli/onionshare_cli/resources/version.txt index 68151b2e..2a15fcc8 100644 --- a/cli/onionshare_cli/resources/version.txt +++ b/cli/onionshare_cli/resources/version.txt @@ -1 +1 @@ -2.5 \ No newline at end of file +2.6.dev1 \ No newline at end of file diff --git a/cli/pyproject.toml b/cli/pyproject.toml index 5ddaf565..1983cf59 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "onionshare_cli" -version = "2.5" +version = "2.6" description = "OnionShare lets you securely and anonymously send and receive files. It works by starting a web server, making it accessible as a Tor onion service, and generating an unguessable web address so others can download files from you, or upload files to you. It does _not_ require setting up a separate server or using a third party file-sharing service." authors = ["Micah Lee "] license = "GPLv3+" diff --git a/desktop/org.onionshare.OnionShare.appdata.xml b/desktop/org.onionshare.OnionShare.appdata.xml index d6afc314..f7ba36a8 100644 --- a/desktop/org.onionshare.OnionShare.appdata.xml +++ b/desktop/org.onionshare.OnionShare.appdata.xml @@ -24,6 +24,6 @@ micah@micahflee.com - + diff --git a/desktop/pyproject.toml b/desktop/pyproject.toml index d6809d99..ece0ddb3 100644 --- a/desktop/pyproject.toml +++ b/desktop/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "onionshare" -version = "2.5" +version = "2.6" description = "OnionShare lets you securely and anonymously send and receive files. It works by starting a web server, making it accessible as a Tor onion service, and generating an unguessable web address so others can download files from you, or upload files to you. It does _not_ require setting up a separate server or using a third party file-sharing service." authors = ["Micah Lee "] license = "GPLv3+" diff --git a/desktop/setup.py b/desktop/setup.py index 24359907..44cfb030 100644 --- a/desktop/setup.py +++ b/desktop/setup.py @@ -3,7 +3,7 @@ import setuptools # The version must be hard-coded because Snapcraft won't have access to ../cli -version = "2.5" +version = "2.6" setuptools.setup( name="onionshare", diff --git a/docs/source/conf.py b/docs/source/conf.py index 36679497..37b7acb3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,6 +1,6 @@ project = "OnionShare" author = copyright = "Micah Lee, et al." -version = release = "2.5" +version = release = "2.6" extensions = ["sphinx_rtd_theme"] templates_path = ["_templates"] diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index acdc3fd2..e0cac6d1 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: onionshare base: core18 -version: '2.5' +version: "2.6" summary: Securely and anonymously share files, host websites, and chat using Tor description: | OnionShare lets you securely and anonymously send and receive files. It works by starting @@ -15,7 +15,7 @@ apps: onionshare: common-id: org.onionshare.OnionShare command: onionshare - extensions: [ gnome-3-34 ] + extensions: [gnome-3-34] plugs: - desktop - home From ac745dd0417b771efe8be70767e97385e28ebf6b Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 23:36:34 -0400 Subject: [PATCH 148/154] Fix PT Windows build scripts to move the binary to the right folder --- desktop/scripts/build-pt-meek.ps1 | 2 +- desktop/scripts/build-pt-obfs4proxy.ps1 | 2 +- desktop/scripts/build-pt-snowflake.ps1 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/desktop/scripts/build-pt-meek.ps1 b/desktop/scripts/build-pt-meek.ps1 index 58a7366a..99ff8de6 100644 --- a/desktop/scripts/build-pt-meek.ps1 +++ b/desktop/scripts/build-pt-meek.ps1 @@ -6,4 +6,4 @@ git clone https://git.torproject.org/pluggable-transports/meek.git cd meek git checkout $MEEK_TAG go build .\meek-client -Move-Item -Path .\meek-client.exe -Destination ..\onionshare\resources\tor\meek-client.exe +Move-Item -Path .\meek-client.exe -Destination ..\..\..\onionshare\resources\tor\meek-client.exe diff --git a/desktop/scripts/build-pt-obfs4proxy.ps1 b/desktop/scripts/build-pt-obfs4proxy.ps1 index e4688c93..fa760b85 100644 --- a/desktop/scripts/build-pt-obfs4proxy.ps1 +++ b/desktop/scripts/build-pt-obfs4proxy.ps1 @@ -6,4 +6,4 @@ git clone https://gitlab.com/yawning/obfs4 cd obfs4 git checkout $OBFS4PROXY_TAG go build .\obfs4proxy -Move-Item -Path .\obfs4proxy.exe -Destination ..\onionshare\resources\tor\obfs4proxy.exe +Move-Item -Path .\obfs4proxy.exe -Destination ..\..\..\onionshare\resources\tor\obfs4proxy.exe diff --git a/desktop/scripts/build-pt-snowflake.ps1 b/desktop/scripts/build-pt-snowflake.ps1 index 5d15759d..51a127c9 100644 --- a/desktop/scripts/build-pt-snowflake.ps1 +++ b/desktop/scripts/build-pt-snowflake.ps1 @@ -6,4 +6,4 @@ git clone https://git.torproject.org/pluggable-transports/snowflake.git cd snowflake git checkout $SNOWFLAKE_TAG go build .\client -Move-Item -Path .\client.exe -Destination ..\onionshare\resources\tor\snowflake-client.exe +Move-Item -Path .\client.exe -Destination ..\..\..\onionshare\resources\tor\snowflake-client.exe From 6a5ac6974184610871844f6f3040f39f88bd330a Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 23:58:41 -0400 Subject: [PATCH 149/154] cx_Freeze cannot handle dev version strings --- desktop/setup-freeze.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/desktop/setup-freeze.py b/desktop/setup-freeze.py index ccb5a541..63920e4d 100644 --- a/desktop/setup-freeze.py +++ b/desktop/setup-freeze.py @@ -108,6 +108,10 @@ if platform.system() == "Darwin" or platform.system() == "Linux": # Discover the version with open(os.path.join("..", "cli", "onionshare_cli", "resources", "version.txt")) as f: version = f.read().strip() + # change a version like 2.6.dev1 to just 2.6, for cx_Freeze's sake + last_digit = version[-1] + if version.endswith(f".dev{last_digit}"): + version = version[0:-4] # Build From 6679e9c4dc6f82ba8f08caa692fd355d6b1b2ff0 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Mon, 20 Jun 2022 00:18:49 -0400 Subject: [PATCH 150/154] Fix sanitizing version string --- desktop/setup-freeze.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/desktop/setup-freeze.py b/desktop/setup-freeze.py index 63920e4d..83959a2b 100644 --- a/desktop/setup-freeze.py +++ b/desktop/setup-freeze.py @@ -111,8 +111,7 @@ with open(os.path.join("..", "cli", "onionshare_cli", "resources", "version.txt" # change a version like 2.6.dev1 to just 2.6, for cx_Freeze's sake last_digit = version[-1] if version.endswith(f".dev{last_digit}"): - version = version[0:-4] - + version = version[0:-5] # Build include_files = [(os.path.join("..", "LICENSE"), "LICENSE")] From 5e600e5f2d4b12eeb14e1735309f77babef7ec55 Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 21:50:32 -0700 Subject: [PATCH 151/154] Sanitize version string for WiX too --- desktop/scripts/build-windows.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/desktop/scripts/build-windows.py b/desktop/scripts/build-windows.py index c7a6d345..76634696 100644 --- a/desktop/scripts/build-windows.py +++ b/desktop/scripts/build-windows.py @@ -177,6 +177,10 @@ def msi_package(build_path, msi_path, product_update_code): ) with open(version_filename) as f: version = f.read().strip() + # change a version like 2.6.dev1 to just 2.6, for cx_Freeze's sake + last_digit = version[-1] + if version.endswith(f".dev{last_digit}"): + version = version[0:-5] data = { "id": "TARGETDIR", From f572623528b643926e00837647e692ba865fd7be Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Sun, 19 Jun 2022 21:51:06 -0700 Subject: [PATCH 152/154] Only make builds if GUI tests pass too --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 560ccf0e..ac0cadd0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,15 +12,15 @@ workflows: - build-win64: requires: - test-cli - # - test-gui + - test-gui - build-win32: requires: - test-cli - # - test-gui + - test-gui - build-macos: requires: - test-cli - # - test-gui + - test-gui jobs: test-cli: From 033c0cb56f3f300b00ad7b51fec7bef13d4ff3fd Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Mon, 20 Jun 2022 09:17:27 -0700 Subject: [PATCH 153/154] Update poetry.lock --- desktop/poetry.lock | 68 +++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/desktop/poetry.lock b/desktop/poetry.lock index 58d430af..65b955b8 100644 --- a/desktop/poetry.lock +++ b/desktop/poetry.lock @@ -62,7 +62,7 @@ python-versions = "*" [[package]] name = "certifi" -version = "2022.5.18.1" +version = "2022.6.15" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -104,7 +104,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "main" optional = false @@ -112,18 +112,24 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "cx-freeze" -version = "6.10" +version = "6.11.0" description = "Create standalone executables from Python scripts" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -cx-logging = {version = ">=3.0", markers = "sys_platform == \"win32\""} +cx-logging = {version = ">=3.0", markers = "sys_platform == \"win32\" and python_version < \"3.10\""} importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -lief = {version = ">=0.11.5", markers = "sys_platform == \"win32\" and python_version < \"3.10\""} +lief = {version = ">=0.11.5", markers = "sys_platform == \"win32\""} +packaging = ">=21.0" patchelf = {version = ">=0.12", markers = "sys_platform == \"linux\""} +[package.extras] +dev = ["bump2version (>=1.0.1)", "pre-commit (>=2.17.0)", "pylint (>=2.13.0)", "cibuildwheel (==2.6.0)"] +doc = ["sphinx (==4.5.0)", "sphinx-rtd-theme (==1.0.0)"] +test = ["nose (==1.3.7)", "pygments (==2.11.2)", "pytest (>=7.0.1)", "pytest-cov (==3.0.0)", "pytest-mock (>=3.6.1)", "pytest-timeout (>=1.4.2)"] + [[package]] name = "cx-logging" version = "3.0" @@ -323,7 +329,7 @@ python-versions = "*" [[package]] name = "onionshare-cli" -version = "2.5" +version = "2.6" description = "OnionShare lets you securely and anonymously send and receive files. It works by starting a web server, making it accessible as a Tor onion service, and generating an unguessable web address so others can download files from you, or upload files to you. It does _not_ require setting up a separate server or using a third party file-sharing service." category = "main" optional = false @@ -353,7 +359,7 @@ url = "../cli" name = "packaging" version = "21.3" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -452,7 +458,7 @@ tests = ["pytest (>=3.2.1,!=3.3.0)", "hypothesis (>=3.27.0)"] name = "pyparsing" version = "3.0.7" description = "Python parsing module" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -706,7 +712,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "1.1" python-versions = ">=3.6.2,<3.11" -content-hash = "60276ce053d95d34415d8bee002e2c39f410ad5c97f2d1948e7ddbdc5da9cc56" +content-hash = "80aa2819ec2d5e4dd5acb42f178c196776e05fb7efa5d1b99e5c2626c2208cd9" [metadata.files] atomicwrites = [ @@ -750,8 +756,8 @@ cepa = [ {file = "cepa-1.8.3.tar.gz", hash = "sha256:1dc6f0b324d37a2ed2ca274648ece8fd2c96a1d2f440f58c0ca17afd4b5ede7a"}, ] certifi = [ - {file = "certifi-2022.5.18.1-py3-none-any.whl", hash = "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"}, - {file = "certifi-2022.5.18.1.tar.gz", hash = "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7"}, + {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, + {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, ] cffi = [ {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, @@ -814,21 +820,35 @@ click = [ {file = "click-8.0.4.tar.gz", hash = "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb"}, ] colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] cx-freeze = [ - {file = "cx_Freeze-6.10-cp310-cp310-win32.whl", hash = "sha256:770e911f70b48e4f309a47e02aaa7e1f6dc659fd1c486f3eebe5c9e9fe70aec2"}, - {file = "cx_Freeze-6.10-cp310-cp310-win_amd64.whl", hash = "sha256:8d936a872c124c2e2a40da2e898e4fa0d225a165ab606217f75136c14ee8e673"}, - {file = "cx_Freeze-6.10-cp36-cp36m-win32.whl", hash = "sha256:2be07ffce2dba23618cf736476df6f9837d9887c85f4f92fb3fbfcd090de041a"}, - {file = "cx_Freeze-6.10-cp36-cp36m-win_amd64.whl", hash = "sha256:59b4cb77f5c82613efbec03815b952a6fe73d4a1805bb370d57a8a1130a181ca"}, - {file = "cx_Freeze-6.10-cp37-cp37m-win32.whl", hash = "sha256:f03928cbcf8282e688dc1f9fa421044c24f5b7492a7b0fe7f45d5cc6f24c1ebf"}, - {file = "cx_Freeze-6.10-cp37-cp37m-win_amd64.whl", hash = "sha256:9c4b491affd10065bcdd20dc0ab5239f787affe6d61857811a937ed266faa213"}, - {file = "cx_Freeze-6.10-cp38-cp38-win32.whl", hash = "sha256:1438db6d59be86e8483ea2bf2ff2f81374ca63d8a1d3a84706173cc1b98dd46f"}, - {file = "cx_Freeze-6.10-cp38-cp38-win_amd64.whl", hash = "sha256:ec752333fa2e40347902730662785da0ec4576cec976552784a6f60a7a07b45d"}, - {file = "cx_Freeze-6.10-cp39-cp39-win32.whl", hash = "sha256:08a537681dcd1bcde9742584e91d86ef001fe798386f2834a904652d5b2a468a"}, - {file = "cx_Freeze-6.10-cp39-cp39-win_amd64.whl", hash = "sha256:a4d2cb00eec6bc72a419370b5b0b5d0f3adf4d8417eb89228981d470cf4c6af6"}, - {file = "cx_Freeze-6.10.tar.gz", hash = "sha256:e5b71bf57b9881ac142fbebeae2c8b0d3294b56f6e48ab64032321e3b1a2ba27"}, + {file = "cx_Freeze-6.11.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0fa63ae171ee34dd644331ad9cba0e1627036a11ef23b0ce8569229e4546762f"}, + {file = "cx_Freeze-6.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8274f4d1c696e6e824f98423b4afa618db4eadecb554d2948c364cb524e46bde"}, + {file = "cx_Freeze-6.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1897bb479e9ad1d5ff339eb2f59009f618e4651e33f1dd695500b36c29045c75"}, + {file = "cx_Freeze-6.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9a3f3d7934b93f1a73579f4b19fc7ee37556bee46768e6e8c54ac0d469a8918"}, + {file = "cx_Freeze-6.11.0-cp310-cp310-win32.whl", hash = "sha256:dfbb70832ecb461798d8d19ca110c1c2f8a622475d22ec49c143a8abf19339b0"}, + {file = "cx_Freeze-6.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:09ccf1c96354d11f694f1a9f6887b069d40fcf853b1b819d2308747d62efebb7"}, + {file = "cx_Freeze-6.11.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:44cb609f7849176e06fe12bb52459de31a3b68fc1d72279e9b081f18c9276c7a"}, + {file = "cx_Freeze-6.11.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:986e666a8473e699f2a4ba689cfa017a4955ed79a56e2093df55af8ca3bff56f"}, + {file = "cx_Freeze-6.11.0-cp36-cp36m-win32.whl", hash = "sha256:07428f54a17cb6e8a889030a357edb6398c1c90b998564b90399e028f538930b"}, + {file = "cx_Freeze-6.11.0-cp36-cp36m-win_amd64.whl", hash = "sha256:83bb02fa03e0c52fdb078b064cb1c17cf6c0410bd74e768ade10d1708c324b74"}, + {file = "cx_Freeze-6.11.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:630d0d0d3bcdc69df9af0cc02d84a9ab4f9305bc74513058e30042bfc379dd20"}, + {file = "cx_Freeze-6.11.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea2f4f5bea120f88d53ca32a71ccf1b6758dc82ed468f944338ed8535eec550e"}, + {file = "cx_Freeze-6.11.0-cp37-cp37m-win32.whl", hash = "sha256:70835e9d6475fb9a7f207d74bd0f3be1bd2d59b6cd09941e032ae512e7fdedee"}, + {file = "cx_Freeze-6.11.0-cp37-cp37m-win_amd64.whl", hash = "sha256:0c141cba6f9e9e9923b1cfbbb0cb44655848bf50c57249d8058fe388c18116f3"}, + {file = "cx_Freeze-6.11.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:04453a2de82f81356be1c183ee0ab2fbf024791451970dbdde4f9f1bf6301b6a"}, + {file = "cx_Freeze-6.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4324f1abf6e0d0a017e9fabb0f3805acd08f5b3b7eb059395e08e7ff2b6e9b83"}, + {file = "cx_Freeze-6.11.0-cp38-cp38-win32.whl", hash = "sha256:81cacbc22c0f3be8e3807b9690c73eb682b16e5a2358f6bd213af6bf88402061"}, + {file = "cx_Freeze-6.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:c2315a1634108fc94398bef3ef63435d53c8ecd1cbcce9e05c4bcca61a833d62"}, + {file = "cx_Freeze-6.11.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f4da77d44b910d4b18fbe95a487ddc081ebe5cd4f35971b496c1fff0420cf495"}, + {file = "cx_Freeze-6.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:868837ca628fe9501c317e475f68d04f442f98e6174cb77b7ec413061e3613d7"}, + {file = "cx_Freeze-6.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b92dda05bd7d5c1a2632de837ea738d73070af6effc035a016ab820ee1ec51a"}, + {file = "cx_Freeze-6.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:920e6ff71b7fe1dd77d7a01b7de04a7856ba4803964ce2acab1962d2212a2746"}, + {file = "cx_Freeze-6.11.0-cp39-cp39-win32.whl", hash = "sha256:81a6a56b8ad83635a441eb0049cff65993c53fcf7c46f49b4cf31998facb24d6"}, + {file = "cx_Freeze-6.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:ee81a20340f830182fba3e42b32a0fb1e455324d6d6c3f6c3b3020d619e04c2d"}, + {file = "cx_Freeze-6.11.0.tar.gz", hash = "sha256:c461fc3a33cbdfc2b6bbdd02cb0ea048d496e9e33067489cae778d929379bee8"}, ] cx-logging = [ {file = "cx_Logging-3.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:9fcd297e5c51470521c47eff0f86ba844aeca6be97e13c3e2114ebdf03fa3c96"}, From 9a1d56d6cbd868a7c1a94b78045d2aa3401f115b Mon Sep 17 00:00:00 2001 From: Micah Lee Date: Mon, 20 Jun 2022 09:24:54 -0700 Subject: [PATCH 154/154] Don't cache poetry deps, since this is causing issues --- .circleci/config.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac0cadd0..57e54f4c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -109,17 +109,17 @@ jobs: name: Install poetry command: C:\Python39\python -m pip install poetry - - restore_cache: - key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + # - restore_cache: + # key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - run: name: Install poetry dependencies command: | cd C:\Users\circleci\project\desktop C:\Python39\scripts\poetry install - - save_cache: - key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - paths: - - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs + # - save_cache: + # key: build-win64-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + # paths: + # - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs - restore_cache: key: get-tor-win64-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }} @@ -224,17 +224,17 @@ jobs: name: Install poetry command: C:\Python39\python -m pip install poetry - - restore_cache: - key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + # - restore_cache: + # key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - run: name: Install poetry dependencies command: | cd ~\project\desktop C:\Python39\Scripts\poetry install - - save_cache: - key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} - paths: - - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs + # - save_cache: + # key: build-win32-desktop-poetry-deps-{{ checksum "~/project/desktop/poetry.lock" }}-{{ .Environment.CACHE_VERSION }} + # paths: + # - C:\Users\circleci\AppData\Local\pypoetry\Cache\virtualenvs - restore_cache: key: get-tor-win32-{{ checksum "desktop/scripts/get-tor.py" }}-{{ .Environment.CACHE_VERSION }}