From 11262cbcf2933f5fe9f32dab5e5fe31845484533 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Fri, 7 Feb 2025 11:42:15 +1100 Subject: [PATCH] Add custom script to fetch golang dependencies and update instructions --- RELEASE.md | 18 +- flatpak/golang-to-requirements.py | 272 ++++++++++++++++++++++++++++++ 2 files changed, 281 insertions(+), 9 deletions(-) create mode 100755 flatpak/golang-to-requirements.py diff --git a/RELEASE.md b/RELEASE.md index 71ef5d3d..fbb0ccf5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -93,17 +93,17 @@ With every commit to the `main` branch, Snapcraft's CI should trigger builds. Ma In `flatpak/org.onionshare.OnionShare.yaml`: -- [ ] Update `tor` and `libevent` -- [ ] Update `obfs4proxy`, `meek-client`, and `snowflake-client` dependencies, if necessary using [this tool](https://github.com/micahflee/flatpak-builder-tools/tree/fix-go/go): +- [ ] Update `tor` and `libevent` +- [ ] Update `obfs4proxy`, `meek-client`, and `snowflake-client` dependencies, if necessary using the script `golang-to-requirements.py` in the `flatpak` folder, like this: ```sh - cd flatpak-builder-tools/go - - # For each these, incorporate the output into the Flatpak manifest - # Make sure to update the version numbers - ./flatpak-go-deps.py gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/meek.git/meek-client@v0.38.0 - ./flatpak-go-deps.py gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git/client@v2.6.0 - ./flatpak-go-deps.py gitlab.com/yawning/obfs4.git/obfs4proxy@obfs4proxy-0.0.14 + cd flatpak + ./golang-to-requirements.py --url gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git --name snowflake --version v2.10.1 --folder client + ./golang-to-requirements.py --url gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/meek.git --name meek-client --version v0.38.0 --folder meek-client + ./golang-to-requirements.py --url gitlab.com/yawning/obfs4.git --name obfs4proxy --version obfs4proxy-0.0.14 ``` + + This will take a long time as it clones each dependency (particularly that of the snowflake-client which has a lot of dependencies). + Merge the output of each of these commands into the Flatpak manifest. - [ ] Update the Python dependencies. This is super hacky. You need to use both the poetry and pip parts of [this tool](https://github.com/flatpak/flatpak-builder-tools), but the version from [this PR](https://github.com/flatpak/flatpak-builder-tools/pull/353): ```sh diff --git a/flatpak/golang-to-requirements.py b/flatpak/golang-to-requirements.py new file mode 100755 index 00000000..735b0cfd --- /dev/null +++ b/flatpak/golang-to-requirements.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import argparse +import subprocess +import sys +import json +import os +import re +import tempfile + +import yaml +import requests +from bs4 import BeautifulSoup + + +def parse_args(): + """ + Parse arguments to the script + """ + parser = argparse.ArgumentParser(description="Extract dependencies for Golang") + parser.add_argument("--url", dest="url", required=True, help="Pass repo URL") + parser.add_argument( + "--name", dest="name", required=True, help="Pass repo short name" + ) + parser.add_argument("--version", default="", dest="version", help="Pass tag name") + parser.add_argument( + "--folder", default="", dest="folder", help="Pass optional folder name" + ) + return parser.parse_args() + + +def extract_commit_id(module_version): + # Regex to match formats like: v0.0.0-20190819201941-24fa4b261c55 + complex_format_regex = re.compile( + r"v\d+\.\d+\.\d+-\d{14}-(?P[a-fA-F0-9]{12,40})" + ) + + match = complex_format_regex.search(module_version) + if match: + return match.group("commit") + + # If the version is just a simple version like v1.4.0 or v0.13.0, return None + return None + + +def get_commit_id_from_git( + git_url, + version=None, + short_commit_id=None, +): + # If short_commit_id is provided, simply expand it + if short_commit_id: + print( + f"✨ Cloning {git_url} to find long commit ID version of {short_commit_id}" + ) + with tempfile.TemporaryDirectory() as tmp_dir: + subprocess.run(["git", "clone", "--bare", git_url, tmp_dir], check=True) + result = subprocess.run( + ["git", "rev-parse", short_commit_id], + cwd=tmp_dir, + check=True, + capture_output=True, + text=True, + ) + commit_id = result.stdout.strip() + print(f"✨ Found commit ID: {commit_id}") + return commit_id + + # If it's a GitHub URL, use the GitHub API + if "github.com" in git_url: + repo_parts = git_url.replace("https://github.com/", "").split("/") + if len(repo_parts) == 2: + owner, repo = repo_parts + tag_url = ( + f"https://api.github.com/repos/{owner}/{repo}/git/refs/tags/{version}" + ) + + response = requests.get(tag_url) + if response.status_code == 200: + json_data = response.json() + commit_id = json_data["object"]["sha"] + print(f"✨ Used GitHub API to find commit ID: {commit_id}") + return commit_id + + # If it's a GitLab URL, use the GitLab API + elif "gitlab.com" in git_url: + repo_parts = ( + git_url.replace("https://gitlab.com/", "").rstrip(".git").split("/") + ) + if len(repo_parts) >= 2: + tag_url = f"https://gitlab.com/api/v4/projects/{'%2F'.join(repo_parts)}/repository/tags/{version}" + + response = requests.get(tag_url) + if response.status_code == 200: + json_data = response.json() + commit_id = json_data["commit"]["id"] + print(f"✨ Used GitHub API to find commit ID: {commit_id}") + return commit_id + + # Otherwise, clone the git repo to find the commit id + with tempfile.TemporaryDirectory() as tmp_dir: + try: + if version: + print(f"✨ Cloning {git_url}@{version} to find commit ID") + subprocess.run( + ["git", "clone", "--bare", "-b", version, git_url, tmp_dir], + check=True, + ) + else: + print(f"✨ Cloning {git_url} to find commit ID") + subprocess.run(["git", "clone", "--bare", git_url, tmp_dir], check=True) + except subprocess.CalledProcessError: + # If cloning with a specific tag fails, fall back to default branch + if version: + print( + f"✨ Tag {version} not found. Cloning {git_url} default branch..." + ) + subprocess.run(["git", "clone", "--bare", git_url, tmp_dir], check=True) + + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=tmp_dir, + check=True, + capture_output=True, + text=True, + ) + commit_id = result.stdout.strip() + print(f"✨ Found commit ID: {commit_id}") + return commit_id + except subprocess.CalledProcessError: + return None + + +def get_module_info(module_name): + result = subprocess.run( + ["go", "list", "-m", "-json", module_name], + check=True, + capture_output=True, + text=True, + ) + return json.loads(result.stdout) + + +def get_git_url(module_name): + # Remove the version suffix, if present + module_name = re.sub(r"/v\d+$", "", module_name) + + # Remove the subdirectory, if present (e.g. github.com/foo/bar/subdir -> github.com/foo/bar) + if "gitlab.com" in module_name or "github.com" in module_name: + url_parts = module_name.split("/") + if len(url_parts) > 3: + module_name = "/".join(url_parts[:3]) + + if "gitlab.com" in module_name: + return f"https://gitlab.com/{module_name.replace('gitlab.com/', '')}" + elif "github.com" in module_name: + return f"https://github.com/{module_name.replace('github.com/', '')}" + elif "git.torproject.org" in module_name: + return f"https://{module_name}" + else: + response = requests.get(f"https://{module_name}/?go-get=1") + if response.status_code != 200: + return None + soup = BeautifulSoup(response.content, "html.parser") + meta_tag = soup.find("meta", {"name": "go-import"}) + if meta_tag: + url = meta_tag["content"].split()[2] + r = requests.get(url, allow_redirects=True) + if r.history: + return r.url + else: + return url + + return None + + +def main(repo, repo_name, version, folder): + """Flatpak Go Generator""" + + with tempfile.TemporaryDirectory() as temp_dir: + os.chdir(temp_dir) + + try: + if not folder: + folder = "." + print("✨ Cloning the target repository") + subprocess.run( + ["git", "clone", f"https://{repo}", f"src/{repo_name}"], check=True + ) + os.chdir(f"src/{repo_name}/{folder}") + + if version: + print(f"✨ Checking out version {version}") + subprocess.run(["git", "checkout", version], check=True) + + except subprocess.CalledProcessError: + print(f"✨ Error fetching {sys.argv[1]}") + sys.exit(1) + + result = subprocess.run( + ["go", "list", "-m", "all"], + check=True, + capture_output=True, + text=True, + cwd=f"{temp_dir}/src/{repo_name}/{folder}", + ) + + modules = result.stdout.strip().split("\n") + modules = modules[1:] # Skip the first module, which is the current module + + print(f"✨ Found {len(modules)} dependencies") + + sources = [] + + for module in modules: + module_name, module_version = module.split(" ", 1) + print(f"✨ Module: {module}") + + short_commit_id = extract_commit_id(module_version) + if short_commit_id: + print(f"✨ Found short_commit_id: {short_commit_id}") + + info = get_module_info(module_name) + path = info.get("Path") + version = info.get("Version") + if version.endswith("+incompatible"): + version = version[:-13] + if not version: + continue + + git_url = get_git_url(module_name) + if not git_url: + git_url = f"https://{module_name}.git" + + print(f"✨ Git URL: {git_url}") + + commit_id = get_commit_id_from_git(git_url, version, short_commit_id) + + if not commit_id: + print( + f"✨ Error: Could not retrieve commit ID for {module_name}@{version}." + ) + continue + + sources.append( + { + "type": "git", + "url": git_url, + "commit": commit_id, + "dest": f"src/{path.replace('.', '/')}", + } + ) + + yaml_data = { + "name": repo_name, + "buildsystem": "simple", + "build-options": {"env": {"GOBIN": "/app/bin/"}}, + "build-commands": [ + f". /usr/lib/sdk/golang/enable.sh; export GOPATH=$PWD; export GO111MODULE=off; go install {repo}/{os.path.basename(repo)}" + ], + "sources": sources, + } + + print("✨ 🌟 ✨") + print(yaml.dump(yaml_data)) + + +if __name__ == "__main__": + args = parse_args() + main(args.url, args.name, args.version, args.folder)