mirror of
https://git.anonymousland.org/anonymousland/synapse.git
synced 2024-09-06 23:32:03 +00:00
Enable docker image caching for the deb build (#10431)
This commit is contained in:
parent
ab82fd6ed1
commit
f22252d4f9
39
.github/workflows/release-artifacts.yml
vendored
39
.github/workflows/release-artifacts.yml
vendored
@ -48,12 +48,43 @@ jobs:
|
|||||||
distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
|
distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
path: src
|
path: src
|
||||||
- uses: actions/setup-python@v2
|
|
||||||
- run: ./src/scripts-dev/build_debian_packages "${{ matrix.distro }}"
|
- name: Set up Docker Buildx
|
||||||
- uses: actions/upload-artifact@v2
|
id: buildx
|
||||||
|
uses: docker/setup-buildx-action@v1
|
||||||
|
with:
|
||||||
|
install: true
|
||||||
|
|
||||||
|
- name: Set up docker layer caching
|
||||||
|
uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: /tmp/.buildx-cache
|
||||||
|
key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||||
|
restore-keys: |
|
||||||
|
${{ runner.os }}-buildx-
|
||||||
|
|
||||||
|
- name: Set up python
|
||||||
|
uses: actions/setup-python@v2
|
||||||
|
|
||||||
|
- name: Build the packages
|
||||||
|
# see https://github.com/docker/build-push-action/issues/252
|
||||||
|
# for the cache magic here
|
||||||
|
run: |
|
||||||
|
./src/scripts-dev/build_debian_packages \
|
||||||
|
--docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
|
||||||
|
--docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
|
||||||
|
--docker-build-arg=--progress=plain \
|
||||||
|
--docker-build-arg=--load \
|
||||||
|
"${{ matrix.distro }}"
|
||||||
|
rm -rf /tmp/.buildx-cache
|
||||||
|
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
|
||||||
|
|
||||||
|
- name: Upload debs as artifacts
|
||||||
|
uses: actions/upload-artifact@v2
|
||||||
with:
|
with:
|
||||||
name: debs
|
name: debs
|
||||||
path: debs/*
|
path: debs/*
|
||||||
|
1
changelog.d/10431.misc
Normal file
1
changelog.d/10431.misc
Normal file
@ -0,0 +1 @@
|
|||||||
|
Use a docker image cache for the prerequisites for the debian package build.
|
@ -17,6 +17,7 @@ import subprocess
|
|||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from typing import Optional, Sequence
|
||||||
|
|
||||||
DISTS = (
|
DISTS = (
|
||||||
"debian:buster",
|
"debian:buster",
|
||||||
@ -39,8 +40,11 @@ projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
|||||||
|
|
||||||
|
|
||||||
class Builder(object):
|
class Builder(object):
|
||||||
def __init__(self, redirect_stdout=False):
|
def __init__(
|
||||||
|
self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None
|
||||||
|
):
|
||||||
self.redirect_stdout = redirect_stdout
|
self.redirect_stdout = redirect_stdout
|
||||||
|
self._docker_build_args = tuple(docker_build_args or ())
|
||||||
self.active_containers = set()
|
self.active_containers = set()
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
self._failed = False
|
self._failed = False
|
||||||
@ -79,8 +83,8 @@ class Builder(object):
|
|||||||
stdout = None
|
stdout = None
|
||||||
|
|
||||||
# first build a docker image for the build environment
|
# first build a docker image for the build environment
|
||||||
subprocess.check_call(
|
build_args = (
|
||||||
[
|
(
|
||||||
"docker",
|
"docker",
|
||||||
"build",
|
"build",
|
||||||
"--tag",
|
"--tag",
|
||||||
@ -89,8 +93,13 @@ class Builder(object):
|
|||||||
"distro=" + dist,
|
"distro=" + dist,
|
||||||
"-f",
|
"-f",
|
||||||
"docker/Dockerfile-dhvirtualenv",
|
"docker/Dockerfile-dhvirtualenv",
|
||||||
"docker",
|
)
|
||||||
],
|
+ self._docker_build_args
|
||||||
|
+ ("docker",)
|
||||||
|
)
|
||||||
|
|
||||||
|
subprocess.check_call(
|
||||||
|
build_args,
|
||||||
stdout=stdout,
|
stdout=stdout,
|
||||||
stderr=subprocess.STDOUT,
|
stderr=subprocess.STDOUT,
|
||||||
cwd=projdir,
|
cwd=projdir,
|
||||||
@ -147,9 +156,7 @@ class Builder(object):
|
|||||||
self.active_containers.remove(c)
|
self.active_containers.remove(c)
|
||||||
|
|
||||||
|
|
||||||
def run_builds(dists, jobs=1, skip_tests=False):
|
def run_builds(builder, dists, jobs=1, skip_tests=False):
|
||||||
builder = Builder(redirect_stdout=(jobs > 1))
|
|
||||||
|
|
||||||
def sig(signum, _frame):
|
def sig(signum, _frame):
|
||||||
print("Caught SIGINT")
|
print("Caught SIGINT")
|
||||||
builder.kill_containers()
|
builder.kill_containers()
|
||||||
@ -180,6 +187,11 @@ if __name__ == "__main__":
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="skip running tests after building",
|
help="skip running tests after building",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--docker-build-arg",
|
||||||
|
action="append",
|
||||||
|
help="specify an argument to pass to docker build",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--show-dists-json",
|
"--show-dists-json",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@ -195,4 +207,12 @@ if __name__ == "__main__":
|
|||||||
if args.show_dists_json:
|
if args.show_dists_json:
|
||||||
print(json.dumps(DISTS))
|
print(json.dumps(DISTS))
|
||||||
else:
|
else:
|
||||||
run_builds(dists=args.dist, jobs=args.jobs, skip_tests=args.no_check)
|
builder = Builder(
|
||||||
|
redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg
|
||||||
|
)
|
||||||
|
run_builds(
|
||||||
|
builder,
|
||||||
|
dists=args.dist,
|
||||||
|
jobs=args.jobs,
|
||||||
|
skip_tests=args.no_check,
|
||||||
|
)
|
||||||
|
Loading…
Reference in New Issue
Block a user