From fc7debba3f48b395ee24876c70c70f64be70ea8c Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Sun, 14 Jul 2024 17:22:50 -0500 Subject: [PATCH 1/2] Adding automatic nightly release builds - Restructured linux repo directory structure to include Stable and Nightly releases - Reworked Earthfile to pass IS_NIGHTLY boolean to package build scripts for use in file names - Added nightly pipeline to CICD config yaml. Pipeline is triggered by IS_NIGHTLY variable - Two distribution scripts now exist, one for Stable and one for Nightly - Reworked RPM build spec files to create appropriate file names - Reworked debian release generation scripts - RPM repo directory structure is prepped for arm64 builds - Switched from scp to rsync for moving the completed repo from ochestration to repo server - Created schedule in Gitlab UI to trigger nightly release at 2300 Central Time --- .gitlab-ci.yml | 88 +++++++++++++++++-- Earthfile | 16 ++-- package/debian/earthly_make_veilid_cli_deb.sh | 13 ++- .../debian/earthly_make_veilid_server_deb.sh | 13 ++- .../veilid-cli/earthly_make_veilid_cli_rpm.sh | 10 +++ package/rpm/veilid-cli/veilid-cli.spec | 2 +- .../earthly_make_veilid_server_rpm.sh | 12 ++- package/rpm/veilid-server/veilid-server.spec | 2 +- .../scp-amd64-debs-to-orchestrator.sh | 2 +- .../distribute-nightly-packages.sh | 79 +++++++++++++++++ .../distribute-packages.sh | 70 --------------- .../distribute-stable-packages.sh | 79 +++++++++++++++++ .../generate-nightly-release.sh | 27 ++++++ ...-release.sh => generate-stable-release.sh} | 2 - 14 files changed, 323 insertions(+), 92 deletions(-) create mode 100644 scripts/cicd/build-orchestration/distribute-nightly-packages.sh delete mode 100644 scripts/cicd/build-orchestration/distribute-packages.sh create mode 100644 scripts/cicd/build-orchestration/distribute-stable-packages.sh create mode 100644 scripts/cicd/build-orchestration/generate-nightly-release.sh rename scripts/cicd/build-orchestration/{generate-release.sh => generate-stable-release.sh} (92%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49e8c04d..bc6e9746 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,7 +106,7 @@ package_amd64_deb: - build-amd64-deb script: - earthly bootstrap - - earthly +package-linux-amd64-deb + - earthly +package-linux-amd64-deb --IS_NIGHTLY=$IS_NIGHTLY - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -119,7 +119,7 @@ package_arm64_deb: - build-arm64-deb script: - earthly bootstrap - - earthly +package-linux-arm64-deb + - earthly +package-linux-arm64-deb --IS_NIGHTLY=$IS_NIGHTLY - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -132,7 +132,7 @@ package_amd64_rpm: - build-amd64-rpm script: - earthly bootstrap - - earthly +package-linux-amd64-rpm + - earthly +package-linux-amd64-rpm --IS_NIGHTLY=$IS_NIGHTLY - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -168,9 +168,9 @@ build_repositories: tags: - build-orchestration variables: - SECURE_FILES_DOWNLOAD_PATH: './' + SECURE_FILES_DOWNLOAD_PATH: '$HOME' script: - - cp scripts/cicd/build-orchestration/generate-release.sh ~ + - cp scripts/cicd/build-orchestration/generate-stable-release.sh ~ - bash scripts/cicd/build-orchestration/distribute-packages.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -311,4 +311,80 @@ dryrun_delete_build_machines: rules: - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ -# Just a note so I can try dry run + +# Nightly build pipeline + +nightly_create_build_machines: + stage: build_packages + tags: + - build-orchestration + script: + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm + rules: + - if: '$IS_NIGHTLY == true' + +nightly_package_amd64_deb: + stage: build_packages + needs: + - nightly_create_build_machines + tags: + - build-amd64-deb + script: + - earthly bootstrap + - earthly +package-linux-amd64-deb --IS_NIGHTLY=$IS_NIGHTLY + - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh + rules: + - if: '$IS_NIGHTLY == true' + +nightly_package_arm64_deb: + stage: build_packages + needs: + - nightly_create_build_machines + tags: + - build-arm64-deb + script: + - earthly bootstrap + - earthly +package-linux-arm64-deb --IS_NIGHTLY=$IS_NIGHTLY + - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh + rules: + - if: '$IS_NIGHTLY == true' + +nightly_package_amd64_rpm: + stage: build_packages + needs: + - nightly_create_build_machines + tags: + - build-amd64-rpm + script: + - earthly bootstrap + - earthly +package-linux-amd64-rpm --IS_NIGHTLY=$IS_NIGHTLY + - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh + rules: + - if: '$IS_NIGHTLY == true' + +nightly_build_repositories: + stage: distribute + tags: + - build-orchestration + variables: + SECURE_FILES_DOWNLOAD_PATH: '$HOME' + script: + - cp scripts/cicd/build-orchestration/generate-nightly-release.sh ~ + - bash scripts/cicd/build-orchestration/distribute-packages.sh + rules: + - if: '$IS_NIGHTLY == true' + +nightly_delete_build_machines: + stage: distribute + needs: + - deploy_repos + tags: + - build-orchestration + script: + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm + rules: + - if: '$IS_NIGHTLY == true' \ No newline at end of file diff --git a/Earthfile b/Earthfile index 4cb5cd72..d53fbeeb 100644 --- a/Earthfile +++ b/Earthfile @@ -212,10 +212,10 @@ package-linux-amd64-deb: ################################# COPY --dir package /veilid # veilid-server - RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh amd64 x86_64-unknown-linux-gnu + RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh amd64 x86_64-unknown-linux-gnu $IS_NIGHTLY SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/ # veilid-cli - RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh amd64 x86_64-unknown-linux-gnu + RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh amd64 x86_64-unknown-linux-gnu $IS_NIGHTLY # save artifacts SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/ @@ -231,10 +231,10 @@ package-linux-amd64-rpm: COPY +build-linux-amd64/x86_64-unknown-linux-gnu /veilid/target/x86_64-unknown-linux-gnu RUN mkdir -p /rpm-work-dir/veilid-server # veilid-server - RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh x86_64 x86_64-unknown-linux-gnu + RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh x86_64 x86_64-unknown-linux-gnu $IS_NIGHTLY #SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/x86_64/*.rpm AS LOCAL ./target/packages/ # veilid-cli - RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh x86_64 x86_64-unknown-linux-gnu + RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh x86_64 x86_64-unknown-linux-gnu $IS_NIGHTLY # save artifacts SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/x86_64/*.rpm AS LOCAL ./target/packages/ @@ -245,10 +245,10 @@ package-linux-arm64-deb: ################################# COPY --dir package /veilid # veilid-server - RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh arm64 aarch64-unknown-linux-gnu + RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh arm64 aarch64-unknown-linux-gnu $IS_NIGHTLY SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/ # veilid-cli - RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh arm64 aarch64-unknown-linux-gnu + RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh arm64 aarch64-unknown-linux-gnu $IS_NIGHTLY # save artifacts SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/ @@ -264,10 +264,10 @@ package-linux-arm64-rpm: COPY +build-linux-arm64/aarch64-unknown-linux-gnu /veilid/target/aarch64-unknown-linux-gnu RUN mkdir -p /rpm-work-dir/veilid-server # veilid-server - RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh aarch64 aarch64-unknown-linux-gnu + RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh aarch64 aarch64-unknown-linux-gnu --IS_NIGHTLY=$IS_NIGHTLY #SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/ # veilid-cli - RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh aarch64 aarch64-unknown-linux-gnu + RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh aarch64 aarch64-unknown-linux-gnu --IS_NIGHTLY=$IS_NIGHTLY # save artifacts SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/ diff --git a/package/debian/earthly_make_veilid_cli_deb.sh b/package/debian/earthly_make_veilid_cli_deb.sh index c3da0d93..5fdec937 100755 --- a/package/debian/earthly_make_veilid_cli_deb.sh +++ b/package/debian/earthly_make_veilid_cli_deb.sh @@ -3,6 +3,8 @@ set -e ARCH=$1 CARGO_ARCH=$2 +IS_NIGHTLY=$3 +BUILD_DATE=$(date '+%Y%m%d') CARGO_VERSION="$(/veilid/package/cargo_version.sh /veilid/veilid-cli/Cargo.toml)" rm -rf /dpkg mkdir -p /dpkg/out @@ -16,4 +18,13 @@ mkdir -p /dpkg/veilid-cli/usr/bin cp -f /veilid/target/$CARGO_ARCH/release/veilid-cli /dpkg/veilid-cli/usr/bin # pack it up dpkg-deb -b /dpkg/veilid-cli/ -mv /dpkg/veilid-cli.deb /dpkg/out/veilid-cli-$CARGO_VERSION\_$ARCH.deb \ No newline at end of file +# Appropriatly name the package for STABLE or NIGHTLY release +if [ "$3" = true ] +then + mv /dpkg/veilid-cli.deb /dpkg/out/veilid-cli-$BUILD_DATE\_$ARCH.deb +elif [ "$3" = false ] +then + mv /dpkg/veilid-cli.deb /dpkg/out/veilid-cli-$CARGO_VERSION\_$ARCH.deb +else + echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY" +fi \ No newline at end of file diff --git a/package/debian/earthly_make_veilid_server_deb.sh b/package/debian/earthly_make_veilid_server_deb.sh index 97108e6c..40a4540b 100755 --- a/package/debian/earthly_make_veilid_server_deb.sh +++ b/package/debian/earthly_make_veilid_server_deb.sh @@ -3,6 +3,8 @@ set -e ARCH=$1 CARGO_ARCH=$2 +IS_NIGHTLY=$3 +BUILD_DATE=$(date '+%Y%m%d') CARGO_VERSION="$(/veilid/package/cargo_version.sh /veilid/veilid-server/Cargo.toml)" rm -rf /dpkg mkdir -p /dpkg/out @@ -22,4 +24,13 @@ mkdir -p /dpkg/veilid-server/usr/bin cp -f /veilid/target/$CARGO_ARCH/release/veilid-server /dpkg/veilid-server/usr/bin # pack it up dpkg-deb -b /dpkg/veilid-server/ -mv /dpkg/veilid-server.deb /dpkg/out/veilid-server-$CARGO_VERSION\_$ARCH.deb \ No newline at end of file +# Appropriatly name the package for STABLE or NIGHTLY release +if [ "$3" = true ] +then + mv /dpkg/veilid-server.deb /dpkg/out/veilid-server-$BUILD_DATE\_$ARCH.deb +elif [ "$3" = false ] +then + mv /dpkg/veilid-server.deb /dpkg/out/veilid-server-$CARGO_VERSION\_$ARCH.deb +else + echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY" +fi \ No newline at end of file diff --git a/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh b/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh index 0d5aa627..f5b07224 100755 --- a/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh +++ b/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh @@ -1,10 +1,20 @@ #!/bin/bash ARCH=$1 CARGO_ARCH=$2 +IS_NIGHTLY=$3 +BUILD_DATE=$(date '+%Y%m%d') CARGO_VERSION="$(/veilid/package/cargo_version.sh /veilid/veilid-cli/Cargo.toml)" # veilid-cli spec file cp -rf /veilid/package/rpm/veilid-cli/veilid-cli.spec /root/rpmbuild/SPECS/ +if [ "$3" = true ] +then + /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-cli.spec RELEASE_VERSION $BUILD_DATE +elif [ "$3" = false ] + /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-cli.spec RELEASE_VERSION $CARGO_VERSION +else + echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY" +fi /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-cli.spec CARGO_VERSION $CARGO_VERSION /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-cli.spec ARCH $ARCH /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-cli.spec CARGO_ARCH $CARGO_ARCH diff --git a/package/rpm/veilid-cli/veilid-cli.spec b/package/rpm/veilid-cli/veilid-cli.spec index 8553de0b..8b15d27f 100644 --- a/package/rpm/veilid-cli/veilid-cli.spec +++ b/package/rpm/veilid-cli/veilid-cli.spec @@ -1,6 +1,6 @@ Summary: Veilid Server Command Line Interface Name: veilid-cli -Version: $CARGO_VERSION +Version: $RELEASE_VERSION Release: 1 URL: https://veilid.com Group: System diff --git a/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh b/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh index 8bbb55aa..590c4f45 100755 --- a/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh +++ b/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh @@ -1,11 +1,21 @@ #!/bin/bash ARCH=$1 CARGO_ARCH=$2 +IS_NIGHTLY=$3 +BUILD_DATE=$(date '+%Y%m%d') CARGO_VERSION="$(/veilid/package/cargo_version.sh /veilid/veilid-server/Cargo.toml)" # veilid-server spec file cp -rf /veilid/package/rpm/veilid-server/veilid-server.spec /root/rpmbuild/SPECS/ -/veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec CARGO_VERSION $CARGO_VERSION +# Select CARGO_VERSION for STABLE releases or BUILD_DATE for NIGHTLY releases +if [ "$3" = true ] +then + /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec RELEASE_VERSION $BUILD_DATE +elif [ "$3" = false ] + /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec RELEASE_VERSION $CARGO_VERSION +else + echo $3 "is not a valid state to determine if the build is STABLE or NIGHTLY" +fi /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec ARCH $ARCH /veilid/package/replace_variable.sh /root/rpmbuild/SPECS/veilid-server.spec CARGO_ARCH $CARGO_ARCH diff --git a/package/rpm/veilid-server/veilid-server.spec b/package/rpm/veilid-server/veilid-server.spec index f8a881c2..5063db71 100644 --- a/package/rpm/veilid-server/veilid-server.spec +++ b/package/rpm/veilid-server/veilid-server.spec @@ -1,6 +1,6 @@ Summary: Install a server grade, headless Veilid node Name: veilid-server -Version: $CARGO_VERSION +Version: $RELEASE_VERSION Release: 1 URL: https://veilid.com Group: System diff --git a/scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh b/scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh index 7acc410c..d0867925 100644 --- a/scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh +++ b/scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh @@ -7,5 +7,5 @@ cd workspace cp ~/builds/t338Uo9fn/0/veilid/veilid/target/packages/*.deb . tar -cf amd64-debs.tar *.deb scp *.tar gitlab-runner@10.116.0.5:~ -cd ~ +cd ../ rm -rf workspace \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/distribute-nightly-packages.sh b/scripts/cicd/build-orchestration/distribute-nightly-packages.sh new file mode 100644 index 00000000..344b828b --- /dev/null +++ b/scripts/cicd/build-orchestration/distribute-nightly-packages.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Clean and reset the workspaces +echo "Setting up the workspace" +# Rsync active repo to local workspace +rsync --archive gitlab-runner@10.116.0.3:/srv $HOME +# Ensure repo directory structure exists +mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}} +# Delete previous versions of packages +rm -rf $HOME/srv/apt/pool/nightly/main/*.deb +rm -rf $HOME/srv/rpm/{nightly/x86_64/*,nightly/x86_64/*} +# Ensure RPM workspace setup +mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64} +rm -rf $HOME/rpm-builder/mount/repo/{nightly/x86_64/*,nightly/x86_64/*} + +# Setup crypto +export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)" +cat $HOME/package-signing-key.private | gpg --import +gpg --armor --export admin@veilid.org > $HOME/srv/gpg/veilid-packages-key.public + +# Copy .deb files into the workspace and generate repo files +echo "Starting deb process" +cd ~ +tar -xf amd64-debs.tar +tar -xf arm64-debs.tar +cp *.deb /home/gitlab-runner/srv/apt/pool/nightly/main +cd /home/gitlab-runner/srv/apt +echo "Creating Packages file" +dpkg-scanpackages --arch amd64 pool/nightly > dists/nightly/main/binary-amd64/Packages +dpkg-scanpackages --arch arm64 pool/nightly > dists/nightly/main/binary-arm64/Packages +cat dists/nightly/main/binary-amd64/Packages | gzip -9 > dists/nightly/main/binary-amd64/Packages.gz +cat dists/nightly/main/binary-arm64/Packages | gzip -9 > dists/nightly/main/binary-arm64/Packages.gz +echo "Creating Release file" +cd /home/gitlab-runner/srv/apt/dists/nightly +~/generate-nightly-release.sh > Release +echo "Signing Release file and creating InRelease" +cat /home/gitlab-runner/srv/apt/dists/nightly/Release | gpg --default-key admin@veilid.org -abs > /home/gitlab-runner/srv/apt/dists/nightly/Release.gpg +cat /home/gitlab-runner/srv/apt/dists/nightly/Release | gpg --default-key admin@veilid.org -abs --clearsign > /home/gitlab-runner/srv/apt/dists/nightly/InRelease + +# Copy .rpm files into the workspace and generate repo files +echo "Starting rpm process" +cd $HOME +tar -xf amd64-rpms.tar +echo "Copying rpms to container workspace" +cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/nightly/x86_64 +echo "Copying signing material to container workspace" +cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore +echo "Executing container actions" +docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8 +sleep 2 +cp -R $HOME/rpm-build-container/mount/repo/nightly $HOME/srv/rpm/nightly +cd $HOME/srv/rpm/nightly/x86_64 +echo "Signing the rpm repository" +gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/nightly/x86_64/repodata/repomd.xml + +echo "[veilid-nightly-x86_64-rpm-repo] +name=Veilid Nightly x86_64 RPM Repo +baseurl=https://packages.veilid.net/rpm/nightly/x86_64 +enabled=1 +gpgcheck=1 +gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/nightly/x86_64/veilid-rpm-repo.repo + +# Tar the repo data and transfer to the repo server +echo "Moving the repo scaffold to the repo server" +cd $HOME +rsync --archive $HOME/srv gitlab-runner@10.116.0.3:/srv +# tar -cf $HOME/repo.tar srv +# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~ + +# Cleanup +echo "Cleaning up the workspace" +rm -rf $GNUPGHOME +# rm $HOME/repo.tar +rm $HOME/*.deb +rm $HOME/*.rpm +rm -rf $HOME/rpm-build-container/mount/keystore +# rm rpm-build-container/mount/repo/*.rpm +# rm -rf rpm-build-container/mount/repo/repodata/* +echo "Process complete" \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/distribute-packages.sh b/scripts/cicd/build-orchestration/distribute-packages.sh deleted file mode 100644 index 546289f3..00000000 --- a/scripts/cicd/build-orchestration/distribute-packages.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -# Clean and reset the workspace -echo "Setting up the workspace" -rm -rf /home/gitlab-runner/srv -mkdir -p /home/gitlab-runner/srv/{gpg,rpm,apt/{dists/stable/main/{binary-amd64,binary-arm64},pool/main}} - -# Setup crypto -export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)" -cat ~/package-signing-key.private | gpg --import -gpg --armor --export admin@veilid.org > ~/srv/gpg/veilid-packages-key.public - -# Copy .deb files into the workspace and generate repo files -echo "Starting deb process" -cd ~ -tar -xf amd64-debs.tar -tar -xf arm64-debs.tar -cp *.deb ~/srv/apt/pool/main -cd ~/srv/apt -echo "Creating Packages file" -dpkg-scanpackages --arch amd64 pool/ > dists/stable/main/binary-amd64/Packages -dpkg-scanpackages --arch arm64 pool/ > dists/stable/main/binary-arm64/Packages -cat dists/stable/main/binary-amd64/Packages | gzip -9 > dists/stable/main/binary-amd64/Packages.gz -cat dists/stable/main/binary-arm64/Packages | gzip -9 > dists/stable/main/binary-arm64/Packages.gz -echo "Creating Release file" -cd ~/srv/apt/dists/stable -~/generate-release.sh > Release -echo "Signing Release file and creating InRelease" -cat ~/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs > ~/srv/apt/dists/stable/Release.gpg -cat ~/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs --clearsign > ~/srv/apt/dists/stable/InRelease - -# Copy .rpm files into the workspace and generate repo files -echo "Starting rpm process" -cd ~ -tar -xf amd64-rpms.tar -echo "Copying rpms to container workspace" -cp *.rpm /home/gitlab-runner/rpm-build-container/mount/repo -echo "Copying signing material to container workspace" -cp -R $GNUPGHOME /home/gitlab-runner/rpm-build-container/mount/keystore -echo "Executing container actions" -docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=/home/gitlab-runner/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8 -sleep 2 -cp -R /home/gitlab-runner/rpm-build-container/mount/repo/* ~/srv/rpm -cd ~/srv/rpm -echo "Signing the rpm repository" -gpg --default-key admin@veilid.org --detach-sign --armor ~/srv/rpm/repodata/repomd.xml - -echo "[veilid-rpm-repo] -name=Veilid RPM Repo -baseurl=https://packages.veilid.net/rpm -enabled=1 -gpgcheck=1 -gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > /home/gitlab-runner/srv/rpm/veilid-rpm-repo.repo - -# Tar the repo data and transfer to the repo server -echo "Moving the repo scaffold to the repo server" -cd ~ -tar -cf /home/gitlab-runner/repo.tar srv -scp -i /home/gitlab-runner/.ssh/id_ed25519 /home/gitlab-runner/repo.tar gitlab-runner@10.116.0.3:~ - -# Cleanup -echo "Cleaning up the workspace" -rm -rf $GNUPGHOME -rm /home/gitlab-runner/repo.tar -rm /home/gitlab-runner/*.deb -rm /home/gitlab-runner/*.rpm -rm -rf /home/gitlab-runner/rpm-build-container/mount/keystore -rm rpm-build-container/mount/repo/*.rpm -rm -rf rpm-build-container/mount/repo/repodata/* -echo "Process complete" \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/distribute-stable-packages.sh b/scripts/cicd/build-orchestration/distribute-stable-packages.sh new file mode 100644 index 00000000..a8b4dcb2 --- /dev/null +++ b/scripts/cicd/build-orchestration/distribute-stable-packages.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Clean and reset the workspaces +echo "Setting up the workspace" +# Rsync active repo to local workspace +rsync --archive gitlab-runner@10.116.0.3:/srv $HOME +# Ensure repo directory structure exists +mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}} +# Delete previous versions of packages +rm -rf $HOME/srv/apt/pool/stable/main/*.deb +rm -rf $HOME/srv/rpm/{stable/x86_64/*,stable/x86_64/*} +# Ensure RPM workspace setup +mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64} +rm -rf $HOME/rpm-builder/mount/repo/{stable/x86_64/*,stable/x86_64/*} + +# Setup crypto +export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)" +cat $HOME/package-signing-key.private | gpg --import +gpg --armor --export admin@veilid.org > $HOME/srv/gpg/veilid-packages-key.public + +# Copy .deb files into the workspace and generate repo files +echo "Starting deb process" +cd $HOME +tar -xf amd64-debs.tar +tar -xf arm64-debs.tar +cp *.deb $HOME/srv/apt/pool/stable/main +cd $HOME/srv/apt +echo "Creating Packages file" +dpkg-scanpackages --arch amd64 pool/stable > dists/stable/main/binary-amd64/Packages +dpkg-scanpackages --arch arm64 pool/stable > dists/stable/main/binary-arm64/Packages +cat dists/stable/main/binary-amd64/Packages | gzip -9 > dists/stable/main/binary-amd64/Packages.gz +cat dists/stable/main/binary-arm64/Packages | gzip -9 > dists/stable/main/binary-arm64/Packages.gz +echo "Creating Release file" +cd $HOME/srv/apt/dists/stable +~/generate-release.sh > Release +echo "Signing Release file and creating InRelease" +cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs > $HOME/srv/apt/dists/stable/Release.gpg +cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs --clearsign > $HOME/srv/apt/dists/stable/InRelease + +# Copy .rpm files into the workspace and generate repo files +echo "Starting rpm process" +cd $HOME +tar -xf amd64-rpms.tar +echo "Copying rpms to container workspace" +cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/stable/x86_64 +echo "Copying signing material to container workspace" +cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore +echo "Executing container actions" +docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8 +sleep 2 +cp -R $HOME/rpm-build-container/mount/repo/stable ~/srv/rpm/stable +cd $HOME/srv/rpm/stable/x86_64 +echo "Signing the rpm repository" +gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/stable/x86_64/repodata/repomd.xml + +echo "[veilid-stable-x86_64-rpm-repo] +name=Veilid Stable x86_64 RPM Repo +baseurl=https://packages.veilid.net/rpm/stable/x86_64 +enabled=1 +gpgcheck=1 +gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-rpm-repo.repo + +# Tar the repo data and transfer to the repo server +echo "Moving the repo scaffold to the repo server" +cd $HOME +rsync --archive $HOME/srv gitlab-runner@10.116.0.3:/srv +# tar -cf $HOME/repo.tar srv +# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~ + +# Cleanup +echo "Cleaning up the workspace" +rm -rf $GNUPGHOME +# rm $HOME/repo.tar +rm $HOME/*.deb +rm $HOME/*.rpm +rm -rf $HOME/rpm-build-container/mount/keystore +# rm rpm-build-container/mount/repo/*.rpm +# rm -rf rpm-build-container/mount/repo/repodata/* +echo "Process complete" \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/generate-nightly-release.sh b/scripts/cicd/build-orchestration/generate-nightly-release.sh new file mode 100644 index 00000000..54da4aed --- /dev/null +++ b/scripts/cicd/build-orchestration/generate-nightly-release.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -e + +do_hash() { + HASH_NAME=$1 + HASH_CMD=$2 + echo "${HASH_NAME}:" + for f in $(find -type f); do + f=$(echo $f | cut -c3-) # remove ./ prefix + if [ "$f" = "Release" ]; then + continue + fi + echo " $(${HASH_CMD} ${f} | cut -d" " -f1) $(wc -c $f)" + done +} + +cat << EOF +Origin: packages.veilid.net +Suite: nightly +Architectures: amd64 arm64 +Components: main +Description: Official repository for nightly released Veilid binaries. +Date: $(date -Ru) +EOF +do_hash "MD5Sum" "md5sum" +do_hash "SHA1" "sha1sum" +do_hash "SHA256" "sha256sum" \ No newline at end of file diff --git a/scripts/cicd/build-orchestration/generate-release.sh b/scripts/cicd/build-orchestration/generate-stable-release.sh similarity index 92% rename from scripts/cicd/build-orchestration/generate-release.sh rename to scripts/cicd/build-orchestration/generate-stable-release.sh index afe0d7d4..d64711cd 100644 --- a/scripts/cicd/build-orchestration/generate-release.sh +++ b/scripts/cicd/build-orchestration/generate-stable-release.sh @@ -16,9 +16,7 @@ do_hash() { cat << EOF Origin: packages.veilid.net -Label: packages.veilid.net Suite: stable -Codename: bullseye Architectures: amd64 arm64 Components: main Description: Official repository for Veilid binaries. From e22d00b92d98ba0af9ef0e954d2c63e2100a196f Mon Sep 17 00:00:00 2001 From: TC Date: Sun, 14 Jul 2024 22:43:24 +0000 Subject: [PATCH 2/2] The CI yaml doesn't accept a bool as a custom variable. I've switched it to a string to pass linting but I think this is going to break stuff. --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc6e9746..1f1c5dc3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -323,7 +323,7 @@ nightly_create_build_machines: - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm rules: - - if: '$IS_NIGHTLY == true' + - if: $IS_NIGHTLY == "true" nightly_package_amd64_deb: stage: build_packages @@ -336,7 +336,7 @@ nightly_package_amd64_deb: - earthly +package-linux-amd64-deb --IS_NIGHTLY=$IS_NIGHTLY - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh rules: - - if: '$IS_NIGHTLY == true' + - if: $IS_NIGHTLY == "true" nightly_package_arm64_deb: stage: build_packages @@ -349,7 +349,7 @@ nightly_package_arm64_deb: - earthly +package-linux-arm64-deb --IS_NIGHTLY=$IS_NIGHTLY - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh rules: - - if: '$IS_NIGHTLY == true' + - if: $IS_NIGHTLY == "true" nightly_package_amd64_rpm: stage: build_packages @@ -362,7 +362,7 @@ nightly_package_amd64_rpm: - earthly +package-linux-amd64-rpm --IS_NIGHTLY=$IS_NIGHTLY - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh rules: - - if: '$IS_NIGHTLY == true' + - if: $IS_NIGHTLY == "true" nightly_build_repositories: stage: distribute @@ -374,7 +374,7 @@ nightly_build_repositories: - cp scripts/cicd/build-orchestration/generate-nightly-release.sh ~ - bash scripts/cicd/build-orchestration/distribute-packages.sh rules: - - if: '$IS_NIGHTLY == true' + - if: $IS_NIGHTLY == "true" nightly_delete_build_machines: stage: distribute @@ -387,4 +387,4 @@ nightly_delete_build_machines: - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm rules: - - if: '$IS_NIGHTLY == true' \ No newline at end of file + - if: $IS_NIGHTLY == "true" \ No newline at end of file