Git-Mirrors/veilid
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2025-09-29 12:18:27 -04:00
Code Issues Projects Releases Packages Wiki Activity

Further adventures in nightly release automation

Browse source 
Pushing this up in order to end-to-end test automated nightly
releases.
 Please enter the commit message for your changes. Lines starting
This commit is contained in:
TC Johnson 2024-07-19 14:08:25 -05:00
parent 6c0b645ba6
commit c9d2b354f7
No known key found for this signature in database
6 changed files with 121 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

28
scripts/cicd/build-orchestration/distribute-nightly-packages.sh
View file

@ -4,14 +4,9 @@
echo "Setting up the workspace"
# Rsync active repo to local workspace
rsync --archive gitlab-runner@10.116.0.3:/srv/ $HOME/srv/
# Ensure repo directory structure exists
mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}}
# Delete previous versions of packages
rm -rf $HOME/srv/apt/pool/nightly/main/*.deb
rm -rf $HOME/srv/rpm/{nightly/x86_64/*,nightly/x86_64/*}
# Ensure RPM workspace setup
mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64}
rm -rf $HOME/rpm-builder/mount/repo/{nightly/x86_64/*,nightly/x86_64/*}
rm -rf $HOME/srv/rpm/nightly/x86_64/*
# Setup crypto
export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)"
@ -44,11 +39,11 @@ tar -xf amd64-rpms.tar
echo "Copying rpms to container workspace"
cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/nightly/x86_64
echo "Copying signing material to container workspace"
cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore
cp -R $GNUPGHOME/* $HOME/rpm-build-container/mount/keystore
echo "Executing container actions"
docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8
docker run --rm -d -it -e IS_NIGHTLY=$IS_NIGHTLY --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v12
sleep 2
cp -R $HOME/rpm-build-container/mount/repo/nightly/* $HOME/srv/rpm/nightly
cp -R $HOME/rpm-build-container/mount/repo/nightly/x86_64/* $HOME/srv/rpm/nightly/x86_64
cd $HOME/srv/rpm/nightly/x86_64
echo "Signing the rpm repository"
gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/nightly/x86_64/repodata/repomd.xml
@ -58,22 +53,19 @@ name=Veilid Nightly x86_64 RPM Repo
baseurl=https://packages.veilid.net/rpm/nightly/x86_64
enabled=1
gpgcheck=1
gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/nightly/x86_64/veilid-rpm-repo.repo
gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/nightly/x86_64/veilid-nightly-x86_64-rpm.repo
# Tar the repo data and transfer to the repo server
echo "Moving the repo scaffold to the repo server"
cd $HOME
rsync --archive $HOME/srv/ gitlab-runner@10.116.0.3:/srv/
# tar -cf $HOME/repo.tar srv
# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~
rsync --archive $HOME/srv/* gitlab-runner@10.116.0.3:/srv
# Cleanup
echo "Cleaning up the workspace"
rm -rf $GNUPGHOME
# rm $HOME/repo.tar
rm $HOME/*.tar
rm $HOME/*.deb
rm $HOME/*.rpm
rm -rf $HOME/rpm-build-container/mount/keystore
# rm rpm-build-container/mount/repo/*.rpm
# rm -rf rpm-build-container/mount/repo/repodata/*
echo "Process complete"
rm -rf $HOME/rpm-build-container/mount/keystore/*
rm -rf $HOME/rpm-build-container/mount/repo/nightly/x86_64/*
echo "Nightly packages distribution process complete"

37
scripts/cicd/build-orchestration/distribute-stable-packages.sh
View file

@ -3,15 +3,10 @@
# Clean and reset the workspaces
echo "Setting up the workspace"
# Rsync active repo to local workspace
rsync --archive gitlab-runner@10.116.0.3:/srv $HOME
# Ensure repo directory structure exists
mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}}
rsync --archive gitlab-runner@10.116.0.3:/srv/ $HOME/srv/
# Delete previous versions of packages
rm -rf $HOME/srv/apt/pool/stable/main/*.deb
rm -rf $HOME/srv/rpm/{stable/x86_64/*,stable/x86_64/*}
# Ensure RPM workspace setup
mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64}
rm -rf $HOME/rpm-builder/mount/repo/{stable/x86_64/*,stable/x86_64/*}
rm -rf $HOME/srv/rpm/stable/x86_64/*
# Setup crypto
export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)"
@ -32,7 +27,7 @@ cat dists/stable/main/binary-amd64/Packages | gzip -9 > dists/stable/main/binary
cat dists/stable/main/binary-arm64/Packages | gzip -9 > dists/stable/main/binary-arm64/Packages.gz
echo "Creating Release file"
cd $HOME/srv/apt/dists/stable
~/generate-release.sh > Release
$HOME/generate-release.sh > Release
echo "Signing Release file and creating InRelease"
cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs > $HOME/srv/apt/dists/stable/Release.gpg
cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs --clearsign > $HOME/srv/apt/dists/stable/InRelease
@ -46,34 +41,40 @@ cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/stable/x86_64
echo "Copying signing material to container workspace"
cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore
echo "Executing container actions"
docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8
docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v12
sleep 2
cp -R $HOME/rpm-build-container/mount/repo/stable ~/srv/rpm/stable
cp -R $HOME/rpm-build-container/mount/repo/stable/x86_64/* $HOME/srv/rpm/stable/x86_64/
cd $HOME/srv/rpm/stable/x86_64
echo "Signing the rpm repository"
gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/stable/x86_64/repodata/repomd.xml
# Generate .repo file for stable x86_64 releases
echo "[veilid-stable-x86_64-rpm-repo]
name=Veilid Stable x86_64 RPM Repo
baseurl=https://packages.veilid.net/rpm/stable/x86_64
enabled=1
gpgcheck=1
gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-rpm-repo.repo
gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-stable-x86_64-rpm.repo
# Generate .repo file for stable arm64 releases -- to be added
# echo "[veilid-stable-arm64-rpm-repo]
# name=Veilid Stable x86_64 RPM Repo
# baseurl=https://packages.veilid.net/rpm/stable/arm64
# enabled=1
# gpgcheck=1
# gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-stable-arm64-rpm.repo
# Tar the repo data and transfer to the repo server
echo "Moving the repo scaffold to the repo server"
cd $HOME
rsync --archive $HOME/srv/* gitlab-runner@10.116.0.3:/srv
# tar -cf $HOME/repo.tar srv
# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~
# Cleanup
echo "Cleaning up the workspace"
rm -rf $GNUPGHOME
# rm $HOME/repo.tar
rm $HOME/*.tar
rm $HOME/*.deb
rm $HOME/*.rpm
rm -rf $HOME/rpm-build-container/mount/keystore
# rm rpm-build-container/mount/repo/*.rpm
# rm -rf rpm-build-container/mount/repo/repodata/*
echo "Process complete"
rm -rf $HOME/rpm-build-container/mount/keystore/*
rm -rf $HOME/rpm-build-container/mount/repo/nightly/x86_64/*
echo "Stable packages distribution process complete"

10
scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile Normal file
View file

@ -0,0 +1,10 @@
FROM rockylinux:8
LABEL author=admin@veilid.org
ENV IS_NIGHTLY false
COPY repobuild.sh repobuild.sh
RUN dnf install -y createrepo rpm-build rpm-sign yum-utils
RUN chmod +x repobuild.sh
ENTRYPOINT ["./repobuild.sh"]

33
scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh Normal file
View file

@ -0,0 +1,33 @@
#!/bin/bash
echo "==========Log start $(date +%F_%T)==========" &>> /mount/logfile
echo "setting GNUPGHOME $(date +%F_%T)" &>> /mount/logfile
export GNUPGHOME=/mount/keystore
echo "Adding key to rpm utility $(date +%F_%T)" &>> /mount/logfile
echo "%_signature gpg
%_gpg_name 516C76D1E372C5C96EE54E22AE0E059BC64CD052" > /root/.rpmmacros
if [ "$IS_NIGHTLY" = "true" ]
then
echo "Taking nightly actions branch $(date +%F_%T)" &>> /mount/logfile
cd /mount/repo/nightly/x86_64
elif [ "$IS_NIGHTLY" = "false" ]
then
echo "Taking stable branch actions $(date +%F_%T)" &>> /mount/logfile
cd /mount/repo/stable/x86_64
else
echo $IS_NIGHTLY "is not a valid state to determine if the build is STABLE or NIGHTLY (RPM RepoBuild)" &>> /mount/logfile
fi
echo "Signing RPMs $(date +%F_%T)" &>> /mount/logfile
rpm --addsign *.rpm &>> /mount/logfile
echo "Creating repo metadata $(date +%F_%T)" &>> /mount/logfile
createrepo . &>> /mount/logfile
echo "Setting file ownership $(date +%F_%T)" &>> /mount/logfile
chown -R 1000:1000 /mount
echo "==========RPM Packaging Process complete $(date +%F_%T)==========" &>> /mount/logfile