mirror of
https://gitlab.com/veilid/veilid.git
synced 2025-05-02 14:46:12 -04:00
Adding automatic nightly release builds
- Restructured linux repo directory structure to include Stable and Nightly releases - Reworked Earthfile to pass IS_NIGHTLY boolean to package build scripts for use in file names - Added nightly pipeline to CICD config yaml. Pipeline is triggered by IS_NIGHTLY variable - Two distribution scripts now exist, one for Stable and one for Nightly - Reworked RPM build spec files to create appropriate file names - Reworked debian release generation scripts - RPM repo directory structure is prepped for arm64 builds - Switched from scp to rsync for moving the completed repo from ochestration to repo server - Created schedule in Gitlab UI to trigger nightly release at 2300 Central Time
This commit is contained in:
TC Johnson
parent
8c297acdee
commit
fc7debba3f
14 changed files with 323 additions and 92 deletions
|
|
@ -7,5 +7,5 @@ cd workspace
|
|||
cp ~/builds/t338Uo9fn/0/veilid/veilid/target/packages/*.deb .
|
||||
tar -cf amd64-debs.tar *.deb
|
||||
scp *.tar gitlab-runner@10.116.0.5:~
|
||||
cd ~
|
||||
cd ../
|
||||
rm -rf workspace
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Clean and reset the workspaces
|
||||
echo "Setting up the workspace"
|
||||
# Rsync active repo to local workspace
|
||||
rsync --archive gitlab-runner@10.116.0.3:/srv $HOME
|
||||
# Ensure repo directory structure exists
|
||||
mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}}
|
||||
# Delete previous versions of packages
|
||||
rm -rf $HOME/srv/apt/pool/nightly/main/*.deb
|
||||
rm -rf $HOME/srv/rpm/{nightly/x86_64/*,nightly/x86_64/*}
|
||||
# Ensure RPM workspace setup
|
||||
mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64}
|
||||
rm -rf $HOME/rpm-builder/mount/repo/{nightly/x86_64/*,nightly/x86_64/*}
|
||||
|
||||
# Setup crypto
|
||||
export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)"
|
||||
cat $HOME/package-signing-key.private | gpg --import
|
||||
gpg --armor --export admin@veilid.org > $HOME/srv/gpg/veilid-packages-key.public
|
||||
|
||||
# Copy .deb files into the workspace and generate repo files
|
||||
echo "Starting deb process"
|
||||
cd ~
|
||||
tar -xf amd64-debs.tar
|
||||
tar -xf arm64-debs.tar
|
||||
cp *.deb /home/gitlab-runner/srv/apt/pool/nightly/main
|
||||
cd /home/gitlab-runner/srv/apt
|
||||
echo "Creating Packages file"
|
||||
dpkg-scanpackages --arch amd64 pool/nightly > dists/nightly/main/binary-amd64/Packages
|
||||
dpkg-scanpackages --arch arm64 pool/nightly > dists/nightly/main/binary-arm64/Packages
|
||||
cat dists/nightly/main/binary-amd64/Packages | gzip -9 > dists/nightly/main/binary-amd64/Packages.gz
|
||||
cat dists/nightly/main/binary-arm64/Packages | gzip -9 > dists/nightly/main/binary-arm64/Packages.gz
|
||||
echo "Creating Release file"
|
||||
cd /home/gitlab-runner/srv/apt/dists/nightly
|
||||
~/generate-nightly-release.sh > Release
|
||||
echo "Signing Release file and creating InRelease"
|
||||
cat /home/gitlab-runner/srv/apt/dists/nightly/Release | gpg --default-key admin@veilid.org -abs > /home/gitlab-runner/srv/apt/dists/nightly/Release.gpg
|
||||
cat /home/gitlab-runner/srv/apt/dists/nightly/Release | gpg --default-key admin@veilid.org -abs --clearsign > /home/gitlab-runner/srv/apt/dists/nightly/InRelease
|
||||
|
||||
# Copy .rpm files into the workspace and generate repo files
|
||||
echo "Starting rpm process"
|
||||
cd $HOME
|
||||
tar -xf amd64-rpms.tar
|
||||
echo "Copying rpms to container workspace"
|
||||
cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/nightly/x86_64
|
||||
echo "Copying signing material to container workspace"
|
||||
cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore
|
||||
echo "Executing container actions"
|
||||
docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8
|
||||
sleep 2
|
||||
cp -R $HOME/rpm-build-container/mount/repo/nightly $HOME/srv/rpm/nightly
|
||||
cd $HOME/srv/rpm/nightly/x86_64
|
||||
echo "Signing the rpm repository"
|
||||
gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/nightly/x86_64/repodata/repomd.xml
|
||||
|
||||
echo "[veilid-nightly-x86_64-rpm-repo]
|
||||
name=Veilid Nightly x86_64 RPM Repo
|
||||
baseurl=https://packages.veilid.net/rpm/nightly/x86_64
|
||||
enabled=1
|
||||
gpgcheck=1
|
||||
gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/nightly/x86_64/veilid-rpm-repo.repo
|
||||
|
||||
# Tar the repo data and transfer to the repo server
|
||||
echo "Moving the repo scaffold to the repo server"
|
||||
cd $HOME
|
||||
rsync --archive $HOME/srv gitlab-runner@10.116.0.3:/srv
|
||||
# tar -cf $HOME/repo.tar srv
|
||||
# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~
|
||||
|
||||
# Cleanup
|
||||
echo "Cleaning up the workspace"
|
||||
rm -rf $GNUPGHOME
|
||||
# rm $HOME/repo.tar
|
||||
rm $HOME/*.deb
|
||||
rm $HOME/*.rpm
|
||||
rm -rf $HOME/rpm-build-container/mount/keystore
|
||||
# rm rpm-build-container/mount/repo/*.rpm
|
||||
# rm -rf rpm-build-container/mount/repo/repodata/*
|
||||
echo "Process complete"
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Clean and reset the workspace
|
||||
echo "Setting up the workspace"
|
||||
rm -rf /home/gitlab-runner/srv
|
||||
mkdir -p /home/gitlab-runner/srv/{gpg,rpm,apt/{dists/stable/main/{binary-amd64,binary-arm64},pool/main}}
|
||||
|
||||
# Setup crypto
|
||||
export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)"
|
||||
cat ~/package-signing-key.private | gpg --import
|
||||
gpg --armor --export admin@veilid.org > ~/srv/gpg/veilid-packages-key.public
|
||||
|
||||
# Copy .deb files into the workspace and generate repo files
|
||||
echo "Starting deb process"
|
||||
cd ~
|
||||
tar -xf amd64-debs.tar
|
||||
tar -xf arm64-debs.tar
|
||||
cp *.deb ~/srv/apt/pool/main
|
||||
cd ~/srv/apt
|
||||
echo "Creating Packages file"
|
||||
dpkg-scanpackages --arch amd64 pool/ > dists/stable/main/binary-amd64/Packages
|
||||
dpkg-scanpackages --arch arm64 pool/ > dists/stable/main/binary-arm64/Packages
|
||||
cat dists/stable/main/binary-amd64/Packages | gzip -9 > dists/stable/main/binary-amd64/Packages.gz
|
||||
cat dists/stable/main/binary-arm64/Packages | gzip -9 > dists/stable/main/binary-arm64/Packages.gz
|
||||
echo "Creating Release file"
|
||||
cd ~/srv/apt/dists/stable
|
||||
~/generate-release.sh > Release
|
||||
echo "Signing Release file and creating InRelease"
|
||||
cat ~/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs > ~/srv/apt/dists/stable/Release.gpg
|
||||
cat ~/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs --clearsign > ~/srv/apt/dists/stable/InRelease
|
||||
|
||||
# Copy .rpm files into the workspace and generate repo files
|
||||
echo "Starting rpm process"
|
||||
cd ~
|
||||
tar -xf amd64-rpms.tar
|
||||
echo "Copying rpms to container workspace"
|
||||
cp *.rpm /home/gitlab-runner/rpm-build-container/mount/repo
|
||||
echo "Copying signing material to container workspace"
|
||||
cp -R $GNUPGHOME /home/gitlab-runner/rpm-build-container/mount/keystore
|
||||
echo "Executing container actions"
|
||||
docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=/home/gitlab-runner/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8
|
||||
sleep 2
|
||||
cp -R /home/gitlab-runner/rpm-build-container/mount/repo/* ~/srv/rpm
|
||||
cd ~/srv/rpm
|
||||
echo "Signing the rpm repository"
|
||||
gpg --default-key admin@veilid.org --detach-sign --armor ~/srv/rpm/repodata/repomd.xml
|
||||
|
||||
echo "[veilid-rpm-repo]
|
||||
name=Veilid RPM Repo
|
||||
baseurl=https://packages.veilid.net/rpm
|
||||
enabled=1
|
||||
gpgcheck=1
|
||||
gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > /home/gitlab-runner/srv/rpm/veilid-rpm-repo.repo
|
||||
|
||||
# Tar the repo data and transfer to the repo server
|
||||
echo "Moving the repo scaffold to the repo server"
|
||||
cd ~
|
||||
tar -cf /home/gitlab-runner/repo.tar srv
|
||||
scp -i /home/gitlab-runner/.ssh/id_ed25519 /home/gitlab-runner/repo.tar gitlab-runner@10.116.0.3:~
|
||||
|
||||
# Cleanup
|
||||
echo "Cleaning up the workspace"
|
||||
rm -rf $GNUPGHOME
|
||||
rm /home/gitlab-runner/repo.tar
|
||||
rm /home/gitlab-runner/*.deb
|
||||
rm /home/gitlab-runner/*.rpm
|
||||
rm -rf /home/gitlab-runner/rpm-build-container/mount/keystore
|
||||
rm rpm-build-container/mount/repo/*.rpm
|
||||
rm -rf rpm-build-container/mount/repo/repodata/*
|
||||
echo "Process complete"
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Clean and reset the workspaces
|
||||
echo "Setting up the workspace"
|
||||
# Rsync active repo to local workspace
|
||||
rsync --archive gitlab-runner@10.116.0.3:/srv $HOME
|
||||
# Ensure repo directory structure exists
|
||||
mkdir -p $HOME/srv/{gpg,rpm/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64},apt/{dists/{stable/main/{binary-amd64,binary-arm64},nightly/main/{binary-amd64,binary-arm64}},pool/{stable/main,nightly/main}}}
|
||||
# Delete previous versions of packages
|
||||
rm -rf $HOME/srv/apt/pool/stable/main/*.deb
|
||||
rm -rf $HOME/srv/rpm/{stable/x86_64/*,stable/x86_64/*}
|
||||
# Ensure RPM workspace setup
|
||||
mkdir -p $HOME/rpm-build-container/mount/repo/{nightly/x86_64,nightly/x86_64,stable/x86_64,stable/x86_64}
|
||||
rm -rf $HOME/rpm-builder/mount/repo/{stable/x86_64/*,stable/x86_64/*}
|
||||
|
||||
# Setup crypto
|
||||
export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)"
|
||||
cat $HOME/package-signing-key.private | gpg --import
|
||||
gpg --armor --export admin@veilid.org > $HOME/srv/gpg/veilid-packages-key.public
|
||||
|
||||
# Copy .deb files into the workspace and generate repo files
|
||||
echo "Starting deb process"
|
||||
cd $HOME
|
||||
tar -xf amd64-debs.tar
|
||||
tar -xf arm64-debs.tar
|
||||
cp *.deb $HOME/srv/apt/pool/stable/main
|
||||
cd $HOME/srv/apt
|
||||
echo "Creating Packages file"
|
||||
dpkg-scanpackages --arch amd64 pool/stable > dists/stable/main/binary-amd64/Packages
|
||||
dpkg-scanpackages --arch arm64 pool/stable > dists/stable/main/binary-arm64/Packages
|
||||
cat dists/stable/main/binary-amd64/Packages | gzip -9 > dists/stable/main/binary-amd64/Packages.gz
|
||||
cat dists/stable/main/binary-arm64/Packages | gzip -9 > dists/stable/main/binary-arm64/Packages.gz
|
||||
echo "Creating Release file"
|
||||
cd $HOME/srv/apt/dists/stable
|
||||
~/generate-release.sh > Release
|
||||
echo "Signing Release file and creating InRelease"
|
||||
cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs > $HOME/srv/apt/dists/stable/Release.gpg
|
||||
cat $HOME/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs --clearsign > $HOME/srv/apt/dists/stable/InRelease
|
||||
|
||||
# Copy .rpm files into the workspace and generate repo files
|
||||
echo "Starting rpm process"
|
||||
cd $HOME
|
||||
tar -xf amd64-rpms.tar
|
||||
echo "Copying rpms to container workspace"
|
||||
cp *x86_64.rpm $HOME/rpm-build-container/mount/repo/stable/x86_64
|
||||
echo "Copying signing material to container workspace"
|
||||
cp -R $GNUPGHOME $HOME/rpm-build-container/mount/keystore
|
||||
echo "Executing container actions"
|
||||
docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=$HOME/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8
|
||||
sleep 2
|
||||
cp -R $HOME/rpm-build-container/mount/repo/stable ~/srv/rpm/stable
|
||||
cd $HOME/srv/rpm/stable/x86_64
|
||||
echo "Signing the rpm repository"
|
||||
gpg --default-key admin@veilid.org --detach-sign --armor $HOME/srv/rpm/stable/x86_64/repodata/repomd.xml
|
||||
|
||||
echo "[veilid-stable-x86_64-rpm-repo]
|
||||
name=Veilid Stable x86_64 RPM Repo
|
||||
baseurl=https://packages.veilid.net/rpm/stable/x86_64
|
||||
enabled=1
|
||||
gpgcheck=1
|
||||
gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > $HOME/srv/rpm/stable/x86_64/veilid-rpm-repo.repo
|
||||
|
||||
# Tar the repo data and transfer to the repo server
|
||||
echo "Moving the repo scaffold to the repo server"
|
||||
cd $HOME
|
||||
rsync --archive $HOME/srv gitlab-runner@10.116.0.3:/srv
|
||||
# tar -cf $HOME/repo.tar srv
|
||||
# scp -i $HOME/.ssh/id_ed25519 $HOME/repo.tar gitlab-runner@10.116.0.3:~
|
||||
|
||||
# Cleanup
|
||||
echo "Cleaning up the workspace"
|
||||
rm -rf $GNUPGHOME
|
||||
# rm $HOME/repo.tar
|
||||
rm $HOME/*.deb
|
||||
rm $HOME/*.rpm
|
||||
rm -rf $HOME/rpm-build-container/mount/keystore
|
||||
# rm rpm-build-container/mount/repo/*.rpm
|
||||
# rm -rf rpm-build-container/mount/repo/repodata/*
|
||||
echo "Process complete"
|
||||
27
scripts/cicd/build-orchestration/generate-nightly-release.sh
Normal file
27
scripts/cicd/build-orchestration/generate-nightly-release.sh
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
do_hash() {
|
||||
HASH_NAME=$1
|
||||
HASH_CMD=$2
|
||||
echo "${HASH_NAME}:"
|
||||
for f in $(find -type f); do
|
||||
f=$(echo $f | cut -c3-) # remove ./ prefix
|
||||
if [ "$f" = "Release" ]; then
|
||||
continue
|
||||
fi
|
||||
echo " $(${HASH_CMD} ${f} | cut -d" " -f1) $(wc -c $f)"
|
||||
done
|
||||
}
|
||||
|
||||
cat << EOF
|
||||
Origin: packages.veilid.net
|
||||
Suite: nightly
|
||||
Architectures: amd64 arm64
|
||||
Components: main
|
||||
Description: Official repository for nightly released Veilid binaries.
|
||||
Date: $(date -Ru)
|
||||
EOF
|
||||
do_hash "MD5Sum" "md5sum"
|
||||
do_hash "SHA1" "sha1sum"
|
||||
do_hash "SHA256" "sha256sum"
|
||||
|
|
@ -16,9 +16,7 @@ do_hash() {
|
|||
|
||||
cat << EOF
|
||||
Origin: packages.veilid.net
|
||||
Label: packages.veilid.net
|
||||
Suite: stable
|
||||
Codename: bullseye
|
||||
Architectures: amd64 arm64
|
||||
Components: main
|
||||
Description: Official repository for Veilid binaries.
|
||||
Loading…
Add table
Add a link
Reference in a new issue