Merge branch 'add-cicd-scripts' into 'main'

Migrating CICD scripts to Veilid repo

See merge request veilid/veilid!271

.gitlab-ci.yml

@@ -71,9 +71,9 @@ create_build_machines:
   tags:
     - build-orchestration
   script:
-    - /home/gitlab-runner/build-machine-ctl.sh create amd64-deb
+    - scripts/cicd/build-orchestration/build-machine-ctl.sh create amd64-deb
-    - /home/gitlab-runner/build-machine-ctl.sh create arm64-deb
+    - scripts/cicd/build-orchestration/build-machine-ctl.sh create arm64-deb
-    - /home/gitlab-runner/build-machine-ctl.sh create amd64-rpm
+    - scripts/cicd/build-orchestration/build-machine-ctl.sh create amd64-rpm
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
   
@@ -86,7 +86,7 @@ package_amd64_deb:
   script:
     - earthly bootstrap
     - earthly +package-linux-amd64-deb
-    - /home/gitlab-runner/scp-to-orchestrator.sh
+    - scripts/cicd/build-machine/scp-to-orchestrator.sh
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
@@ -99,7 +99,7 @@ package_arm64_deb:
   script:
     - earthly bootstrap
     - earthly +package-linux-arm64-deb
-    - /home/gitlab-runner/scp-to-orchestrator.sh
+    - scripts/cicd/build-machine/scp-to-orchestrator.sh
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
@@ -112,7 +112,7 @@ package_amd64_rpm:
   script:
     - earthly bootstrap
     - earthly +package-linux-amd64-rpm
-    - /home/gitlab-runner/scp-to-orchestrator.sh
+    - - scripts/cicd/build-machine/scp-to-orchestrator.sh
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
@@ -146,8 +146,11 @@ build_repositories:
     - publish_python
   tags:
     - build-orchestration
+  variables:
+    SECURE_FILES_DOWNLOAD_PATH: './'
   script:
-    - /home/gitlab-runner/distribute-packages.sh
+    - cp scripts/cicd/build-orchestration/generate-release.sh ~
+    - scripts/cicd/build-orchestration/distribute-packages.sh
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
@@ -158,7 +161,7 @@ deploy_repos:
   tags: 
     - repo-server
   script: 
-    - /home/gitlab-runner/deploy-repo.sh
+    - scripts/cicd/repo-server/deploy-repo.sh
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
@@ -169,8 +172,8 @@ delete_build_machines:
   tags:
     - build-orchestration
   script:
-    - /home/gitlab-runner/build-machine-ctl.sh delete amd64-deb
+    - scripts/cicd/build-orchestration/build-machine-ctl.sh delete amd64-deb
-    - /home/gitlab-runner/build-machine-ctl.sh delete arm64-deb
+    - scripts/cicd/build-orchestration/build-machine-ctl.sh delete arm64-deb
-    - /home/gitlab-runner/build-machine-ctl.sh delete amd64-rpm
+    - scripts/cicd/build-orchestration/build-machine-ctl.sh delete amd64-rpm
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'

scripts/cicd/build machine/scp-to-orchestrator.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+mkdir workspace
+cd workspace
+# if a new GitLab runner is created, the identifier below that follows build/ will be invalid
+# it might be available as a runner variable but IDK
+cp ~/builds/t338Uo9fn/0/veilid/veilid/target/packages/*.deb .
+tar -cf amd64-debs.tar *.deb
+scp *.tar gitlab-runner@10.116.0.5:~
+cd ~
+rm -rf workspace

scripts/cicd/build-orchestration/build-machine-ctrl.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+if [ "$1" = "create" ] && [ "$2" = "amd64-deb" ]
+then
+	## Create amd64-deb build machine
+	echo "Creating amd64-deb build machine"
+	doctl compute droplet create build-server-amd64-deb-tmp --image 154584863 \
+		--size c2-8vcpu-16gb-intel --region nyc1 --enable-private-networking \
+		--ssh-keys 38852180,38632397,41187560 --tag-names build-machines,build-orchestration --wait
+	echo "Done"
+
+elif [ "$1" = "create" ] && [ "$2" = "arm64-deb" ]
+then
+	## Create arm64-deb build machine
+	echo "Creating arm64-deb build machine"
+	doctl compute droplet create build-server-arm64-deb-tmp --image 154584861 \
+		--size c2-8vcpu-16gb-intel --region nyc1 --enable-private-networking \
+		--ssh-keys 38852180,38632397,41187560 --tag-names build-machines,build-orchestration --wait
+	echo "Done"
+
+elif [ "$1" = "create" ] && [ "$2" = "amd64-rpm" ]
+then
+	## Create amd64-rpm build machine
+	echo "Creating amd64-rpm build machine"
+	doctl compute droplet create build-server-amd64-rpm-tmp --image 154584864 \
+		--size c2-8vcpu-16gb-intel --region nyc1 --enable-private-networking \
+		--ssh-keys 38852180,38632397,41187560 --tag-names build-machines,build-orchestration --wait
+	echo "Done"
+
+elif [ "$1" = "create" ] && [ "$2" = "arm64-rpm" ] ## This snapshot does not yet exist
+then
+	## Create arm64-rpm build machine
+	echo "Creating arm64-rpm build machine"
+	doctl compute droplet create build-server-arm64-rpm-tmp --image 154584864 \
+		--size c2-8vcpu-16gb-intel --region nyc1 --enable-private-networking \
+		--ssh-keys 38852180,38632397,41187560 --tag-names build-machines,build-orchestration --wait
+	echo "Done"
+
+elif [ "$1" = "delete" ] && [ "$2" = "amd64-deb" ]
+then
+	## Delete amd64-deb build machine
+	echo "Deleting amd64-deb build machine"
+	doctl compute droplet delete build-server-amd64-deb-tmp --force
+	echo "Done"
+
+elif [ "$1" = "delete" ] && [ "$2" = "arm64-deb" ]
+then
+	## Delete arm64-deb build machine
+    echo "Deleting arm64-deb build machine"
+    doctl compute droplet delete build-server-arm64-deb-tmp --force
+    echo "Done"
+
+elif [ "$1" = "delete" ] && [ "$2" = "amd64-rpm" ]
+then
+	## Delete amd64-rpm build machine
+    echo "Deleting amd64-rpm build machine"
+    doctl compute droplet delete build-server-amd64-rpm-tmp --force
+    echo "Done"
+
+elif [ "$1" = "delete" ] && [ "$2" = "arm64-rpm" ] ## This snapshot does not exist yet
+then
+	## Delete arm64-rpm build machine
+    echo "Deleting arm64-rpm build machine"
+    doctl compute droplet delete build-server-arm64-rpm-tmp --force
+    echo "Done"
+else
+	echo $1 "is not a valid command to execute for "$2
+fi

scripts/cicd/build-orchestration/distribute-packages.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Clean and reset the workspace
+echo "Setting up the workspace"
+rm -rf /home/gitlab-runner/srv
+mkdir -p /home/gitlab-runner/srv/{gpg,rpm,apt/{dists/stable/main/{binary-amd64,binary-arm64},pool/main}}
+
+# Setup crypto
+export GNUPGHOME="$(mktemp -d ~/pgpkeys-XXXXXX)"
+cat ~/package-signing-key.private | gpg --import
+gpg --armor --export admin@veilid.org > ~/srv/gpg/veilid-packages-key.public
+
+# Copy .deb files into the workspace and generate repo files
+echo "Starting deb process"
+cd ~
+tar -xf amd64-debs.tar
+tar -xf arm64-debs.tar
+cp *.deb ~/srv/apt/pool/main
+cd ~/srv/apt
+echo "Creating Packages file"
+dpkg-scanpackages --arch amd64 pool/ > dists/stable/main/binary-amd64/Packages
+dpkg-scanpackages --arch arm64 pool/ > dists/stable/main/binary-arm64/Packages
+cat dists/stable/main/binary-amd64/Packages | gzip -9 > dists/stable/main/binary-amd64/Packages.gz
+cat dists/stable/main/binary-arm64/Packages | gzip -9 > dists/stable/main/binary-arm64/Packages.gz
+echo "Creating Release file"
+cd ~/srv/apt/dists/stable
+~/generate-release.sh > Release
+echo "Signing Release file and creating InRelease"
+cat ~/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs > ~/srv/apt/dists/stable/Release.gpg
+cat ~/srv/apt/dists/stable/Release | gpg --default-key admin@veilid.org -abs --clearsign > ~/srv/apt/dists/stable/InRelease
+
+# Copy .rpm files into the workspace and generate repo files
+echo "Starting rpm process"
+cd ~
+tar -xf amd64-rpms.tar
+echo "Copying rpms to container workspace"
+cp *.rpm /home/gitlab-runner/rpm-build-container/mount/repo
+echo "Copying signing material to container workspace"
+cp -R $GNUPGHOME /home/gitlab-runner/rpm-build-container/mount/keystore
+echo "Executing container actions"
+docker run --rm -d -it --name rpm-repo-builder --mount type=bind,source=/home/gitlab-runner/rpm-build-container/mount,target=/mount rpm-repo-builder-img:v8
+sleep 2
+cp -R /home/gitlab-runner/rpm-build-container/mount/repo/* ~/srv/rpm
+cd ~/srv/rpm
+echo "Signing the rpm repository"
+gpg --default-key admin@veilid.org --detach-sign --armor ~/srv/rpm/repodata/repomd.xml
+
+echo "[veilid-rpm-repo]
+name=Veilid RPM Repo
+baseurl=https://packages.veilid.net/rpm
+enabled=1
+gpgcheck=1
+gpgkey=https://packages.veilid.net/gpg/veilid-packages-key.public" > /home/gitlab-runner/srv/rpm/veilid-rpm-repo.repo
+
+# Tar the repo data and transfer to the repo server
+echo "Moving the repo scaffold to the repo server"
+cd ~
+tar -cf /home/gitlab-runner/repo.tar srv
+scp -i /home/gitlab-runner/.ssh/id_ed25519 /home/gitlab-runner/repo.tar gitlab-runner@10.116.0.3:~
+
+# Cleanup
+echo "Cleaning up the workspace"
+rm -rf $GNUPGHOME
+rm /home/gitlab-runner/repo.tar
+rm /home/gitlab-runner/*.deb
+rm /home/gitlab-runner/*.rpm
+rm -rf /home/gitlab-runner/rpm-build-container/mount/keystore
+rm rpm-build-container/mount/repo/*.rpm
+rm -rf rpm-build-container/mount/repo/repodata/*
+echo "Process complete"

scripts/cicd/build-orchestration/generate-release.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+set -e
+
+do_hash() {
+    HASH_NAME=$1
+    HASH_CMD=$2
+    echo "${HASH_NAME}:"
+    for f in $(find -type f); do
+        f=$(echo $f | cut -c3-) # remove ./ prefix
+        if [ "$f" = "Release" ]; then
+            continue
+        fi
+        echo " $(${HASH_CMD} ${f}  | cut -d" " -f1) $(wc -c $f)"
+    done
+}
+
+cat << EOF
+Origin: packages.veilid.net
+Label: packages.veilid.net
+Suite: stable
+Codename: bullseye
+Architectures: amd64 arm64
+Components: main
+Description: Official repository for Veilid binaries.
+Date: $(date -Ru)
+EOF
+do_hash "MD5Sum" "md5sum"
+do_hash "SHA1" "sha1sum"
+do_hash "SHA256" "sha256sum"

scripts/cicd/cicd-infra-description.md

@@ -0,0 +1,36 @@
+# Veilid Automated Build and Distribution CICD
+
+## Description
+
+The release process for Veilid results in builds, packages, and libraries being distributed to various repositories. This is accomplished through Gitlab's Runner system interacting with droplets on Digital Ocean. Some of the droplets are always up while others are built at the time of release and deleted after the release job is accomplished.
+
+The droplets are divided into three categories: build machines, build orchestration, and repo server. Build machines are ephemeral whereas build orchestration and repo server are online 24/7.
+
+* Build Machines
+  * Individual Debian high resource machine for each arch/OS combo being built
+  * Enrolled with Gitlab Runner
+  * Earthly installed
+  * Droplet size: c2-8vcpu-16gb-intel
+  * The amd64-deb build machine also builds and uploads the veilid-core and veilid-tools Rust crates and veilid-python module to crates.io and Pypi
+  * SCPs compiled packages to the orchestration machine ovber private networking
+* Build Orchestration
+  * Single Debian machine with minimal resources
+  * Creates and deletes build machines
+  * Natively constructs and signs the .deb repository directory structure
+  * Uses Docker with Rocky container to constuct and sign the .rpm directory structure
+  * SCPs the repos to the repo server
+* Repo Server
+  * Single Debian machine with moderate resources
+  * Hosts the .deb and .rpm package repositories for veilid-server and veilid-cli
+
+## Process Flow
+
+1. The release process is triggered by creating a new version number tag on Gitlab. The tag format must be `vX.X.X`.
+2. Gitlab CICD builds a SaaS container in which Earthly tests are performed on the latest version of the Main branch. A test fail will exit the CICD process.
+3. The Gitlab Runner registered to the build orchestration machine executes the build-machine-ctl.sh script to create the build machines.
+4. The Gitlab Runners registered to the build machines execute their specified arch/OS build as defined in the Earthly execution command in .gitlab-ci.yml.
+5. The build machine for amd64-deb additionally compiles veilid-core and veilid-tools Rust crates and the veilid-python module. These are uploaded to crates.io and Pypi as part of their respective build processes.
+6. When a build completes, the Gitlab Runner then executes the scp-to-orchestrator.sh script when sends the .deb or .rpm packages to build orchestration.
+7. Once the build jobs in CICD have completed, the Gitlab Runner registered to build orchestration executes the distribute-packages.sh script which results in signed .deb and .rpm repositories being sent to the repo server.
+8. The build orchecstration machine sends droplet delete commands to Digital Ocean for each of the build machines.
+9. The Gitlab Runner registered to the repo server executes the deploy-repo.sh script which updates the web server's file directory with the latest packages versions.

scripts/cicd/repo-server/deploy-repo.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+cd ~
+
+rm -rf /srv/*
+
+rm -rf ~/srv
+
+tar -xf repo.tar
+
+cp -R ~/srv/* /srv
+
+#chown -R www-data:www-data /srv