Just going to re-run the release but without pushing crates or python. [skip ci]

.gitlab-ci.yml

@@ -5,13 +5,169 @@ variables:
   GIT_SUBMODULE_STRATEGY: normal
 
 stages:
-  #- prepare
+  - prepare
-  #- test
+  - test
-  #- build_packages
+  - build_packages
   - release
   - distribute
 
+# base earthly setup for jobs
+.base:
+  tags: [ saas-linux-medium-amd64 ]
+  image: docker
+  services:
+    - docker:dind
 
+.earthly: &earthly_setup
+  - apk update && apk add git
+  - wget https://github.com/earthly/earthly/releases/download/v0.7.15/earthly-linux-amd64 -O /usr/local/bin/earthly
+  - chmod +x /usr/local/bin/earthly
+  - earthly bootstrap
+  - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+  - test "$CI_PROJECT_PATH" != "veilid/veilid" && project_args="--CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE"
+
+# Create the build container if:
+#  - no container in the registry
+#  - run as part of a schedule
+#  - run manually from the pipelines web page
+build_cache:
+  extends: .base
+  stage: prepare
+  script:
+    - apk update && apk add jq && apk add curl
+    - if ! docker manifest inspect $CI_REGISTRY_IMAGE/build-cache:latest > /dev/null; then
+    -   CACHE_EPOCH=0
+    - else
+    -   'CONT_REPO_ID=$(curl "https://gitlab.com/api/graphql" --header "Content-Type: application/json" --request POST --data "{\"query\": \"query { project(fullPath: \\\"$CI_PROJECT_PATH\\\" ) { containerRepositories( name: \\\"build-cache\\\" ) { nodes { id }}}} \"}" | jq -r ".data.project.containerRepositories.nodes[0].id")'
+    -   echo "CONTAINER REPO ID = $CONT_REPO_ID"
+    -   'CACHE_TS=$(curl "https://gitlab.com/api/graphql" --header "Content-Type: application/json" --request POST --data "{\"query\": \"query { containerRepository(id: \\\"$CONT_REPO_ID\\\") { tags( first: 1 name: \\\"latest\\\" ) { nodes { location publishedAt }}}} \"}" | jq -r ".data.containerRepository.tags.nodes[0].publishedAt" | cut -d "+" -f1 | sed "s/T/ /g" )'
+    -   CACHE_EPOCH=$(date -d "$CACHE_TS" +%s)
+    - fi
+    - EARTHLY_EPOCH=$(git log -1 --format=%ct Earthfile)
+    - echo "CACHE EPOCH = $CACHE_EPOCH, EARTHLY EPOCH = $EARTHLY_EPOCH"
+    - if [[ $EARTHLY_EPOCH -gt $CACHE_EPOCH ]] || [[ "$CI_PIPELINE_SOURCE" == "schedule" ]] || [[ "$CI_PIPELINE_SOURCE" == "web" ]] ; then
+    -   *earthly_setup
+    -   earthly --use-inline-cache --save-inline-cache --strict --push -P +build-linux-cache $project_args
+    - else
+    -   echo "No need to rebuild"
+    - fi
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+      when: never
+    - if: $IS_NIGHTLY == "true"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+
+# Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry
+test_build:
+  extends: .base
+  stage: test
+  script:
+    - *earthly_setup
+    - earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args
+  resource_group: test
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+      when: never
+    - if: $IS_NIGHTLY == "true"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+# Actual release -- triggered by pushing a new version tag
+
+release_job:
+  stage: release
+  image: registry.gitlab.com/gitlab-org/release-cli:latest
+  script:
+    - echo "running release_job"
+  release:                               # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties
+    tag_name: '$CI_COMMIT_TAG'
+    description: '$CI_COMMIT_TAG'
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
+
+create_build_machines:
+  stage: build_packages
+  tags:
+    - build-orchestration
+  script:
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
+  
+package_amd64_deb:
+  stage: build_packages
+  needs:
+    - create_build_machines
+  tags:
+    - build-amd64-deb
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-amd64-deb
+    - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
+
+package_arm64_deb:
+  stage: build_packages
+  needs:
+    - create_build_machines
+  tags:
+    - build-arm64-deb
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-arm64-deb
+    - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
+
+package_amd64_rpm:
+  stage: build_packages
+  needs:
+    - create_build_machines
+  tags:
+    - build-amd64-rpm
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-amd64-rpm
+    - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
+
+# publish_crates:
+#   stage: build_packages
+#   needs:
+#     - package_amd64_deb
+#   tags:
+#     - build-amd64-deb
+#   script:
+#     - vlt login
+#     - vlt run --command="cargo publish -p veilid-tools --dry-run"
+#     - vlt run --command="cargo publish -p veilid-tools"
+#     - vlt run --command="cargo publish -p veilid-core --dry-run"
+#     - vlt run --command="cargo publish -p veilid-core"
+#   rules:
+#     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
+
+# publish_python:
+#   stage: build_packages
+#   needs:
+#     - package_amd64_deb
+#   tags:
+#     - build-amd64-deb
+#   script:
+#     - vlt login
+#     - cd veilid-python && /home/gitlab-runner/.local/bin/poetry build
+#     - vlt run --command="/home/gitlab-runner/.local/bin/poetry publish"
+#   rules:
+#     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
 build_repositories:
   stage: distribute
@@ -30,16 +186,207 @@ build_repositories:
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
-deploy_repos:
+delete_build_machines:
   stage: distribute
   needs:
-    - build_repositories
+    - deploy_repos
   tags:
-    - repo-server
+    - build-orchestration
   script:
-    - bash scripts/cicd/repo-server/deploy-repo.sh
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
+# Dryrun release -- triggered by changes in .gitlab-ci.yml, CICD scripts, or Earthfile
+
+dryrun_create_build_machines:
+  stage: build_packages
+  tags:
+    - build-orchestration
+  script:
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+  
+dryrun_package_amd64_deb:
+  stage: build_packages
+  needs:
+    - dryrun_create_build_machines
+  tags:
+    - build-amd64-deb
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-amd64-deb
+    - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+
+dryrun_package_arm64_deb:
+  stage: build_packages
+  needs:
+    - dryrun_create_build_machines
+  tags:
+    - build-arm64-deb
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-arm64-deb
+    - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+    
+dryrun_package_amd64_rpm:
+  stage: build_packages
+  needs:
+    - dryrun_create_build_machines
+  tags:
+    - build-amd64-rpm
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-amd64-rpm
+    - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+
+dryrun_publish_crates:
+  stage: build_packages
+  needs:
+    - dryrun_create_build_machines
+  tags:
+    - build-amd64-deb
+  script:
+    - vlt login
+    - vlt run --command="cargo publish -p veilid-tools --dry-run"
+    - vlt run --command="cargo publish -p veilid-core --dry-run"
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+
+dryrun_publish_python:
+  stage: build_packages
+  needs:
+    - dryrun_create_build_machines
+  tags:
+    - build-amd64-deb
+  script:
+    - cd veilid-python && /home/gitlab-runner/.local/bin/poetry build
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+
+dryrun_build_repositories:
+  stage: distribute
+  tags:
+    - build-orchestration
+  variables:
+    SECURE_FILES_DOWNLOAD_PATH: './'
+  script:
+    - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash
+    - cp scripts/cicd/build-orchestration/generate-release.sh ~
+    - bash scripts/cicd/build-orchestration/distribute-packages.sh
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+
+dryrun_deploy_repos:
+  stage: distribute
+  needs:
+    - dryrun_build_repositories
+  tags: 
+    - repo-server
+  script: 
+    - ls -al $HOME/repo.tar
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
+
+dryrun_delete_build_machines:
+  stage: distribute
+  needs:
+    - dryrun_deploy_repos
+  tags:
+    - build-orchestration
+  script:
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
 
 
+# Nightly build pipeline
+
+nightly_create_build_machines:
+  stage: build_packages
+  tags:
+    - build-orchestration
+  script:
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
+  rules:
+    - if: $IS_NIGHTLY == "true"
+  
+nightly_package_amd64_deb:
+  stage: build_packages
+  needs:
+    - nightly_create_build_machines
+  tags:
+    - build-amd64-deb
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-amd64-deb --IS_NIGHTLY="$IS_NIGHTLY"
+    - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
+  rules:
+    - if: $IS_NIGHTLY == "true"
+
+nightly_package_arm64_deb:
+  stage: build_packages
+  needs:
+    - nightly_create_build_machines
+  tags:
+    - build-arm64-deb
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-arm64-deb --IS_NIGHTLY="$IS_NIGHTLY"
+    - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
+  rules:
+    - if: $IS_NIGHTLY == "true"
+
+nightly_package_amd64_rpm:
+  stage: build_packages
+  needs:
+    - nightly_create_build_machines
+  tags:
+    - build-amd64-rpm
+  script:
+    - earthly bootstrap
+    - earthly +package-linux-amd64-rpm --IS_NIGHTLY="$IS_NIGHTLY"
+    - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
+  rules:
+    - if: $IS_NIGHTLY == "true"
+
+nightly_build_repositories:
+  stage: distribute
+  tags:
+    - build-orchestration
+  variables:
+    SECURE_FILES_DOWNLOAD_PATH: './'
+  script:
+    - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash
+    - cp scripts/cicd/build-orchestration/generate-nightly-release.sh ~
+    - bash scripts/cicd/build-orchestration/distribute-nightly-packages.sh
+  rules:
+    - if: $IS_NIGHTLY == "true"
+
+nightly_delete_build_machines:
+  stage: distribute
+  needs:
+    - nightly_build_repositories
+  tags:
+    - build-orchestration
+  script:
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
+    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
+  rules:
+    - if: $IS_NIGHTLY == "true"