diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index db50e13d..7c4afcad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,13 +5,169 @@ variables: GIT_SUBMODULE_STRATEGY: normal stages: - #- prepare - #- test - #- build_packages + - prepare + - test + - build_packages - release - distribute +# base earthly setup for jobs +.base: + tags: [ saas-linux-medium-amd64 ] + image: docker + services: + - docker:dind +.earthly: &earthly_setup + - apk update && apk add git + - wget https://github.com/earthly/earthly/releases/download/v0.7.15/earthly-linux-amd64 -O /usr/local/bin/earthly + - chmod +x /usr/local/bin/earthly + - earthly bootstrap + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + - test "$CI_PROJECT_PATH" != "veilid/veilid" && project_args="--CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE" + +# Create the build container if: +# - no container in the registry +# - run as part of a schedule +# - run manually from the pipelines web page +build_cache: + extends: .base + stage: prepare + script: + - apk update && apk add jq && apk add curl + - if ! docker manifest inspect $CI_REGISTRY_IMAGE/build-cache:latest > /dev/null; then + - CACHE_EPOCH=0 + - else + - 'CONT_REPO_ID=$(curl "https://gitlab.com/api/graphql" --header "Content-Type: application/json" --request POST --data "{\"query\": \"query { project(fullPath: \\\"$CI_PROJECT_PATH\\\" ) { containerRepositories( name: \\\"build-cache\\\" ) { nodes { id }}}} \"}" | jq -r ".data.project.containerRepositories.nodes[0].id")' + - echo "CONTAINER REPO ID = $CONT_REPO_ID" + - 'CACHE_TS=$(curl "https://gitlab.com/api/graphql" --header "Content-Type: application/json" --request POST --data "{\"query\": \"query { containerRepository(id: \\\"$CONT_REPO_ID\\\") { tags( first: 1 name: \\\"latest\\\" ) { nodes { location publishedAt }}}} \"}" | jq -r ".data.containerRepository.tags.nodes[0].publishedAt" | cut -d "+" -f1 | sed "s/T/ /g" )' + - CACHE_EPOCH=$(date -d "$CACHE_TS" +%s) + - fi + - EARTHLY_EPOCH=$(git log -1 --format=%ct Earthfile) + - echo "CACHE EPOCH = $CACHE_EPOCH, EARTHLY EPOCH = $EARTHLY_EPOCH" + - if [[ $EARTHLY_EPOCH -gt $CACHE_EPOCH ]] || [[ "$CI_PIPELINE_SOURCE" == "schedule" ]] || [[ "$CI_PIPELINE_SOURCE" == "web" ]] ; then + - *earthly_setup + - earthly --use-inline-cache --save-inline-cache --strict --push -P +build-linux-cache $project_args + - else + - echo "No need to rebuild" + - fi + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + when: never + - if: $IS_NIGHTLY == "true" + when: never + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_PIPELINE_SOURCE == "push" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + + +# Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry +test_build: + extends: .base + stage: test + script: + - *earthly_setup + - earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args + resource_group: test + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + when: never + - if: $IS_NIGHTLY == "true" + when: never + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_PIPELINE_SOURCE == "push" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +# Actual release -- triggered by pushing a new version tag + +release_job: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + script: + - echo "running release_job" + release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties + tag_name: '$CI_COMMIT_TAG' + description: '$CI_COMMIT_TAG' + rules: + - if: '$CI_COMMIT_TAG =~ /v\d.+/' + +create_build_machines: + stage: build_packages + tags: + - build-orchestration + script: + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm + rules: + - if: '$CI_COMMIT_TAG =~ /v\d.+/' + +package_amd64_deb: + stage: build_packages + needs: + - create_build_machines + tags: + - build-amd64-deb + script: + - earthly bootstrap + - earthly +package-linux-amd64-deb + - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh + rules: + - if: '$CI_COMMIT_TAG =~ /v\d.+/' + +package_arm64_deb: + stage: build_packages + needs: + - create_build_machines + tags: + - build-arm64-deb + script: + - earthly bootstrap + - earthly +package-linux-arm64-deb + - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh + rules: + - if: '$CI_COMMIT_TAG =~ /v\d.+/' + +package_amd64_rpm: + stage: build_packages + needs: + - create_build_machines + tags: + - build-amd64-rpm + script: + - earthly bootstrap + - earthly +package-linux-amd64-rpm + - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh + rules: + - if: '$CI_COMMIT_TAG =~ /v\d.+/' + +# publish_crates: +# stage: build_packages +# needs: +# - package_amd64_deb +# tags: +# - build-amd64-deb +# script: +# - vlt login +# - vlt run --command="cargo publish -p veilid-tools --dry-run" +# - vlt run --command="cargo publish -p veilid-tools" +# - vlt run --command="cargo publish -p veilid-core --dry-run" +# - vlt run --command="cargo publish -p veilid-core" +# rules: +# - if: '$CI_COMMIT_TAG =~ /v\d.+/' + +# publish_python: +# stage: build_packages +# needs: +# - package_amd64_deb +# tags: +# - build-amd64-deb +# script: +# - vlt login +# - cd veilid-python && /home/gitlab-runner/.local/bin/poetry build +# - vlt run --command="/home/gitlab-runner/.local/bin/poetry publish" +# rules: +# - if: '$CI_COMMIT_TAG =~ /v\d.+/' build_repositories: stage: distribute @@ -30,16 +186,207 @@ build_repositories: rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' -deploy_repos: +delete_build_machines: stage: distribute needs: - - build_repositories - tags: - - repo-server - script: - - bash scripts/cicd/repo-server/deploy-repo.sh + - deploy_repos + tags: + - build-orchestration + script: + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' +# Dryrun release -- triggered by changes in .gitlab-ci.yml, CICD scripts, or Earthfile + +dryrun_create_build_machines: + stage: build_packages + tags: + - build-orchestration + script: + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + +dryrun_package_amd64_deb: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-amd64-deb + script: + - earthly bootstrap + - earthly +package-linux-amd64-deb + - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + +dryrun_package_arm64_deb: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-arm64-deb + script: + - earthly bootstrap + - earthly +package-linux-arm64-deb + - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + +dryrun_package_amd64_rpm: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-amd64-rpm + script: + - earthly bootstrap + - earthly +package-linux-amd64-rpm + - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + +dryrun_publish_crates: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-amd64-deb + script: + - vlt login + - vlt run --command="cargo publish -p veilid-tools --dry-run" + - vlt run --command="cargo publish -p veilid-core --dry-run" + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + +dryrun_publish_python: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-amd64-deb + script: + - cd veilid-python && /home/gitlab-runner/.local/bin/poetry build + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + +dryrun_build_repositories: + stage: distribute + tags: + - build-orchestration + variables: + SECURE_FILES_DOWNLOAD_PATH: './' + script: + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash + - cp scripts/cicd/build-orchestration/generate-release.sh ~ + - bash scripts/cicd/build-orchestration/distribute-packages.sh + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + +dryrun_deploy_repos: + stage: distribute + needs: + - dryrun_build_repositories + tags: + - repo-server + script: + - ls -al $HOME/repo.tar + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + +dryrun_delete_build_machines: + stage: distribute + needs: + - dryrun_deploy_repos + tags: + - build-orchestration + script: + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ +# Nightly build pipeline + +nightly_create_build_machines: + stage: build_packages + tags: + - build-orchestration + script: + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm + rules: + - if: $IS_NIGHTLY == "true" + +nightly_package_amd64_deb: + stage: build_packages + needs: + - nightly_create_build_machines + tags: + - build-amd64-deb + script: + - earthly bootstrap + - earthly +package-linux-amd64-deb --IS_NIGHTLY="$IS_NIGHTLY" + - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh + rules: + - if: $IS_NIGHTLY == "true" + +nightly_package_arm64_deb: + stage: build_packages + needs: + - nightly_create_build_machines + tags: + - build-arm64-deb + script: + - earthly bootstrap + - earthly +package-linux-arm64-deb --IS_NIGHTLY="$IS_NIGHTLY" + - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh + rules: + - if: $IS_NIGHTLY == "true" + +nightly_package_amd64_rpm: + stage: build_packages + needs: + - nightly_create_build_machines + tags: + - build-amd64-rpm + script: + - earthly bootstrap + - earthly +package-linux-amd64-rpm --IS_NIGHTLY="$IS_NIGHTLY" + - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh + rules: + - if: $IS_NIGHTLY == "true" + +nightly_build_repositories: + stage: distribute + tags: + - build-orchestration + variables: + SECURE_FILES_DOWNLOAD_PATH: './' + script: + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash + - cp scripts/cicd/build-orchestration/generate-nightly-release.sh ~ + - bash scripts/cicd/build-orchestration/distribute-nightly-packages.sh + rules: + - if: $IS_NIGHTLY == "true" + +nightly_delete_build_machines: + stage: distribute + needs: + - nightly_build_repositories + tags: + - build-orchestration + script: + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm + rules: + - if: $IS_NIGHTLY == "true" \ No newline at end of file