Git-Mirrors/veilid
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2024-12-28 16:59:33 -05:00
Code Issues Packages Projects Releases Wiki Activity
24f3438b31
veilid/.gitlab-ci.yml

456 lines
14 KiB
YAML
Raw Blame History

variables:
NO_DOCKER: 1
FORCE_COLOR: 1
EARTHLY_EXEC_CMD: "/bin/sh"
GIT_SUBMODULE_STRATEGY: normal
stages:
- prepare
- test
- build_packages
- release
- distribute
- failed
format:
stage: test
image: rust:latest
before_script:
- rustup component add rustfmt
script:
- cargo fmt --all -- --check --verbose
rules:
- if: $CI_COMMIT_TAG
when: never
- when: always
# base earthly setup for jobs
.base:
tags: [ saas-linux-medium-amd64 ]
image: docker
services:
- docker:dind
.earthly: &earthly_setup
- apk update && apk add git
- wget https://github.com/earthly/earthly/releases/download/v0.7.15/earthly-linux-amd64 -O /usr/local/bin/earthly
- chmod +x /usr/local/bin/earthly
- earthly bootstrap
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- test "$CI_PROJECT_PATH" != "veilid/veilid" && project_args="--CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE"
# Create the build container if:
# - no container in the registry
# - run as part of a schedule
# - run manually from the pipelines web page
build_cache:
extends: .base
stage: prepare
script:
- apk update && apk add jq && apk add curl
- if ! docker manifest inspect $CI_REGISTRY_IMAGE/build-cache:latest > /dev/null; then
- CACHE_EPOCH=0
- else
- 'CONT_REPO_ID=$(curl "https://gitlab.com/api/graphql" --header "Content-Type: application/json" --request POST --data "{\"query\": \"query { project(fullPath: \\\"$CI_PROJECT_PATH\\\" ) { containerRepositories( name: \\\"build-cache\\\" ) { nodes { id }}}} \"}" | jq -r ".data.project.containerRepositories.nodes[0].id")'
- echo "CONTAINER REPO ID = $CONT_REPO_ID"
- 'CACHE_TS=$(curl "https://gitlab.com/api/graphql" --header "Content-Type: application/json" --request POST --data "{\"query\": \"query { containerRepository(id: \\\"$CONT_REPO_ID\\\") { tags( first: 1 name: \\\"latest\\\" ) { nodes { location publishedAt }}}} \"}" | jq -r ".data.containerRepository.tags.nodes[0].publishedAt" | cut -d "+" -f1 | sed "s/T/ /g" )'
- CACHE_EPOCH=$(date -d "$CACHE_TS" +%s)
- fi
- EARTHLY_EPOCH=$(git log -1 --format=%ct Earthfile)
- echo "CACHE EPOCH = $CACHE_EPOCH, EARTHLY EPOCH = $EARTHLY_EPOCH"
- if [[ $EARTHLY_EPOCH -gt $CACHE_EPOCH ]] || [[ "$CI_PIPELINE_SOURCE" == "schedule" ]] || [[ "$CI_PIPELINE_SOURCE" == "web" ]] ; then
- *earthly_setup
- earthly --use-inline-cache --save-inline-cache --strict --push -P +build-linux-cache $project_args
- else
- echo "No need to rebuild"
- fi
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
when: never
- if: $IS_NIGHTLY == "true"
when: never
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_PIPELINE_SOURCE == "push"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry
test_build:
extends: .base
stage: test
script:
- *earthly_setup
- earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args
resource_group: test
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
when: never
- if: $IS_NIGHTLY == "true"
when: never
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_PIPELINE_SOURCE == "push"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- changes:
- veilid-*/**
- '**/*[Cc]argo*'
semgrep:
# A Docker image with Semgrep installed.
image: semgrep/semgrep
rules:
# Scan changed files in MRs, (diff-aware scanning):
- if: $CI_MERGE_REQUEST_IID
# Scan mainline (default) branches and report all findings.
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
# Connect to Semgrep AppSec Platform through your SEMGREP_APP_TOKEN.
# Generate a token from Semgrep AppSec Platform > Settings
# and add it as a variable in your GitLab CI/CD project settings.
SEMGREP_APP_TOKEN: $SEMGREP_APP_TOKEN
# Upload findings to GitLab SAST Dashboard:
SEMGREP_GITLAB_JSON: "1"
# Other optional settings in the `variables` block:
# Receive inline MR comments (requires Semgrep AppSec Platform account)
# Setup instructions:
# https://semgrep.dev/docs/semgrep-appsec-platform/gitlab-mr-comments
# GITLAB_TOKEN: $PAT
# Run the "semgrep ci" command on the command line of the docker image and send findings
# to GitLab SAST.
script: semgrep ci --gitlab-sast > gl-sast-report.json || true
artifacts:
reports:
sast: gl-sast-report.json
# Actual release -- triggered by pushing a new version tag
release_job:
stage: release
image: registry.gitlab.com/gitlab-org/release-cli:latest
script:
- echo "running release_job"
release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties
tag_name: '$CI_COMMIT_TAG'
description: '$CI_COMMIT_TAG'
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
create_build_machines:
stage: build_packages
tags:
- build-orchestration
script:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
package_amd64_deb:
stage: build_packages
needs:
- create_build_machines
tags:
- build-amd64-deb
script:
- earthly bootstrap
- earthly +package-linux-amd64-deb
- bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
package_arm64_deb:
stage: build_packages
needs:
- create_build_machines
tags:
- build-arm64-deb
script:
- earthly bootstrap
- earthly +package-linux-arm64-deb
- bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
package_amd64_rpm:
stage: build_packages
needs:
- create_build_machines
tags:
- build-amd64-rpm
script:
- earthly bootstrap
- earthly +package-linux-amd64-rpm
- bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
#publish_crates:
# stage: build_packages
# needs:
# - package_amd64_deb
# tags:
# - build-amd64-deb
# script:
# - vlt login
# - vlt run --command="cargo publish -p veilid-tools --dry-run"
# - vlt run --command="cargo publish -p veilid-tools"
# - vlt run --command="cargo publish -p veilid-core --dry-run"
# - vlt run --command="cargo publish -p veilid-core"
# rules:
# - if: '$CI_COMMIT_TAG =~ /v\d.+/'
#publish_python:
# stage: build_packages
# needs:
# - publish_crates
# tags:
# - build-amd64-deb
# script:
# - vlt login
# - cd veilid-python && /home/gitlab-runner/.local/bin/poetry build
# - vlt run --command="/home/gitlab-runner/.local/bin/poetry publish"
# rules:
# - if: '$CI_COMMIT_TAG =~ /v\d.+/'
build_repositories:
stage: distribute
#needs:
# - publish_python
tags:
- build-orchestration
variables:
SECURE_FILES_DOWNLOAD_PATH: './'
script:
- curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash
- cp scripts/cicd/build-orchestration/rpm-repo-building/Dockerfile ~/rpm-build-container
- cp scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh ~/rpm-build-container
- cp scripts/cicd/build-orchestration/generate-stable-release.sh ~
- bash scripts/cicd/build-orchestration/distribute-stable-packages.sh
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
delete_build_machines:
stage: distribute
needs:
- build_repositories
tags:
- build-orchestration
script:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
rules:
- if: '$CI_COMMIT_TAG =~ /v\d.+/'
# Dryrun release -- triggered by changes in .gitlab-ci.yml, CICD scripts, or Earthfile
dryrun_create_build_machines:
stage: build_packages
tags:
- build-orchestration
script:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_package_amd64_deb:
stage: build_packages
needs:
- dryrun_create_build_machines
tags:
- build-amd64-deb
script:
- earthly bootstrap
- earthly +package-linux-amd64-deb
- bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_package_arm64_deb:
stage: build_packages
needs:
- dryrun_create_build_machines
tags:
- build-arm64-deb
script:
- earthly bootstrap
- earthly +package-linux-arm64-deb
- bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_package_amd64_rpm:
stage: build_packages
needs:
- dryrun_create_build_machines
tags:
- build-amd64-rpm
script:
- earthly bootstrap
- earthly +package-linux-amd64-rpm
- bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_publish_crates:
stage: build_packages
needs:
- dryrun_create_build_machines
tags:
- build-amd64-deb
script:
- vlt login
- vlt run --command="cargo publish -p veilid-tools --dry-run"
- vlt run --command="cargo publish -p veilid-core --dry-run"
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_publish_python:
stage: build_packages
needs:
- dryrun_create_build_machines
tags:
- build-amd64-deb
script:
- cd veilid-python && /home/gitlab-runner/.local/bin/poetry build
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_build_repositories:
stage: distribute
tags:
- build-orchestration
variables:
SECURE_FILES_DOWNLOAD_PATH: './'
script:
- curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash
- cp scripts/cicd/build-orchestration/generate-release.sh ~
- bash scripts/cicd/build-orchestration/distribute-packages.sh
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_deploy_repos:
stage: distribute
needs:
- dryrun_build_repositories
tags:
- repo-server
script:
- ls -al $HOME/repo.tar
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
dryrun_delete_build_machines:
stage: distribute
needs:
- dryrun_deploy_repos
tags:
- build-orchestration
script:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
rules:
- if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
# Nightly build pipeline
nightly_create_build_machines:
stage: build_packages
tags:
- build-orchestration
script:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
rules:
- if: $IS_NIGHTLY == "true"
nightly_package_amd64_deb:
stage: build_packages
needs:
- nightly_create_build_machines
tags:
- build-amd64-deb
script:
- earthly bootstrap
- earthly +package-linux-amd64-deb --IS_NIGHTLY="$IS_NIGHTLY"
- bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
rules:
- if: $IS_NIGHTLY == "true"
nightly_package_arm64_deb:
stage: build_packages
needs:
- nightly_create_build_machines
tags:
- build-arm64-deb
script:
- earthly bootstrap
- earthly +package-linux-arm64-deb --IS_NIGHTLY="$IS_NIGHTLY"
- bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
rules:
- if: $IS_NIGHTLY == "true"
nightly_package_amd64_rpm:
stage: build_packages
needs:
- nightly_create_build_machines
tags:
- build-amd64-rpm
script:
- earthly bootstrap
- earthly +package-linux-amd64-rpm --IS_NIGHTLY="$IS_NIGHTLY"
- bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
rules:
- if: $IS_NIGHTLY == "true"
nightly_build_repositories:
stage: distribute
tags:
- build-orchestration
variables:
SECURE_FILES_DOWNLOAD_PATH: './'
script:
- curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash
- cp scripts/cicd/build-orchestration/generate-nightly-release.sh ~
- bash scripts/cicd/build-orchestration/distribute-nightly-packages.sh
rules:
- if: $IS_NIGHTLY == "true"
nightly_delete_build_machines:
stage: distribute
needs:
- nightly_build_repositories
tags:
- build-orchestration
script:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
rules:
- if: $IS_NIGHTLY == "true"
# If any steps fail this will run the build machine tear down scripts
failed_pipeline_actions:
stage: failed
tags:
- build-orchestration
script:
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
- bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
when: on_failure
Reference in New Issue View Git Blame Copy Permalink