From 038f4d21218d0d6270fd5de0f383ef944e685e85 Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Mon, 29 Apr 2024 14:40:07 -0500 Subject: [PATCH 01/12] Added CICD dry run This is the first attempt at setting up a dry run pipeline to test changes to the CICD config without actually publishing the compiled binaries and packages built by the process. The dry run should be triggered by any changes to .gitlab-ci.yml or changes to any of the scripts under scripts/cicd/. --- .gitlab-ci.yml | 142 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c16d688b..d48765ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,8 @@ test_build: resource_group: test # when: manual +# Actual release -- triggered by pushing a new version tag + release_job: stage: release image: registry.gitlab.com/gitlab-org/release-cli:latest @@ -177,3 +179,143 @@ delete_build_machines: - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' + +# Dryrun release -- triggered by changes in .gitlab-ci.yml or CICD scripts + +dryrun_create_build_machines: + stage: build_packages + tags: + - build-orchestration + script: + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - .gitlab-ci.yml + - scripts/cicd/**/* + +dryrun_package_amd64_deb: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-amd64-deb + script: + - earthly bootstrap + - earthly +package-linux-amd64-deb + - bash scripts/cicd/build-machine/scp-to-orchestrator.sh + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - .gitlab-ci.yml + - scripts/cicd/**/* + +dryrun_package_arm64_deb: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-arm64-deb + script: + - earthly bootstrap + - earthly +package-linux-arm64-deb + - bash scripts/cicd/build-machine/scp-to-orchestrator.sh + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - .gitlab-ci.yml + - scripts/cicd/**/* + +dryrun_package_amd64_rpm: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-amd64-rpm + script: + - earthly bootstrap + - earthly +package-linux-amd64-rpm + - bash scripts/cicd/build-machine/scp-to-orchestrator.sh + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - .gitlab-ci.yml + - scripts/cicd/**/* + +dryrun_publish_crates: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-amd64-deb + script: + - vlt login + - vlt run --command="cargo publish -p veilid-tools --dry-run" + - vlt run --command="cargo publish -p veilid-core --dry-run" + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - .gitlab-ci.yml + - scripts/cicd/**/* + +dryrun_publish_python: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-amd64-deb + script: + - vlt login + - cd veilid-python && /home/gitlab-runner/.local/bin/poetry build + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - .gitlab-ci.yml + - scripts/cicd/**/* + +dryrun_build_repositories: + stage: distribute + tags: + - build-orchestration + variables: + SECURE_FILES_DOWNLOAD_PATH: './' + script: + - cp scripts/cicd/build-orchestration/generate-release.sh ~ + - bash scripts/cicd/build-orchestration/distribute-packages.sh + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - .gitlab-ci.yml + - scripts/cicd/**/* + +dryrun_deploy_repos: + stage: distribute + needs: + - dryrun_build_repositories + tags: + - repo-server + script: + - ls -al repo.tar + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - .gitlab-ci.yml + - scripts/cicd/**/* + +dryrun_delete_build_machines: + stage: distribute + needs: + - dryrun_deploy_repos + tags: + - build-orchestration + script: + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb + - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - .gitlab-ci.yml + - scripts/cicd/**/* \ No newline at end of file From c33f0f19daea7732705a99529e40538ae1c5c97a Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Mon, 29 Apr 2024 15:24:04 -0500 Subject: [PATCH 02/12] Added Earthfile condition and RPM spec fix Dry run will now trigger for Earthfile changes. I also reordered the changelog in veilid-server.spec to descending to correct that error. Commented out the crates.io publishing dry run. This branch does not have access to the protected variables. --- .gitlab-ci.yml | 43 ++++++++++++-------- package/rpm/veilid-server/veilid-server.spec | 5 ++- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d48765ff..bc1ddf12 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -180,7 +180,7 @@ delete_build_machines: rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' -# Dryrun release -- triggered by changes in .gitlab-ci.yml or CICD scripts +# Dryrun release -- triggered by changes in .gitlab-ci.yml, CICD scripts, or Earthfile dryrun_create_build_machines: stage: build_packages @@ -195,6 +195,7 @@ dryrun_create_build_machines: changes: - .gitlab-ci.yml - scripts/cicd/**/* + - Earthfile dryrun_package_amd64_deb: stage: build_packages @@ -211,6 +212,7 @@ dryrun_package_amd64_deb: changes: - .gitlab-ci.yml - scripts/cicd/**/* + - Earthfile dryrun_package_arm64_deb: stage: build_packages @@ -227,6 +229,7 @@ dryrun_package_arm64_deb: changes: - .gitlab-ci.yml - scripts/cicd/**/* + - Earthfile dryrun_package_amd64_rpm: stage: build_packages @@ -243,22 +246,24 @@ dryrun_package_amd64_rpm: changes: - .gitlab-ci.yml - scripts/cicd/**/* + - Earthfile -dryrun_publish_crates: - stage: build_packages - needs: - - dryrun_create_build_machines - tags: - - build-amd64-deb - script: - - vlt login - - vlt run --command="cargo publish -p veilid-tools --dry-run" - - vlt run --command="cargo publish -p veilid-core --dry-run" - rules: - - if: $CI_PIPELINE_SOURCE == "push" - changes: - - .gitlab-ci.yml - - scripts/cicd/**/* +# dryrun_publish_crates: +# stage: build_packages +# needs: +# - dryrun_create_build_machines +# tags: +# - build-amd64-deb +# script: +# - vlt login +# - vlt run --command="cargo publish -p veilid-tools --dry-run" +# - vlt run --command="cargo publish -p veilid-core --dry-run" +# rules: +# - if: $CI_PIPELINE_SOURCE == "push" +# changes: +# - .gitlab-ci.yml +# - scripts/cicd/**/* +# - Earthfile dryrun_publish_python: stage: build_packages @@ -274,6 +279,7 @@ dryrun_publish_python: changes: - .gitlab-ci.yml - scripts/cicd/**/* + - Earthfile dryrun_build_repositories: stage: distribute @@ -289,6 +295,7 @@ dryrun_build_repositories: changes: - .gitlab-ci.yml - scripts/cicd/**/* + - Earthfile dryrun_deploy_repos: stage: distribute @@ -303,6 +310,7 @@ dryrun_deploy_repos: changes: - .gitlab-ci.yml - scripts/cicd/**/* + - Earthfile dryrun_delete_build_machines: stage: distribute @@ -318,4 +326,5 @@ dryrun_delete_build_machines: - if: $CI_PIPELINE_SOURCE == "push" changes: - .gitlab-ci.yml - - scripts/cicd/**/* \ No newline at end of file + - scripts/cicd/**/* + - Earthfile \ No newline at end of file diff --git a/package/rpm/veilid-server/veilid-server.spec b/package/rpm/veilid-server/veilid-server.spec index 9d3d50d1..f8a881c2 100644 --- a/package/rpm/veilid-server/veilid-server.spec +++ b/package/rpm/veilid-server/veilid-server.spec @@ -57,7 +57,8 @@ else fi %changelog -* Sun Jul 2 2023 TC -- experimental RPM building * Sun Apr 28 2024 Christien Rioux - add ipc directory to installation +* Sun Jul 2 2023 TC +- experimental RPM building + From fa1ca77d35e5b10f65a81b3d109d76a05f46a4bc Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Mon, 29 Apr 2024 16:03:38 -0500 Subject: [PATCH 03/12] Further dry run trigger refinements The dry run should now be configured if changes to .gitlab-ci.yml, Earthfile, contents of scripts/cicd, or contents of package arrive as the result of a push OR merge but NOT when a tag is present. I think that will prevent the dry run executing alongside the actual release process when a new release tag is pushed but that also contains changes to the above mentioned files. --- .gitlab-ci.yml | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc1ddf12..cdc4e271 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -191,11 +191,12 @@ dryrun_create_build_machines: - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm rules: - - if: $CI_PIPELINE_SOURCE == "push" + - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL changes: - .gitlab-ci.yml - scripts/cicd/**/* - Earthfile + - package/**/* dryrun_package_amd64_deb: stage: build_packages @@ -208,11 +209,12 @@ dryrun_package_amd64_deb: - earthly +package-linux-amd64-deb - bash scripts/cicd/build-machine/scp-to-orchestrator.sh rules: - - if: $CI_PIPELINE_SOURCE == "push" + - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL changes: - .gitlab-ci.yml - scripts/cicd/**/* - Earthfile + - package/**/* dryrun_package_arm64_deb: stage: build_packages @@ -225,12 +227,13 @@ dryrun_package_arm64_deb: - earthly +package-linux-arm64-deb - bash scripts/cicd/build-machine/scp-to-orchestrator.sh rules: - - if: $CI_PIPELINE_SOURCE == "push" + - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL changes: - .gitlab-ci.yml - scripts/cicd/**/* - Earthfile - + - package/**/* + dryrun_package_amd64_rpm: stage: build_packages needs: @@ -242,11 +245,12 @@ dryrun_package_amd64_rpm: - earthly +package-linux-amd64-rpm - bash scripts/cicd/build-machine/scp-to-orchestrator.sh rules: - - if: $CI_PIPELINE_SOURCE == "push" + - ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL changes: - .gitlab-ci.yml - scripts/cicd/**/* - Earthfile + - package/**/* # dryrun_publish_crates: # stage: build_packages @@ -259,11 +263,12 @@ dryrun_package_amd64_rpm: # - vlt run --command="cargo publish -p veilid-tools --dry-run" # - vlt run --command="cargo publish -p veilid-core --dry-run" # rules: -# - if: $CI_PIPELINE_SOURCE == "push" +# - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL # changes: # - .gitlab-ci.yml # - scripts/cicd/**/* # - Earthfile +# - package/**/* dryrun_publish_python: stage: build_packages @@ -272,14 +277,14 @@ dryrun_publish_python: tags: - build-amd64-deb script: - - vlt login - cd veilid-python && /home/gitlab-runner/.local/bin/poetry build rules: - - if: $CI_PIPELINE_SOURCE == "push" + - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL changes: - .gitlab-ci.yml - scripts/cicd/**/* - Earthfile + - package/**/* dryrun_build_repositories: stage: distribute @@ -291,11 +296,12 @@ dryrun_build_repositories: - cp scripts/cicd/build-orchestration/generate-release.sh ~ - bash scripts/cicd/build-orchestration/distribute-packages.sh rules: - - if: $CI_PIPELINE_SOURCE == "push" + - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL changes: - .gitlab-ci.yml - scripts/cicd/**/* - Earthfile + - package/**/* dryrun_deploy_repos: stage: distribute @@ -306,11 +312,12 @@ dryrun_deploy_repos: script: - ls -al repo.tar rules: - - if: $CI_PIPELINE_SOURCE == "push" + - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL changes: - .gitlab-ci.yml - scripts/cicd/**/* - Earthfile + - package/**/* dryrun_delete_build_machines: stage: distribute @@ -323,8 +330,9 @@ dryrun_delete_build_machines: - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm rules: - - if: $CI_PIPELINE_SOURCE == "push" + - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL changes: - .gitlab-ci.yml - scripts/cicd/**/* - - Earthfile \ No newline at end of file + - Earthfile + - package/**/* \ No newline at end of file From 0a1a37f048cd0ed41935dceb1d340c7f5ef58af2 Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Mon, 29 Apr 2024 16:12:37 -0500 Subject: [PATCH 04/12] Replaced missing 'if:' on line 249. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cdc4e271..ef1838bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -245,7 +245,7 @@ dryrun_package_amd64_rpm: - earthly +package-linux-amd64-rpm - bash scripts/cicd/build-machine/scp-to-orchestrator.sh rules: - - ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL + - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL changes: - .gitlab-ci.yml - scripts/cicd/**/* From bd4600576f2a9e7899f6db40229e002a61f463a5 Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Mon, 29 Apr 2024 16:22:58 -0500 Subject: [PATCH 05/12] Switched to regex trigger The previous trigger condition wasn't liked by Gitlab's yaml parser. I'm switching to trying a commit message based regex trigger. This message does not contain the trigger and so should not fire the dry run. --- .gitlab-ci.yml | 56 ++++++++------------------------------------------ 1 file changed, 8 insertions(+), 48 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ef1838bf..41f95a56 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -191,12 +191,7 @@ dryrun_create_build_machines: - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm rules: - - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL - changes: - - .gitlab-ci.yml - - scripts/cicd/**/* - - Earthfile - - package/**/* + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ dryrun_package_amd64_deb: stage: build_packages @@ -209,12 +204,7 @@ dryrun_package_amd64_deb: - earthly +package-linux-amd64-deb - bash scripts/cicd/build-machine/scp-to-orchestrator.sh rules: - - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL - changes: - - .gitlab-ci.yml - - scripts/cicd/**/* - - Earthfile - - package/**/* + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ dryrun_package_arm64_deb: stage: build_packages @@ -227,12 +217,7 @@ dryrun_package_arm64_deb: - earthly +package-linux-arm64-deb - bash scripts/cicd/build-machine/scp-to-orchestrator.sh rules: - - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL - changes: - - .gitlab-ci.yml - - scripts/cicd/**/* - - Earthfile - - package/**/* + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ dryrun_package_amd64_rpm: stage: build_packages @@ -245,12 +230,7 @@ dryrun_package_amd64_rpm: - earthly +package-linux-amd64-rpm - bash scripts/cicd/build-machine/scp-to-orchestrator.sh rules: - - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL - changes: - - .gitlab-ci.yml - - scripts/cicd/**/* - - Earthfile - - package/**/* + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ # dryrun_publish_crates: # stage: build_packages @@ -279,12 +259,7 @@ dryrun_publish_python: script: - cd veilid-python && /home/gitlab-runner/.local/bin/poetry build rules: - - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL - changes: - - .gitlab-ci.yml - - scripts/cicd/**/* - - Earthfile - - package/**/* + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ dryrun_build_repositories: stage: distribute @@ -296,12 +271,7 @@ dryrun_build_repositories: - cp scripts/cicd/build-orchestration/generate-release.sh ~ - bash scripts/cicd/build-orchestration/distribute-packages.sh rules: - - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL - changes: - - .gitlab-ci.yml - - scripts/cicd/**/* - - Earthfile - - package/**/* + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ dryrun_deploy_repos: stage: distribute @@ -312,12 +282,7 @@ dryrun_deploy_repos: script: - ls -al repo.tar rules: - - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL - changes: - - .gitlab-ci.yml - - scripts/cicd/**/* - - Earthfile - - package/**/* + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ dryrun_delete_build_machines: stage: distribute @@ -330,9 +295,4 @@ dryrun_delete_build_machines: - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm rules: - - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL - changes: - - .gitlab-ci.yml - - scripts/cicd/**/* - - Earthfile - - package/**/* \ No newline at end of file + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ \ No newline at end of file From 37dbe70e0570f6352c643c038890697a0374ea78 Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Mon, 29 Apr 2024 16:26:13 -0500 Subject: [PATCH 06/12] Testing dry run trigger The trigger is [ci dryrun] --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41f95a56..2ef0b8c6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -295,4 +295,6 @@ dryrun_delete_build_machines: - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm rules: - - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ \ No newline at end of file + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + +# Added to test commit message dry run trigger \ No newline at end of file From 4d9787fb86f4b96a5947c49db59c5bd3ac945cae Mon Sep 17 00:00:00 2001 From: TC Johnson Date: Mon, 29 Apr 2024 17:07:40 -0500 Subject: [PATCH 07/12] Fixes scp from build machines to orchestrator Each machine needed a script this specified that machine's arch and OS type. Also added a rule on the cache and test jobs to not run if the dry run trigger is present. [ci dryrun] --- .gitlab-ci.yml | 20 ++++++++++++------- ...r.sh => scp-amd64-debs-to-orchestrator.sh} | 0 .../scp-amd64-rpms-to-orchestrator.sh | 9 +++++++++ .../scp-arm64-debs-to-orchestrator.sh | 9 +++++++++ 4 files changed, 31 insertions(+), 7 deletions(-) rename scripts/cicd/build-machine/{scp-to-orchestrator.sh => scp-amd64-debs-to-orchestrator.sh} (100%) create mode 100644 scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh create mode 100644 scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ef0b8c6..3dc2cd4a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,6 +40,10 @@ build_cache: - else - echo "No need to rebuild" - fi + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + - when: never + # Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry test_build: @@ -53,7 +57,9 @@ test_build: - *earthly_setup - earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args resource_group: test - # when: manual + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ + - when: never # Actual release -- triggered by pushing a new version tag @@ -88,7 +94,7 @@ package_amd64_deb: script: - earthly bootstrap - earthly +package-linux-amd64-deb - - bash scripts/cicd/build-machine/scp-to-orchestrator.sh + - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -101,7 +107,7 @@ package_arm64_deb: script: - earthly bootstrap - earthly +package-linux-arm64-deb - - bash scripts/cicd/build-machine/scp-to-orchestrator.sh + - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -114,7 +120,7 @@ package_amd64_rpm: script: - earthly bootstrap - earthly +package-linux-amd64-rpm - - bash scripts/cicd/build-machine/scp-to-orchestrator.sh + - bash scripts/cicd/build-machine/scp-amd64-rmps-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -202,7 +208,7 @@ dryrun_package_amd64_deb: script: - earthly bootstrap - earthly +package-linux-amd64-deb - - bash scripts/cicd/build-machine/scp-to-orchestrator.sh + - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh rules: - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ @@ -215,7 +221,7 @@ dryrun_package_arm64_deb: script: - earthly bootstrap - earthly +package-linux-arm64-deb - - bash scripts/cicd/build-machine/scp-to-orchestrator.sh + - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh rules: - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ @@ -228,7 +234,7 @@ dryrun_package_amd64_rpm: script: - earthly bootstrap - earthly +package-linux-amd64-rpm - - bash scripts/cicd/build-machine/scp-to-orchestrator.sh + - bash scripts/cicd/build-machine/scp-amd64-rmps-to-orchestrator.sh rules: - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ diff --git a/scripts/cicd/build-machine/scp-to-orchestrator.sh b/scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh similarity index 100% rename from scripts/cicd/build-machine/scp-to-orchestrator.sh rename to scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh diff --git a/scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh b/scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh new file mode 100644 index 00000000..1f616f8d --- /dev/null +++ b/scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +mkdir workspace +cd workspace +cp ~/builds/C6yRimG-M/0/veilid/veilid/target/packages/*.rpm . +tar -cf amd64-rpms.tar *.rpm +scp *.tar gitlab-runner@10.116.0.5:~ +cd ~ +rm -rf workspace \ No newline at end of file diff --git a/scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh b/scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh new file mode 100644 index 00000000..7216ec59 --- /dev/null +++ b/scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +mkdir workspace +cd workspace +cp ~/builds/7TYBLKUtG/0/veilid/veilid/target/packages/*.deb . +tar -cf arm64-debs.tar *.deb +scp *.tar gitlab-runner@10.116.0.5:~ +cd ~ +rm -rf workspace \ No newline at end of file From 31347f3ce077661363cee03f177a238a25af90f2 Mon Sep 17 00:00:00 2001 From: TC Date: Mon, 29 Apr 2024 22:15:19 +0000 Subject: [PATCH 08/12] [ci dryrun] --- .gitlab-ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3dc2cd4a..117306cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,17 +49,16 @@ build_cache: test_build: extends: .base stage: test - only: - - main - - pushes - - merge_requests script: - *earthly_setup - earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args resource_group: test rules: - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ - - when: never + when: never + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_PIPELINE_SOURCE == "push" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Actual release -- triggered by pushing a new version tag From 058df249339c6da5a9467f010bfb75971d5fc09e Mon Sep 17 00:00:00 2001 From: TC Date: Mon, 29 Apr 2024 22:16:38 +0000 Subject: [PATCH 09/12] [ci dryrun] --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 117306cb..5da597ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ build_cache: - fi rules: - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ - - when: never + when: never # Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry From 59d261b899a2e869047430b7b7f008dbbcf965f9 Mon Sep 17 00:00:00 2001 From: TC Date: Mon, 29 Apr 2024 22:34:52 +0000 Subject: [PATCH 10/12] [ci dryrun] --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5da597ff..ff8fc6bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,7 +119,7 @@ package_amd64_rpm: script: - earthly bootstrap - earthly +package-linux-amd64-rpm - - bash scripts/cicd/build-machine/scp-amd64-rmps-to-orchestrator.sh + - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh rules: - if: '$CI_COMMIT_TAG =~ /v\d.+/' @@ -233,7 +233,7 @@ dryrun_package_amd64_rpm: script: - earthly bootstrap - earthly +package-linux-amd64-rpm - - bash scripts/cicd/build-machine/scp-amd64-rmps-to-orchestrator.sh + - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh rules: - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ From c3035a040611e48f0744da924854563b57aa1cac Mon Sep 17 00:00:00 2001 From: TC Date: Tue, 30 Apr 2024 00:02:59 +0000 Subject: [PATCH 11/12] Uncommented crates.io section. --- .gitlab-ci.yml | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ff8fc6bf..60919f5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -237,23 +237,23 @@ dryrun_package_amd64_rpm: rules: - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ -# dryrun_publish_crates: -# stage: build_packages -# needs: -# - dryrun_create_build_machines -# tags: -# - build-amd64-deb -# script: -# - vlt login -# - vlt run --command="cargo publish -p veilid-tools --dry-run" -# - vlt run --command="cargo publish -p veilid-core --dry-run" -# rules: -# - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL -# changes: -# - .gitlab-ci.yml -# - scripts/cicd/**/* -# - Earthfile -# - package/**/* + dryrun_publish_crates: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-amd64-deb + script: + - vlt login + - vlt run --command="cargo publish -p veilid-tools --dry-run" + - vlt run --command="cargo publish -p veilid-core --dry-run" + rules: + - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL + changes: + - .gitlab-ci.yml + - scripts/cicd/**/* + - Earthfile + - package/**/* dryrun_publish_python: stage: build_packages @@ -301,5 +301,3 @@ dryrun_delete_build_machines: - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm rules: - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ - -# Added to test commit message dry run trigger \ No newline at end of file From 6a7afe9017ccf39d7844edc54cda00f368f32673 Mon Sep 17 00:00:00 2001 From: TC Date: Tue, 30 Apr 2024 00:04:30 +0000 Subject: [PATCH 12/12] Uncommented crates.io section. --- .gitlab-ci.yml | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 60919f5b..bd016d37 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -237,23 +237,18 @@ dryrun_package_amd64_rpm: rules: - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ - dryrun_publish_crates: - stage: build_packages - needs: - - dryrun_create_build_machines - tags: - - build-amd64-deb - script: - - vlt login - - vlt run --command="cargo publish -p veilid-tools --dry-run" - - vlt run --command="cargo publish -p veilid-core --dry-run" - rules: - - if: ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event") && $CI_COMMIT_TAG == NULL - changes: - - .gitlab-ci.yml - - scripts/cicd/**/* - - Earthfile - - package/**/* +dryrun_publish_crates: + stage: build_packages + needs: + - dryrun_create_build_machines + tags: + - build-amd64-deb + script: + - vlt login + - vlt run --command="cargo publish -p veilid-tools --dry-run" + - vlt run --command="cargo publish -p veilid-core --dry-run" + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/ dryrun_publish_python: stage: build_packages