From e9c6e44befe8587384a30afc384b639a5f803040 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 25 Sep 2022 18:18:48 -0600 Subject: [PATCH 01/48] add earthly build to cicd --- .gitlab-ci.yml | 94 +++++++------------------------------------------- 1 file changed, 13 insertions(+), 81 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1aef5266..35db32ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,82 +1,14 @@ -variables: - GIT_SUBMODULE_STRATEGY: recursive - BUILD_IMAGE_LINUX_AMD64: $CI_REGISTRY/veilid/ci-cd/veilid-build-linux-amd64:latest - -stages: - - clippy - - test - - build - -############# Clippy Lint - -clippy: - stage: clippy - image: ${BUILD_IMAGE_LINUX_AMD64} - cache: - key: $CI_COMMIT_REF_SLUG-linux-amd64 - paths: - - target/ - tags: - - linux - - amd64 +image: docker +services: + - docker:dind +before_script: + - apk update && apk add git + - wget https://github.com/earthly/earthly/releases/download/v0.6.23/earthly-linux-amd64 -O /usr/local/bin/earthly + - chmod +x /usr/local/bin/earthly + - export FORCE_COLOR=1 + - /usr/local/bin/earthly bootstrap + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY +earthly: + stage: package-linux script: - - cargo clippy - # Only run clippy on non-protected branches, for development - rules: - - if: $CI_COMMIT_TAG - when: never - - if: $CI_MERGE_REQUEST_IID - when: never - - if: $CI_COMMIT_REF_PROTECTED == "false" - -############# Unit Testing - -.test_rules_common: - # Only do tests for tags, protected branches, or merge requests - rules: - - if: $CI_COMMIT_TAG - - if: $CI_MERGE_REQUEST_IID - - if: $CI_COMMIT_REF_PROTECTED == "true" - -test_linux_amd64: - stage: test - image: ${BUILD_IMAGE_LINUX_AMD64} - cache: - key: $CI_COMMIT_REF_SLUG-linux-amd64 - paths: - - target/ - tags: - - linux - - amd64 - script: - - RUST_BACKTRACE=1 dbus-run-session -- cargo test -- --nocapture - rules: - - !reference [.test_rules_common, rules] - -############# Build - -.build_rules_common: - # Only build for tags or protected branches - rules: - - if: $CI_COMMIT_TAG - - if: $CI_COMMIT_REF_PROTECTED == "true" - -build_linux_amd64: - stage: build - image: ${BUILD_IMAGE_LINUX_AMD64} - cache: - key: $CI_COMMIT_REF_SLUG-linux-amd64 - paths: - - target/ - tags: - - linux - - amd64 - script: - - cargo build --release - artifacts: - name: $CI_COMMIT_REF_SLUG-linux-amd64 - paths: - - target/release/veilid-cli - - target/release/veilid-server - rules: - - !reference [.build_rules_common, rules] + - earthly --ci --push -P +build From 725a60596c634d924ec776cc5dd91e845ba8c883 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 25 Sep 2022 18:20:49 -0600 Subject: [PATCH 02/48] correct build target --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 35db32ca..2f45b748 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,6 @@ before_script: - /usr/local/bin/earthly bootstrap - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY earthly: - stage: package-linux + stage: build script: - - earthly --ci --push -P +build + - earthly --ci --push -P +package-linux From edc6b85046103fe1d885dc0e663f4071414ebbde Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 25 Sep 2022 20:18:21 -0600 Subject: [PATCH 03/48] Create terraform for gitlab runner --- cicd/.gitignore | 2 ++ cicd/provider.tf | 20 ++++++++++++++++++++ cicd/runner.tf | 29 +++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 cicd/.gitignore create mode 100644 cicd/provider.tf create mode 100644 cicd/runner.tf diff --git a/cicd/.gitignore b/cicd/.gitignore new file mode 100644 index 00000000..5dfe3103 --- /dev/null +++ b/cicd/.gitignore @@ -0,0 +1,2 @@ +.terraform* +terraform.tfstate* diff --git a/cicd/provider.tf b/cicd/provider.tf new file mode 100644 index 00000000..7bbfa001 --- /dev/null +++ b/cicd/provider.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + } +} + +variable "do_token" {} +variable "ssh_key" {} +variable "pvt_key" {} + +provider "digitalocean" { + token = var.do_token +} + +data "digitalocean_ssh_key" "ssh_key" { + name = var.ssh_key +} diff --git a/cicd/runner.tf b/cicd/runner.tf new file mode 100644 index 00000000..491a7bd6 --- /dev/null +++ b/cicd/runner.tf @@ -0,0 +1,29 @@ +resource "digitalocean_droplet" "veilid-runner-1" { + image = "debian-11-x64" + name = "veilid-runner-1" + region = "nyc1" + size = "s-1vcpu-512mb-10gb" + ssh_keys = [ + data.digitalocean_ssh_key.ssh_key.id + ] + + connection { + host = self.ipv4_address + user = "root" + type = "ssh" + private_key = file(var.pvt_key) + timeout = "2m" + } + + provisioner "remote-exec" { + inline = [ + "apt-get update", + "apt-get -y install ca-certificates curl gnupg lsb-release", + "mkdir -p /etc/apt/keyrings/", + "curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg", + "echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable\" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null", + "apt-get update", + "apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin" + ] + } +} From 97c664a20081d1969c3f8271d6e0b2f2b2e54b29 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 25 Sep 2022 20:35:10 -0600 Subject: [PATCH 04/48] remove errant file, add README --- .gitlab-ci.yml | 94 +++++++++++++++++++++++++++++++++++++++++++------- cicd/README.md | 40 +++++++++++++++++++++ 2 files changed, 121 insertions(+), 13 deletions(-) create mode 100644 cicd/README.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f45b748..1aef5266 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,82 @@ -image: docker -services: - - docker:dind -before_script: - - apk update && apk add git - - wget https://github.com/earthly/earthly/releases/download/v0.6.23/earthly-linux-amd64 -O /usr/local/bin/earthly - - chmod +x /usr/local/bin/earthly - - export FORCE_COLOR=1 - - /usr/local/bin/earthly bootstrap - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY -earthly: - stage: build +variables: + GIT_SUBMODULE_STRATEGY: recursive + BUILD_IMAGE_LINUX_AMD64: $CI_REGISTRY/veilid/ci-cd/veilid-build-linux-amd64:latest + +stages: + - clippy + - test + - build + +############# Clippy Lint + +clippy: + stage: clippy + image: ${BUILD_IMAGE_LINUX_AMD64} + cache: + key: $CI_COMMIT_REF_SLUG-linux-amd64 + paths: + - target/ + tags: + - linux + - amd64 script: - - earthly --ci --push -P +package-linux + - cargo clippy + # Only run clippy on non-protected branches, for development + rules: + - if: $CI_COMMIT_TAG + when: never + - if: $CI_MERGE_REQUEST_IID + when: never + - if: $CI_COMMIT_REF_PROTECTED == "false" + +############# Unit Testing + +.test_rules_common: + # Only do tests for tags, protected branches, or merge requests + rules: + - if: $CI_COMMIT_TAG + - if: $CI_MERGE_REQUEST_IID + - if: $CI_COMMIT_REF_PROTECTED == "true" + +test_linux_amd64: + stage: test + image: ${BUILD_IMAGE_LINUX_AMD64} + cache: + key: $CI_COMMIT_REF_SLUG-linux-amd64 + paths: + - target/ + tags: + - linux + - amd64 + script: + - RUST_BACKTRACE=1 dbus-run-session -- cargo test -- --nocapture + rules: + - !reference [.test_rules_common, rules] + +############# Build + +.build_rules_common: + # Only build for tags or protected branches + rules: + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_REF_PROTECTED == "true" + +build_linux_amd64: + stage: build + image: ${BUILD_IMAGE_LINUX_AMD64} + cache: + key: $CI_COMMIT_REF_SLUG-linux-amd64 + paths: + - target/ + tags: + - linux + - amd64 + script: + - cargo build --release + artifacts: + name: $CI_COMMIT_REF_SLUG-linux-amd64 + paths: + - target/release/veilid-cli + - target/release/veilid-server + rules: + - !reference [.build_rules_common, rules] diff --git a/cicd/README.md b/cicd/README.md new file mode 100644 index 00000000..11fbf336 --- /dev/null +++ b/cicd/README.md @@ -0,0 +1,40 @@ +# Terraform for Gitlab Runner + +After having had trouble with my Gitlab Runner, I decided to put together a plan +for creating runners more automatically, thus this Terraform configuration. + +This plan assumes running a Gitlab Runner, Docker Executor on a DigitalOcean +droplet. Running this plan requires an active DigitalOcean account, a configured +SSH key that will be installed on any created droplet, and a DigitalOcean +personal access token (PAT). + +## Creating the runner + +Before creating the runner, we run a `plan` to ensure we are creating the +droplet that we expect. First, we will export our access token as an environment +variable: + +```shell +export DO_PAT="$(cat .config/doctl/config.yaml | yq e '.access-token' -)" +``` + +Then we can run our plan: + +```shell +terraform plan \ + -var "do_token=${DO_PAT}" \ + -var "pvt_key=$HOME/.ssh/id_rsa" +``` + +If the output is what was expected, we may now create the droplet: + +```shell +terraform apply \ + -var "do_token=${DO_PAT}" \ + -var "pvt_key=$HOME/.ssh/id_rsa" +``` + +**TODO** + +Update the configuration to accept the runner registration token as a variable +and automatically self-register. From d4d3186521c57e6adecb98a7cd03b90c7619023c Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 9 Oct 2022 23:55:28 -0600 Subject: [PATCH 05/48] Update to working config --- cicd/README.md | 17 ++++++++++++++--- cicd/docker-install.yml | 20 ++++++++++++++++++++ cicd/docker-sources.sh | 17 +++++++++++++++++ cicd/runner.tf | 15 +++++++++------ 4 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 cicd/docker-install.yml create mode 100755 cicd/docker-sources.sh diff --git a/cicd/README.md b/cicd/README.md index 11fbf336..cfef9216 100644 --- a/cicd/README.md +++ b/cicd/README.md @@ -15,7 +15,7 @@ droplet that we expect. First, we will export our access token as an environment variable: ```shell -export DO_PAT="$(cat .config/doctl/config.yaml | yq e '.access-token' -)" +export DO_PAT="$(cat ~/.config/doctl/config.yaml | yq e '.access-token' -)" ``` Then we can run our plan: @@ -23,7 +23,8 @@ Then we can run our plan: ```shell terraform plan \ -var "do_token=${DO_PAT}" \ - -var "pvt_key=$HOME/.ssh/id_rsa" + -var "pvt_key=$HOME/.ssh/id_rsa" \ + -var "ssh_key=$KEYNAME" ``` If the output is what was expected, we may now create the droplet: @@ -31,7 +32,17 @@ If the output is what was expected, we may now create the droplet: ```shell terraform apply \ -var "do_token=${DO_PAT}" \ - -var "pvt_key=$HOME/.ssh/id_rsa" + -var "pvt_key=$HOME/.ssh/id_rsa" \ + -var "ssh_key=$KEYNAME" +``` + +## Destroying the runner + +```shell +terraform destroy \ + -var "do_token=${DO_PAT}" \ + -var "pvt_key=$HOME/.ssh/id_rsa" \ + -var "ssh_key=$KEYNAME" ``` **TODO** diff --git a/cicd/docker-install.yml b/cicd/docker-install.yml new file mode 100644 index 00000000..7966face --- /dev/null +++ b/cicd/docker-install.yml @@ -0,0 +1,20 @@ +- name: Prepare Docker Executor + become: yes + hosts: all + tasks: + - name: Install Dependencies + ansible.builtin.apt: + pkg: + - ca-certificates + - curl + - gnupg + - lsb-release + - name: Install Docker Sources + ansible.builtin.script: ./docker-sources.sh + - name: Install Docker Packages + ansible.builtin.apt: + pkg: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin diff --git a/cicd/docker-sources.sh b/cicd/docker-sources.sh new file mode 100755 index 00000000..0e2b88d0 --- /dev/null +++ b/cicd/docker-sources.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -e + +KEYRING=/etc/apt/keyrings/docker.gpg + +# Download Docker source keyring +mkdir -p /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/debian/gpg \ + | gpg --dearmor -o ${KEYRING} + +# Set Docker apt source +echo "deb [arch=$(dpkg --print-architecture) signed-by=${KEYRING}] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \ + | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +# Update sources +apt-get update diff --git a/cicd/runner.tf b/cicd/runner.tf index 491a7bd6..6267dd19 100644 --- a/cicd/runner.tf +++ b/cicd/runner.tf @@ -18,12 +18,15 @@ resource "digitalocean_droplet" "veilid-runner-1" { provisioner "remote-exec" { inline = [ "apt-get update", - "apt-get -y install ca-certificates curl gnupg lsb-release", - "mkdir -p /etc/apt/keyrings/", - "curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg", - "echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable\" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null", - "apt-get update", - "apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin" + "apt-get install python3-apt -y" ] } + + provisioner "local-exec" { + command = "ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -u root -i '${self.ipv4_address},' --private-key ${var.pvt_key} docker-install.yml" + } +} + +output "droplet_ip_address" { + value = digitalocean_droplet.veilid-runner-1 } From 3bf2e0cd5be943e4ff95d97bef43906cf04076bd Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Mon, 17 Oct 2022 20:35:49 -0600 Subject: [PATCH 06/48] WIP, expanding runner construction --- cicd/docker-install.yml | 13 ++++++++++--- cicd/earthly-setup.sh | 6 ++++++ cicd/gitlab-runner.sh | 25 +++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100755 cicd/earthly-setup.sh create mode 100755 cicd/gitlab-runner.sh diff --git a/cicd/docker-install.yml b/cicd/docker-install.yml index 7966face..b97a8926 100644 --- a/cicd/docker-install.yml +++ b/cicd/docker-install.yml @@ -2,19 +2,26 @@ become: yes hosts: all tasks: - - name: Install Dependencies + - name: install-dependencies ansible.builtin.apt: pkg: - ca-certificates - curl - gnupg - lsb-release - - name: Install Docker Sources + - git + - name: install-docker-sources ansible.builtin.script: ./docker-sources.sh - - name: Install Docker Packages + - name: install-docker-packages ansible.builtin.apt: pkg: - docker-ce - docker-ce-cli - containerd.io - docker-compose-plugin + - name: install-earthly + ansible.builtin.script: ./earthly-setup.sh + - name: install-gitlab-runner + ansible.builtin.script: ./gitlab-runner.sh install + - name: register-gitlab-runner + ansible.buildin.script: ./gitlab-runner.sh register diff --git a/cicd/earthly-setup.sh b/cicd/earthly-setup.sh new file mode 100755 index 00000000..5b486cab --- /dev/null +++ b/cicd/earthly-setup.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +wget https://github.com/earthly/earthly/releases/download/v0.6.27/earthly-linux-amd64 \ + -O /usr/local/bin/earthly +chmod +x /usr/local/bin/earthly +/usr/local/bin/earthly bootstrap diff --git a/cicd/gitlab-runner.sh b/cicd/gitlab-runner.sh new file mode 100755 index 00000000..2183a679 --- /dev/null +++ b/cicd/gitlab-runner.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + + +install () { + docker run -d --name gitlab-runner --restart always \ + -v /srv/gitlab-runner/config:/etc/gitlab-runner \ + -v /var/run/docker.sock:/var/run/docker.sock \ + gitlab/gitlab-runner:latest +} + +register () { + docker run --rm -it \ + -v /srv/gitlab-runner/config:/etc/gitlab-runner gitlab/gitlab-runner register +} + +case $1 in + install) + install + ;; + + register) + register + ;; + +esac From 6aca07d927afb4601956ee1679cbd5bb0c7a67cf Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sat, 29 Oct 2022 16:17:52 -0600 Subject: [PATCH 07/48] Self-registering runner! --- cicd/Makefile | 33 +++++++++++++++++++++++++++++++++ cicd/README.md | 19 ++++++++----------- cicd/docker-install.yml | 6 +++++- cicd/gitlab-runner.sh | 9 ++++++++- cicd/provider.tf | 3 +++ cicd/runner.tf | 8 +++++++- cicd/secrets.yaml | 28 ++++++++++++++++++++++++++++ 7 files changed, 92 insertions(+), 14 deletions(-) create mode 100644 cicd/Makefile create mode 100644 cicd/secrets.yaml diff --git a/cicd/Makefile b/cicd/Makefile new file mode 100644 index 00000000..79acd75f --- /dev/null +++ b/cicd/Makefile @@ -0,0 +1,33 @@ +DO_PAT := $(shell cat ~/.config/doctl/config.yaml | yq e '.access-token' -) +GITLAB_REG_KEY := $(shell sops -d secrets.yaml | yq e '.gitlab-reg-key' -) +GITLAB_SERVER_URL := $(shell sops -d secrets.yaml | yq e '.gitlab-server-url' -) +RUNNER_NAME := "veilid-runner-1" +KEYNAME := "pensfabriko" + + +plan-runner: + terraform plan \ + -var "do_token=${DO_PAT}" \ + -var "pvt_key=${HOME}/.ssh/id_rsa" \ + -var "ssh_key=${KEYNAME}" \ + -var "reg_key=${GITLAB_REG_KEY}" \ + -var "ci_server_url=${GITLAB_SERVER_URL}" \ + -var "runner_name=${RUNNER_NAME}" + +create-runner: + terraform apply \ + -var "do_token=${DO_PAT}" \ + -var "pvt_key=${HOME}/.ssh/id_rsa" \ + -var "ssh_key=${KEYNAME}" \ + -var "reg_key=${GITLAB_REG_KEY}" \ + -var "ci_server_url=${GITLAB_SERVER_URL}" \ + -var "runner_name=${RUNNER_NAME}" + +destroy-runner: + terraform destroy \ + -var "do_token=${DO_PAT}" \ + -var "pvt_key=${HOME}/.ssh/id_rsa" \ + -var "ssh_key=${KEYNAME}" \ + -var "reg_key=${GITLAB_REG_KEY}" \ + -var "ci_server_url=${GITLAB_SERVER_URL}" \ + -var "runner_name=${RUNNER_NAME}" diff --git a/cicd/README.md b/cicd/README.md index cfef9216..0dd46980 100644 --- a/cicd/README.md +++ b/cicd/README.md @@ -23,8 +23,9 @@ Then we can run our plan: ```shell terraform plan \ -var "do_token=${DO_PAT}" \ - -var "pvt_key=$HOME/.ssh/id_rsa" \ - -var "ssh_key=$KEYNAME" + -var "pvt_key=${HOME}/.ssh/id_rsa" \ + -var "ssh_key=${KEYNAME}" \ + -var "reg_key=${GITLAB_REG_KEY}" ``` If the output is what was expected, we may now create the droplet: @@ -32,8 +33,9 @@ If the output is what was expected, we may now create the droplet: ```shell terraform apply \ -var "do_token=${DO_PAT}" \ - -var "pvt_key=$HOME/.ssh/id_rsa" \ - -var "ssh_key=$KEYNAME" + -var "pvt_key=${HOME}/.ssh/id_rsa" \ + -var "ssh_key=${KEYNAME}" + -var "reg_key=${GITLAB_REG_KEY}" ``` ## Destroying the runner @@ -41,11 +43,6 @@ terraform apply \ ```shell terraform destroy \ -var "do_token=${DO_PAT}" \ - -var "pvt_key=$HOME/.ssh/id_rsa" \ - -var "ssh_key=$KEYNAME" + -var "pvt_key=${HOME}/.ssh/id_rsa" \ + -var "ssh_key=${KEYNAME}" ``` - -**TODO** - -Update the configuration to accept the runner registration token as a variable -and automatically self-register. diff --git a/cicd/docker-install.yml b/cicd/docker-install.yml index b97a8926..cfe410b1 100644 --- a/cicd/docker-install.yml +++ b/cicd/docker-install.yml @@ -24,4 +24,8 @@ - name: install-gitlab-runner ansible.builtin.script: ./gitlab-runner.sh install - name: register-gitlab-runner - ansible.buildin.script: ./gitlab-runner.sh register + ansible.builtin.script: ./gitlab-runner.sh register + environment: + CI_SERVER_URL: "{{ ci_server_url }}" + REGISTRATION_TOKEN: "{{ regkey }}" + RUNNER_NAME: "{{ runner_name }}" diff --git a/cicd/gitlab-runner.sh b/cicd/gitlab-runner.sh index 2183a679..d2be1d55 100755 --- a/cicd/gitlab-runner.sh +++ b/cicd/gitlab-runner.sh @@ -10,7 +10,14 @@ install () { register () { docker run --rm -it \ - -v /srv/gitlab-runner/config:/etc/gitlab-runner gitlab/gitlab-runner register + -v /srv/gitlab-runner/config:/etc/gitlab-runner gitlab/gitlab-runner register \ + --non-interactive \ + --executor "docker" \ + --docker-image alpine:latest \ + --url "${CI_SERVER_URL}" \ + --registration-token "${REGISTRATION_TOKEN}" \ + --description "${RUNNER_NAME}" \ + --tag-list "amd64,linux" } case $1 in diff --git a/cicd/provider.tf b/cicd/provider.tf index 7bbfa001..f3b86427 100644 --- a/cicd/provider.tf +++ b/cicd/provider.tf @@ -10,6 +10,9 @@ terraform { variable "do_token" {} variable "ssh_key" {} variable "pvt_key" {} +variable "reg_key" {} +variable "ci_server_url" {} +variable "runner_name" {} provider "digitalocean" { token = var.do_token diff --git a/cicd/runner.tf b/cicd/runner.tf index 6267dd19..ab21b9f2 100644 --- a/cicd/runner.tf +++ b/cicd/runner.tf @@ -23,7 +23,13 @@ resource "digitalocean_droplet" "veilid-runner-1" { } provisioner "local-exec" { - command = "ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -u root -i '${self.ipv4_address},' --private-key ${var.pvt_key} docker-install.yml" + command = < Date: Sat, 29 Oct 2022 16:28:50 -0600 Subject: [PATCH 08/48] Push new ci config --- .gitlab-ci.yml | 85 +++++--------------------------------------------- 1 file changed, 7 insertions(+), 78 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1aef5266..a0489393 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,82 +1,11 @@ +services: + - docker:dind + variables: - GIT_SUBMODULE_STRATEGY: recursive - BUILD_IMAGE_LINUX_AMD64: $CI_REGISTRY/veilid/ci-cd/veilid-build-linux-amd64:latest + DOCKER_HOST: tcp://docker:2375 + EARTHLY_EXEC_CMD: "/bin/sh" -stages: - - clippy - - test - - build - -############# Clippy Lint - -clippy: - stage: clippy - image: ${BUILD_IMAGE_LINUX_AMD64} - cache: - key: $CI_COMMIT_REF_SLUG-linux-amd64 - paths: - - target/ - tags: - - linux - - amd64 - script: - - cargo clippy - # Only run clippy on non-protected branches, for development - rules: - - if: $CI_COMMIT_TAG - when: never - - if: $CI_MERGE_REQUEST_IID - when: never - - if: $CI_COMMIT_REF_PROTECTED == "false" - -############# Unit Testing - -.test_rules_common: - # Only do tests for tags, protected branches, or merge requests - rules: - - if: $CI_COMMIT_TAG - - if: $CI_MERGE_REQUEST_IID - - if: $CI_COMMIT_REF_PROTECTED == "true" - -test_linux_amd64: - stage: test - image: ${BUILD_IMAGE_LINUX_AMD64} - cache: - key: $CI_COMMIT_REF_SLUG-linux-amd64 - paths: - - target/ - tags: - - linux - - amd64 - script: - - RUST_BACKTRACE=1 dbus-run-session -- cargo test -- --nocapture - rules: - - !reference [.test_rules_common, rules] - -############# Build - -.build_rules_common: - # Only build for tags or protected branches - rules: - - if: $CI_COMMIT_TAG - - if: $CI_COMMIT_REF_PROTECTED == "true" - -build_linux_amd64: +earthly: stage: build - image: ${BUILD_IMAGE_LINUX_AMD64} - cache: - key: $CI_COMMIT_REF_SLUG-linux-amd64 - paths: - - target/ - tags: - - linux - - amd64 script: - - cargo build --release - artifacts: - name: $CI_COMMIT_REF_SLUG-linux-amd64 - paths: - - target/release/veilid-cli - - target/release/veilid-server - rules: - - !reference [.build_rules_common, rules] + - earthly --ci --push -P +package-linux From 8686719a564ab3d7be5ffe59cf37e55a9e0f6f76 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sat, 29 Oct 2022 16:35:19 -0600 Subject: [PATCH 09/48] add tags --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0489393..0806f00b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,5 +7,8 @@ variables: earthly: stage: build + tags: + - linux + - amd64 script: - earthly --ci --push -P +package-linux From 7a544f67a04c13015542781cd2b27db84d8c3338 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sat, 29 Oct 2022 16:49:55 -0600 Subject: [PATCH 10/48] force earthly image --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0806f00b..0f157dd1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,11 @@ variables: DOCKER_HOST: tcp://docker:2375 EARTHLY_EXEC_CMD: "/bin/sh" +image: earthly/earthly:v0.6.27 + +before_script: + - earthly bootstrap + earthly: stage: build tags: From 7bff43e414cec9e4e995042d8511a7a9b35151b9 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sat, 29 Oct 2022 16:56:58 -0600 Subject: [PATCH 11/48] add buildkit-host --- .gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f157dd1..2931b6da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,13 +7,10 @@ variables: image: earthly/earthly:v0.6.27 -before_script: - - earthly bootstrap - earthly: stage: build tags: - linux - amd64 script: - - earthly --ci --push -P +package-linux + - earthly --ci --push --buildkit-host localhost:8373 -P +package-linux From 1f0b9fc8d6b28ae8aff6e5c3314ecb636fd1c5ac Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sat, 29 Oct 2022 17:03:02 -0600 Subject: [PATCH 12/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2931b6da..00c6afbe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,4 +13,4 @@ earthly: - linux - amd64 script: - - earthly --ci --push --buildkit-host localhost:8373 -P +package-linux + - earthly --ci --buildkit-host localhost:8373 -P +package-linux From 2c030095214555c27934f115de5c73e39f2f6769 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sat, 29 Oct 2022 17:17:41 -0600 Subject: [PATCH 13/48] update earthly version --- .gitlab-ci.yml | 2 +- cicd/earthly-setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00c6afbe..add8621e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ variables: DOCKER_HOST: tcp://docker:2375 EARTHLY_EXEC_CMD: "/bin/sh" -image: earthly/earthly:v0.6.27 +image: earthly/earthly:v0.6.28 earthly: stage: build diff --git a/cicd/earthly-setup.sh b/cicd/earthly-setup.sh index 5b486cab..de6e313c 100755 --- a/cicd/earthly-setup.sh +++ b/cicd/earthly-setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -wget https://github.com/earthly/earthly/releases/download/v0.6.27/earthly-linux-amd64 \ +wget https://github.com/earthly/earthly/releases/download/v0.6.28/earthly-linux-amd64 \ -O /usr/local/bin/earthly chmod +x /usr/local/bin/earthly /usr/local/bin/earthly bootstrap From df1a4b81eb4f13927782cbc876e9c32595274655 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 30 Oct 2022 16:49:51 -0600 Subject: [PATCH 14/48] up the privs? --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index add8621e..414f290f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,4 +13,4 @@ earthly: - linux - amd64 script: - - earthly --ci --buildkit-host localhost:8373 -P +package-linux + - earthly --ci --buildkit-host localhost:8373 --privileged +package-linux From 70a79dddf1ef13f6372897fff2823fdd0cea6036 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 30 Oct 2022 17:00:33 -0600 Subject: [PATCH 15/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 414f290f..ea06e53d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,4 +13,4 @@ earthly: - linux - amd64 script: - - earthly --ci --buildkit-host localhost:8373 --privileged +package-linux + - earthly --ci --buildkit-host localhost:8373 --allow-privileged +package-linux From 99749b8b3e2a67a5b7e81fbf495f43168a44ff7d Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 30 Oct 2022 17:28:46 -0600 Subject: [PATCH 16/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ea06e53d..e13925f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,4 +13,4 @@ earthly: - linux - amd64 script: - - earthly --ci --buildkit-host localhost:8373 --allow-privileged +package-linux + - earthly --ci --buildkit-host tcp://127.0.0.1:8373 --allow-privileged +package-linux From 43abe7af58daec4b0accaadf8e7e3ba729d58a12 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Tue, 1 Nov 2022 16:18:30 -0600 Subject: [PATCH 17/48] remove services --- .gitlab-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e13925f6..aab5fc1a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,3 @@ -services: - - docker:dind - variables: DOCKER_HOST: tcp://docker:2375 EARTHLY_EXEC_CMD: "/bin/sh" From ec49bc9b25233f09263bef2c2da9183bf72c2aaa Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Tue, 1 Nov 2022 16:57:36 -0600 Subject: [PATCH 18/48] Start own earthly container --- cicd/{docker-install.yml => docker-install.yaml} | 0 cicd/earthly-setup.sh | 11 +++++++---- cicd/runner.tf | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) rename cicd/{docker-install.yml => docker-install.yaml} (100%) diff --git a/cicd/docker-install.yml b/cicd/docker-install.yaml similarity index 100% rename from cicd/docker-install.yml rename to cicd/docker-install.yaml diff --git a/cicd/earthly-setup.sh b/cicd/earthly-setup.sh index de6e313c..0b8321ce 100755 --- a/cicd/earthly-setup.sh +++ b/cicd/earthly-setup.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash -wget https://github.com/earthly/earthly/releases/download/v0.6.28/earthly-linux-amd64 \ - -O /usr/local/bin/earthly -chmod +x /usr/local/bin/earthly -/usr/local/bin/earthly bootstrap +docker run -d --restart always \ + --privileged \ + --name earthly-buildkit \ + -p 8372:8372 \ + -t -v earthly-tmp:/tmp/earthly:rw \ + --env BUILDKIT_TCP_TRANSPORT_ENABLED=true \ + earthly/buildkitd:v0.6.28 diff --git a/cicd/runner.tf b/cicd/runner.tf index ab21b9f2..239c07f5 100644 --- a/cicd/runner.tf +++ b/cicd/runner.tf @@ -28,7 +28,7 @@ ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -u root \ -i '${self.ipv4_address},' \ --private-key ${var.pvt_key} \ -e "regkey=${var.reg_key} ci_server_url=${var.ci_server_url} runner_name=${var.runner_name}" \ - docker-install.yml + docker-install.yaml EOF } } From bb5314ff5e611a24d2d086a58ddf42b5d2d8a953 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Tue, 1 Nov 2022 17:00:15 -0600 Subject: [PATCH 19/48] wip --- .gitlab-ci.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aab5fc1a..664ab365 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,8 @@ -variables: - DOCKER_HOST: tcp://docker:2375 - EARTHLY_EXEC_CMD: "/bin/sh" - -image: earthly/earthly:v0.6.28 - earthly: stage: build + image: earthly/earthly:v0.6.28 tags: - linux - amd64 script: - - earthly --ci --buildkit-host tcp://127.0.0.1:8373 --allow-privileged +package-linux + - earthly --ci --buildkit_host tcp://127.0.0.1:8373 --allow-privileged +package-linux From 1636e15b5686133cd538b072e6c2b90e4439d1a8 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sat, 5 Nov 2022 11:51:18 -0600 Subject: [PATCH 20/48] Use template config to allow privileged containers --- cicd/docker-install.yaml | 4 ++++ cicd/gitlab-runner.sh | 1 + cicd/template.config.toml | 6 ++++++ 3 files changed, 11 insertions(+) create mode 100644 cicd/template.config.toml diff --git a/cicd/docker-install.yaml b/cicd/docker-install.yaml index cfe410b1..cf683c14 100644 --- a/cicd/docker-install.yaml +++ b/cicd/docker-install.yaml @@ -21,6 +21,10 @@ - docker-compose-plugin - name: install-earthly ansible.builtin.script: ./earthly-setup.sh + - name: install-gitlab-runner-config + ansible.builtin.copy: + src: template.config.toml + dest: /srv/gitlab-runner/config/ - name: install-gitlab-runner ansible.builtin.script: ./gitlab-runner.sh install - name: register-gitlab-runner diff --git a/cicd/gitlab-runner.sh b/cicd/gitlab-runner.sh index d2be1d55..2b0674c5 100755 --- a/cicd/gitlab-runner.sh +++ b/cicd/gitlab-runner.sh @@ -17,6 +17,7 @@ register () { --url "${CI_SERVER_URL}" \ --registration-token "${REGISTRATION_TOKEN}" \ --description "${RUNNER_NAME}" \ + --template-config /srv/gitlab-runner/config/template.config.toml \ --tag-list "amd64,linux" } diff --git a/cicd/template.config.toml b/cicd/template.config.toml new file mode 100644 index 00000000..d33f8247 --- /dev/null +++ b/cicd/template.config.toml @@ -0,0 +1,6 @@ +[[runners]] + executor = "docker" + [runners.docker] + tls_verify = false + image = "alpine:latest" + privileged = true From 243d343bc609ba5e287783af52c0edda97d2c3c6 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 14:19:59 -0700 Subject: [PATCH 21/48] Correctly configures buildkit, set new remote host --- cicd/config/config.toml | 21 +++++++++++++++++++++ cicd/config/template.config.toml | 3 +++ cicd/docker-install.yaml | 10 +++++++--- cicd/earthly-setup.sh | 5 +++-- cicd/gitlab-runner.sh | 8 ++++++-- cicd/template.config.toml | 6 ------ 6 files changed, 40 insertions(+), 13 deletions(-) create mode 100644 cicd/config/config.toml create mode 100644 cicd/config/template.config.toml delete mode 100644 cicd/template.config.toml diff --git a/cicd/config/config.toml b/cicd/config/config.toml new file mode 100644 index 00000000..6c2ab68c --- /dev/null +++ b/cicd/config/config.toml @@ -0,0 +1,21 @@ +concurrent = 1 +check_interval = 0 + +[session_server] + session_timeout = 1800 + +[[runners]] + [runners.custom_build_dir] + [runners.cache] + [runners.cache.s3] + [runners.cache.gcs] + [runners.cache.azure] + [runners.docker] + privileged = true + tls_verify = false + disable_entrypoint_overwrite = false + oom_kill_disable = false + disable_cache = false + volumes = ["/cache"] + shm_size = 0 + diff --git a/cicd/config/template.config.toml b/cicd/config/template.config.toml new file mode 100644 index 00000000..f973b917 --- /dev/null +++ b/cicd/config/template.config.toml @@ -0,0 +1,3 @@ +[[runners]] + [runners.docker] + privileged = true diff --git a/cicd/docker-install.yaml b/cicd/docker-install.yaml index cf683c14..52fd9cbe 100644 --- a/cicd/docker-install.yaml +++ b/cicd/docker-install.yaml @@ -21,10 +21,14 @@ - docker-compose-plugin - name: install-earthly ansible.builtin.script: ./earthly-setup.sh - - name: install-gitlab-runner-config + - name: copy-config ansible.builtin.copy: - src: template.config.toml - dest: /srv/gitlab-runner/config/ + src: ./config/config.toml + dest: /etc/gitlab-runner/ + - name: copy-config-template + ansible.builtin.copy: + src: ./config/template.config.toml + dest: /tmp/gitlab-runner/ - name: install-gitlab-runner ansible.builtin.script: ./gitlab-runner.sh install - name: register-gitlab-runner diff --git a/cicd/earthly-setup.sh b/cicd/earthly-setup.sh index 0b8321ce..8377c2cf 100755 --- a/cicd/earthly-setup.sh +++ b/cicd/earthly-setup.sh @@ -3,7 +3,8 @@ docker run -d --restart always \ --privileged \ --name earthly-buildkit \ - -p 8372:8372 \ - -t -v earthly-tmp:/tmp/earthly:rw \ + -t -p 8372:8372 \ + -v earthly-tmp:/tmp/earthly:rw \ + -v /var/run/docker.sock:/var/run/docker.sock \ --env BUILDKIT_TCP_TRANSPORT_ENABLED=true \ earthly/buildkitd:v0.6.28 diff --git a/cicd/gitlab-runner.sh b/cicd/gitlab-runner.sh index 2b0674c5..f5cad329 100755 --- a/cicd/gitlab-runner.sh +++ b/cicd/gitlab-runner.sh @@ -9,15 +9,19 @@ install () { } register () { + docker run --rm -it \ - -v /srv/gitlab-runner/config:/etc/gitlab-runner gitlab/gitlab-runner register \ + -v /srv/gitlab-runner/config:/etc/gitlab-runner \ + -v /tmp/gitlab-runner:/tmp/gitlab-runner \ + gitlab/gitlab-runner register \ + --config /etc/gitlab-runner/config.toml \ + --template-config /tmp/gitlab-runner/template.config.toml \ --non-interactive \ --executor "docker" \ --docker-image alpine:latest \ --url "${CI_SERVER_URL}" \ --registration-token "${REGISTRATION_TOKEN}" \ --description "${RUNNER_NAME}" \ - --template-config /srv/gitlab-runner/config/template.config.toml \ --tag-list "amd64,linux" } diff --git a/cicd/template.config.toml b/cicd/template.config.toml deleted file mode 100644 index d33f8247..00000000 --- a/cicd/template.config.toml +++ /dev/null @@ -1,6 +0,0 @@ -[[runners]] - executor = "docker" - [runners.docker] - tls_verify = false - image = "alpine:latest" - privileged = true From 74cc753652eb205ee31094880ac51e35901b3d6f Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 15:29:08 -0700 Subject: [PATCH 22/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 664ab365..3e7f69c7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,4 +5,4 @@ earthly: - linux - amd64 script: - - earthly --ci --buildkit_host tcp://127.0.0.1:8373 --allow-privileged +package-linux + - earthly --ci --buildkit_host tcp://earthly-buildkit:8372 -e NO_DOCKER=1 --allow-privileged +package-linux From 4c994e24b2cd5ac0f112c8f4edaa15255280a1aa Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 15:30:54 -0700 Subject: [PATCH 23/48] wip --- .gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e7f69c7..b8feab11 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,6 @@ +variables: + NO_DOCKER: 1 + earthly: stage: build image: earthly/earthly:v0.6.28 @@ -5,4 +8,4 @@ earthly: - linux - amd64 script: - - earthly --ci --buildkit_host tcp://earthly-buildkit:8372 -e NO_DOCKER=1 --allow-privileged +package-linux + - earthly --ci --buildkit_host tcp://earthly-buildkit:8372 --allow-privileged +package-linux From 99f42bedb630fe670693e3d7251b39d5fdf18820 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 15:37:30 -0700 Subject: [PATCH 24/48] wip --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b8feab11..704461c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ variables: NO_DOCKER: 1 + BUILDKIT_HOST: tcp://earthly-buildkit:8372 earthly: stage: build @@ -8,4 +9,4 @@ earthly: - linux - amd64 script: - - earthly --ci --buildkit_host tcp://earthly-buildkit:8372 --allow-privileged +package-linux + - earthly --ci -P +package-linux From d8efbe26ad8222277a13d5ea7f53cda86e025e40 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 15:45:31 -0700 Subject: [PATCH 25/48] wip --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 704461c3..71c9c099 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ variables: NO_DOCKER: 1 BUILDKIT_HOST: tcp://earthly-buildkit:8372 + EARTHLY_EXEC_CMD: "/bin/sh" earthly: stage: build From 7fba8cf705cff62603c230c7ae9dab392959f3e1 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 15:49:12 -0700 Subject: [PATCH 26/48] wip --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71c9c099..779fc5bc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ variables: NO_DOCKER: 1 - BUILDKIT_HOST: tcp://earthly-buildkit:8372 + #BUILDKIT_HOST: tcp://earthly-buildkit:8372 + BUILDKIT_HOST: tcp://127.0.0.1:8372 EARTHLY_EXEC_CMD: "/bin/sh" earthly: From 58f514be6e13d62dbb51b7295cb3e3198bf1787f Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 16:13:54 -0700 Subject: [PATCH 27/48] wip --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 779fc5bc..689422be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,6 @@ variables: - NO_DOCKER: 1 - #BUILDKIT_HOST: tcp://earthly-buildkit:8372 - BUILDKIT_HOST: tcp://127.0.0.1:8372 + #NO_DOCKER: 1 + BUILDKIT_HOST: tcp://earthly-buildkit:8372 EARTHLY_EXEC_CMD: "/bin/sh" earthly: From f31e8c6a372738978dbd0a548bfd9acff2e47380 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 16:15:01 -0700 Subject: [PATCH 28/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 689422be..11e00421 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - #NO_DOCKER: 1 + DOCKER_HOST: /var/run/docker.sock BUILDKIT_HOST: tcp://earthly-buildkit:8372 EARTHLY_EXEC_CMD: "/bin/sh" From 35cb8d9315b48b85e077281616fe6d0c9f9d4441 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 16:59:31 -0700 Subject: [PATCH 29/48] wip --- .gitlab-ci.yml | 2 +- cicd/config/config.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11e00421..b3e84a0a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DOCKER_HOST: /var/run/docker.sock + NO_DOCKER: true BUILDKIT_HOST: tcp://earthly-buildkit:8372 EARTHLY_EXEC_CMD: "/bin/sh" diff --git a/cicd/config/config.toml b/cicd/config/config.toml index 6c2ab68c..1e2505ef 100644 --- a/cicd/config/config.toml +++ b/cicd/config/config.toml @@ -13,6 +13,7 @@ check_interval = 0 [runners.docker] privileged = true tls_verify = false + network_mode = host disable_entrypoint_overwrite = false oom_kill_disable = false disable_cache = false From 50464d5ba4056cda5bd3f007c8298e75f72d8888 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 17:01:24 -0700 Subject: [PATCH 30/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3e84a0a..71c9c099 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - NO_DOCKER: true + NO_DOCKER: 1 BUILDKIT_HOST: tcp://earthly-buildkit:8372 EARTHLY_EXEC_CMD: "/bin/sh" From 5b8684373962540d347175a8f500da0d99f9f71c Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Sun, 6 Nov 2022 21:51:06 -0700 Subject: [PATCH 31/48] connect docker sock, set network mode --- cicd/config/config.toml | 2 -- cicd/config/template.config.toml | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cicd/config/config.toml b/cicd/config/config.toml index 1e2505ef..c093b912 100644 --- a/cicd/config/config.toml +++ b/cicd/config/config.toml @@ -13,10 +13,8 @@ check_interval = 0 [runners.docker] privileged = true tls_verify = false - network_mode = host disable_entrypoint_overwrite = false oom_kill_disable = false disable_cache = false - volumes = ["/cache"] shm_size = 0 diff --git a/cicd/config/template.config.toml b/cicd/config/template.config.toml index f973b917..96e46a8d 100644 --- a/cicd/config/template.config.toml +++ b/cicd/config/template.config.toml @@ -1,3 +1,5 @@ [[runners]] [runners.docker] privileged = true + network_mode = "host" + volumes = ["/cache", "/var/run/docker.sock:/var/run/docker.sock:rw"] From 71eac66b93f63716484b8f594e2c666d96a08f1d Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Thu, 10 Nov 2022 16:34:12 -0700 Subject: [PATCH 32/48] wip, get containers on the same network --- cicd/earthly-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cicd/earthly-setup.sh b/cicd/earthly-setup.sh index 8377c2cf..71edae22 100755 --- a/cicd/earthly-setup.sh +++ b/cicd/earthly-setup.sh @@ -3,6 +3,7 @@ docker run -d --restart always \ --privileged \ --name earthly-buildkit \ + --network host \ -t -p 8372:8372 \ -v earthly-tmp:/tmp/earthly:rw \ -v /var/run/docker.sock:/var/run/docker.sock \ From 0beb4ca84222d6791edacb146a26edc445e9e3e4 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Thu, 10 Nov 2022 16:45:23 -0700 Subject: [PATCH 33/48] change listener host --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71c9c099..2ca5a341 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ variables: NO_DOCKER: 1 - BUILDKIT_HOST: tcp://earthly-buildkit:8372 + BUILDKIT_HOST: tcp://127.0.0.1:8372 EARTHLY_EXEC_CMD: "/bin/sh" earthly: From f9bc78d89c08979ed9fb76f2e6b9861f1c9b2a50 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Thu, 10 Nov 2022 16:46:48 -0700 Subject: [PATCH 34/48] change listener host --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ca5a341..5f69005e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ variables: NO_DOCKER: 1 - BUILDKIT_HOST: tcp://127.0.0.1:8372 + BUILDKIT_HOST: tcp://veilid-runner-1:8372 EARTHLY_EXEC_CMD: "/bin/sh" earthly: From 919d91b930fc887a5f17b3a588427ba131a8b0dc Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Wed, 16 Nov 2022 22:31:12 -0700 Subject: [PATCH 35/48] wip, reenable docker --- .gitlab-ci.yml | 2 +- cicd/earthly-setup.sh | 1 + cicd/gitlab-runner.sh | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5f69005e..7884a045 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - NO_DOCKER: 1 + # NO_DOCKER: 1 BUILDKIT_HOST: tcp://veilid-runner-1:8372 EARTHLY_EXEC_CMD: "/bin/sh" diff --git a/cicd/earthly-setup.sh b/cicd/earthly-setup.sh index 71edae22..e3f1be44 100755 --- a/cicd/earthly-setup.sh +++ b/cicd/earthly-setup.sh @@ -8,4 +8,5 @@ docker run -d --restart always \ -v earthly-tmp:/tmp/earthly:rw \ -v /var/run/docker.sock:/var/run/docker.sock \ --env BUILDKIT_TCP_TRANSPORT_ENABLED=true \ + --env CNI_MTU=1500 \ earthly/buildkitd:v0.6.28 diff --git a/cicd/gitlab-runner.sh b/cicd/gitlab-runner.sh index f5cad329..3099094c 100755 --- a/cicd/gitlab-runner.sh +++ b/cicd/gitlab-runner.sh @@ -5,6 +5,7 @@ install () { docker run -d --name gitlab-runner --restart always \ -v /srv/gitlab-runner/config:/etc/gitlab-runner \ -v /var/run/docker.sock:/var/run/docker.sock \ + --network="host" \ gitlab/gitlab-runner:latest } @@ -13,6 +14,7 @@ register () { docker run --rm -it \ -v /srv/gitlab-runner/config:/etc/gitlab-runner \ -v /tmp/gitlab-runner:/tmp/gitlab-runner \ + --network="host" \ gitlab/gitlab-runner register \ --config /etc/gitlab-runner/config.toml \ --template-config /tmp/gitlab-runner/template.config.toml \ From 630eb05400fca11ff27fbcbb171a525a11ec046c Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Thu, 17 Nov 2022 20:17:16 -0700 Subject: [PATCH 36/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7884a045..9cb69013 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,4 +10,4 @@ earthly: - linux - amd64 script: - - earthly --ci -P +package-linux + - earthly --ci -P --network=host +package-linux From e1b54fb6a750a8df6e84fed282baef2a0214c969 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Thu, 17 Nov 2022 20:19:23 -0700 Subject: [PATCH 37/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9cb69013..7884a045 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,4 +10,4 @@ earthly: - linux - amd64 script: - - earthly --ci -P --network=host +package-linux + - earthly --ci -P +package-linux From ec785f8b571c084ea644a55f833cd23b05ef24f2 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Thu, 17 Nov 2022 20:29:42 -0700 Subject: [PATCH 38/48] wip --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7884a045..1646d98d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ variables: - # NO_DOCKER: 1 + NO_DOCKER: 1 + CNI_MTU: 1500 BUILDKIT_HOST: tcp://veilid-runner-1:8372 EARTHLY_EXEC_CMD: "/bin/sh" From 4c107f24be2746f9364f56747d3ebe21cf8c676e Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Thu, 17 Nov 2022 20:32:52 -0700 Subject: [PATCH 39/48] wip --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1646d98d..99e48d9e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ variables: CNI_MTU: 1500 BUILDKIT_HOST: tcp://veilid-runner-1:8372 EARTHLY_EXEC_CMD: "/bin/sh" + EARTHLY_NO_CACHE: true earthly: stage: build From 53c259df27d623402ffe5b758ebcb4274791b87c Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Thu, 17 Nov 2022 20:35:56 -0700 Subject: [PATCH 40/48] wip --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99e48d9e..f6b3ee5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ variables: CNI_MTU: 1500 BUILDKIT_HOST: tcp://veilid-runner-1:8372 EARTHLY_EXEC_CMD: "/bin/sh" - EARTHLY_NO_CACHE: true earthly: stage: build @@ -12,4 +11,4 @@ earthly: - linux - amd64 script: - - earthly --ci -P +package-linux + - earthly --ci -P --no-cache +package-linux From 1aa03b08fc79f958fe87cf1f5ea35eeda63f621d Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Mon, 21 Nov 2022 19:16:37 -0700 Subject: [PATCH 41/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f6b3ee5a..4d5e5e78 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ variables: NO_DOCKER: 1 - CNI_MTU: 1500 + NETWORK_MODE: host BUILDKIT_HOST: tcp://veilid-runner-1:8372 EARTHLY_EXEC_CMD: "/bin/sh" From 4caf64edfe0ba6db030ed0bd54aa5f75c0fe4642 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Tue, 22 Nov 2022 09:20:16 -0700 Subject: [PATCH 42/48] wip --- .gitlab-ci.yml | 2 +- Earthfile | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d5e5e78..9124623d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,5 @@ variables: NO_DOCKER: 1 - NETWORK_MODE: host BUILDKIT_HOST: tcp://veilid-runner-1:8372 EARTHLY_EXEC_CMD: "/bin/sh" @@ -12,3 +11,4 @@ earthly: - amd64 script: - earthly --ci -P --no-cache +package-linux + diff --git a/Earthfile b/Earthfile index 8ce1a7bd..07a70dd7 100644 --- a/Earthfile +++ b/Earthfile @@ -6,6 +6,11 @@ FROM --platform amd64 ubuntu:16.04 # Install build prerequisites deps-base: + RUN echo "APT start" + RUN nslookup 1.1.1.1 + RUN cat /etc/apt/sources.list + RUN cat /etc/resolv.conf + RUN echo "APT stop" RUN apt-get -y update RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev @@ -130,4 +135,4 @@ package-linux-arm64: package-linux: BUILD +package-linux-amd64 - BUILD +package-linux-arm64 \ No newline at end of file + BUILD +package-linux-arm64 From 16013e1b4dfdb5a022032aed14ffc5a8c3bbc50c Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Tue, 22 Nov 2022 09:21:58 -0700 Subject: [PATCH 43/48] wip --- Earthfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Earthfile b/Earthfile index 07a70dd7..e7ddcba4 100644 --- a/Earthfile +++ b/Earthfile @@ -7,7 +7,6 @@ FROM --platform amd64 ubuntu:16.04 # Install build prerequisites deps-base: RUN echo "APT start" - RUN nslookup 1.1.1.1 RUN cat /etc/apt/sources.list RUN cat /etc/resolv.conf RUN echo "APT stop" From 624a2cb8b34dc5a7c32c54d95d71d812f5cc02d2 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Tue, 22 Nov 2022 09:22:48 -0700 Subject: [PATCH 44/48] wip --- Earthfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Earthfile b/Earthfile index e7ddcba4..d4218f26 100644 --- a/Earthfile +++ b/Earthfile @@ -7,6 +7,7 @@ FROM --platform amd64 ubuntu:16.04 # Install build prerequisites deps-base: RUN echo "APT start" + RUN ping -c 1.1.1.1 RUN cat /etc/apt/sources.list RUN cat /etc/resolv.conf RUN echo "APT stop" From 6aae099391ab571a36a4e690af71d56e7457ab01 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Tue, 22 Nov 2022 09:38:43 -0700 Subject: [PATCH 45/48] wip --- Earthfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Earthfile b/Earthfile index d4218f26..251688bd 100644 --- a/Earthfile +++ b/Earthfile @@ -6,11 +6,6 @@ FROM --platform amd64 ubuntu:16.04 # Install build prerequisites deps-base: - RUN echo "APT start" - RUN ping -c 1.1.1.1 - RUN cat /etc/apt/sources.list - RUN cat /etc/resolv.conf - RUN echo "APT stop" RUN apt-get -y update RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev From db91e0fa52c6d4b1f6897f46b1528d6a1227a228 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Tue, 22 Nov 2022 11:40:04 -0700 Subject: [PATCH 46/48] wip --- .gitlab-ci.yml | 2 +- cicd/earthly-setup.sh | 1 + cicd/gitlab-runner.sh | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9124623d..fa0ea653 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,5 +10,5 @@ earthly: - linux - amd64 script: - - earthly --ci -P --no-cache +package-linux + - earthly --ci -P --no-cache -i +package-linux diff --git a/cicd/earthly-setup.sh b/cicd/earthly-setup.sh index e3f1be44..6dd46a83 100755 --- a/cicd/earthly-setup.sh +++ b/cicd/earthly-setup.sh @@ -3,6 +3,7 @@ docker run -d --restart always \ --privileged \ --name earthly-buildkit \ + --hostname earthly-buildkit \ --network host \ -t -p 8372:8372 \ -v earthly-tmp:/tmp/earthly:rw \ diff --git a/cicd/gitlab-runner.sh b/cicd/gitlab-runner.sh index 3099094c..5b6e8929 100755 --- a/cicd/gitlab-runner.sh +++ b/cicd/gitlab-runner.sh @@ -5,6 +5,7 @@ install () { docker run -d --name gitlab-runner --restart always \ -v /srv/gitlab-runner/config:/etc/gitlab-runner \ -v /var/run/docker.sock:/var/run/docker.sock \ + --hostname="gitlab-runner" \ --network="host" \ gitlab/gitlab-runner:latest } From e21fb8985bdcccd232d36596ebaff4e52681d789 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Tue, 22 Nov 2022 11:41:34 -0700 Subject: [PATCH 47/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa0ea653..b29b9705 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,5 +10,5 @@ earthly: - linux - amd64 script: - - earthly --ci -P --no-cache -i +package-linux + - earthly -i -P --no-cache +package-linux From 474ab4c7060c9a8026cf6eb2ed662cc49661a7b6 Mon Sep 17 00:00:00 2001 From: Adam Shamblin Date: Tue, 22 Nov 2022 11:42:34 -0700 Subject: [PATCH 48/48] wip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b29b9705..9124623d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,5 +10,5 @@ earthly: - linux - amd64 script: - - earthly -i -P --no-cache +package-linux + - earthly --ci -P --no-cache +package-linux