From 8d3fe6f4777e8fe9854382924a0ff0c25f90ddf1 Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Fri, 17 Mar 2023 16:20:39 +0100 Subject: [PATCH] bazel: add terrafrom to //:check and //:generate Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com> --- .github/workflows/test-tf.yml | 66 --------------------- .github/workflows/test-tidy.yml | 51 ++++++++++++++-- BUILD.bazel | 5 ++ bazel/ci/BUILD.bazel | 44 ++++++++++++++ bazel/ci/tf.sh.in | 82 ++++++++++++++++++++++++++ bazel/toolchains/BUILD.terraform.bazel | 8 +++ bazel/toolchains/ci_deps.bzl | 36 +++++++++++ 7 files changed, 221 insertions(+), 71 deletions(-) delete mode 100644 .github/workflows/test-tf.yml create mode 100644 bazel/ci/tf.sh.in create mode 100644 bazel/toolchains/BUILD.terraform.bazel diff --git a/.github/workflows/test-tf.yml b/.github/workflows/test-tf.yml deleted file mode 100644 index 56d85f2e3..000000000 --- a/.github/workflows/test-tf.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Terraform validation - -on: - workflow_dispatch: - push: - branches: - - main - - "release/**" - paths: - - "**.tf" - - "**.lock.hcl" - - ".github/workflows/test-tf.yml" - pull_request: - paths: - - "**.tf" - - "**.lock.hcl" - - ".github/workflows/test-tf.yml" - -jobs: - tfsec: - name: terraform - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - with: - ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - # No token available for forks, so we can't push changes - token: ${{ !github.event.pull_request.head.repo.fork && secrets.CI_COMMIT_PUSH_PR || '' }} - - - name: Setup Terraform - uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # tag=v2.0.3 - - - name: Terraform format and validate - shell: bash - run: | - dirs=$(find . -type f -name "*.tf" -exec dirname "{}" \; | sort -ud) - result=0 - for dir in $dirs; do - echo "Checking $dir" - terraform -chdir="$dir" init || result=1 - terraform -chdir="$dir" fmt -check=true -diff=true || result=1 - terraform -chdir="$dir" validate -no-color || result=1 - done - exit $result - - - name: Check HCL lock files are up to date - id: hcl-lock - shell: bash - run: | - dirs=$( find . -type f -name "*.lock.hcl" -exec dirname "{}" \; | sort -ud) - for dir in $dirs; do - echo "Checking $dir" - terraform -chdir="$dir" init - terraform -chdir="$dir" providers lock -platform=linux_arm64 -platform=linux_amd64 -platform=darwin_arm64 -platform=darwin_amd64 -platform=windows_amd64 - done - git diff --exit-code - - - name: Push changes on renovate - if: failure() && (steps.hcl-lock.conclusion == 'failure') && startsWith(github.head_ref, 'renovate/') - shell: bash - run: | - git config --global user.name "edgelessci" - git config --global user.email "edgelessci@users.noreply.github.com" - git commit -am "deps: update HCL lock files" - git push diff --git a/.github/workflows/test-tidy.yml b/.github/workflows/test-tidy.yml index d5cd8d234..60be79134 100644 --- a/.github/workflows/test-tidy.yml +++ b/.github/workflows/test-tidy.yml @@ -34,21 +34,62 @@ jobs: shell: bash run: bazelisk run //:tidy - - name: Check if tidy made modifications - id: tidycheck + - name: Check if untidy + id: untidy shell: bash run: | diff=$(git diff) if [[ -z "$diff" ]]; then - echo "Everything is tidy" + echo "Everything is tidy." + echo "untidy=false" >> "$GITHUB_OUTPUT" exit 0 fi + echo "Detected changes after tidy" + echo "untidy=true" >> "$GITHUB_OUTPUT" + + - name: Run Bazel generate + shell: bash + run: bazelisk run //:generate + + - name: Check if ungenerated + id: ungenerated + shell: bash + run: | + diff=$(git diff) + if [[ -z "$diff" ]]; then + echo "Everything is tidy." + echo "ungenerated=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "Detected changes after tidy" + echo "ungenerated=true" >> "$GITHUB_OUTPUT" + + - name: Check if tidy or generate made modifications + id: modified + shell: bash + run: | + diff=$(git diff) + if [[ -z "$diff" ]]; then + echo "Everything is tidy and generated." + exit 0 + fi + cat << EOF >> "${GITHUB_STEP_SUMMARY}" \`\`\`diff ${diff} \`\`\` EOF - echo "::error::The repo is not tidy. Please run 'bazel run //:tidy' and commit the changes." + + if [[ "${{ steps.untidy.outputs.untidy }}" == "true" ]] && + [[ "${{ steps.ungenerated.outputs.ungenerated }}" == "true" ]]; then + suggestCmd="'bazel run //:generate' &&' bazel run //:tidy'" + elif [[ "${{ steps.untidy.outputs.untidy }}" == "true" ]]; then + suggestCmd="'bazel run //:tidy'" + elif [[ "${{ steps.ungenerated.outputs.ungenerated }}" == "true" ]]; then + suggestCmd="'bazel run //:generate'" + fi + + echo "::error::The repo is not tidy. Please run ${suggestCmd} and commit the changes." exit 1 - name: Run Bazel check @@ -62,7 +103,7 @@ jobs: - name: Push changes if: | failure() && - (steps.tidycheck.conclusion == 'failure') && + (steps.modified.conclusion == 'failure') && startsWith(github.head_ref, 'renovate/') && !github.event.pull_request.head.repo.fork shell: bash diff --git a/BUILD.bazel b/BUILD.bazel index 30ce61269..c50c54e54 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -16,6 +16,11 @@ alias( actual = "//bazel/ci:check", ) +alias( + name = "generate", + actual = "//bazel/ci:generate", +) + alias( name = "devbuild", actual = "//bazel/devbuild:devbuild", diff --git a/bazel/ci/BUILD.bazel b/bazel/ci/BUILD.bazel index 77d4dd9aa..a4215cca5 100644 --- a/bazel/ci/BUILD.bazel +++ b/bazel/ci/BUILD.bazel @@ -131,6 +131,40 @@ sh_template( template = "tfsec.sh.in", ) +alias( + name = "com_github_hashicorp_terraform", + actual = select({ + "@io_bazel_rules_go//go/platform:darwin_amd64": "@com_github_hashicorp_terraform_darwin_amd64//:terraform_bin", + "@io_bazel_rules_go//go/platform:darwin_arm64": "@com_github_hashicorp_terraform_darwin_arm64//:terraform_bin", + "@io_bazel_rules_go//go/platform:linux_amd64": "@com_github_hashicorp_terraform_linux_amd64//:terraform_bin", + "@io_bazel_rules_go//go/platform:linux_arm64": "@com_github_hashicorp_terraform_linux_arm64//:terraform_bin", + }), +) + +sh_template( + name = "terraform_gen", + data = [ + ":com_github_hashicorp_terraform", + ], + substitutions = { + "@@MODE@@": "generate", + "@@TERRAFORM@@": "$(rootpath :com_github_hashicorp_terraform)", + }, + template = "tf.sh.in", +) + +sh_template( + name = "terraform_check", + data = [ + ":com_github_hashicorp_terraform", + ], + substitutions = { + "@@MODE@@": "check", + "@@TERRAFORM@@": "$(rootpath :com_github_hashicorp_terraform)", + }, + template = "tf.sh.in", +) + multirun( name = "tidy", commands = [ @@ -151,6 +185,7 @@ multirun( commands = [ ":gazelle_check", ":buildifier_check", + ":terraform_check", ] + select({ "@io_bazel_rules_go//go/platform:darwin_arm64": [ ":shellcheck_noop_warning", @@ -164,3 +199,12 @@ multirun( jobs = 1, # execute sequentially visibility = ["//visibility:public"], ) + +multirun( + name = "generate", + commands = [ + ":terraform_gen", + ], + jobs = 1, # execute sequentially + visibility = ["//visibility:public"], +) diff --git a/bazel/ci/tf.sh.in b/bazel/ci/tf.sh.in new file mode 100644 index 000000000..f6875f110 --- /dev/null +++ b/bazel/ci/tf.sh.in @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +lib=$(realpath @@BASE_LIB@@) || exit 1 +terraform=$(realpath @@TERRAFORM@@) || exit 1 +mode="@@MODE@@" || exit 1 + +# shellcheck source=../sh/lib.bash +if ! source "${lib}"; then + echo "Error: could not find import" + exit 1 +fi + +cd "${BUILD_WORKSPACE_DIRECTORY}" || exit 1 + +readarray -t <<< "$( + find "$(pwd)" -type f -name "*.tf" -exec dirname "{}" \; | + sort -ud +)" +terraformPaths=("${MAPFILE[@]}") +terraformModules=() +pathPrefix="${terraformPaths[0]}" +for ((i = 1; i < ${#terraformPaths[@]}; i++)); do + path="${terraformPaths[i]}" + if [[ ${path} == "${pathPrefix}"* ]]; then + continue + fi + terraformModules+=("${pathPrefix}") + pathPrefix="${path}" +done + +excludeDirs=( + "build" +) + +echo "The following Terraform modules are excluded and won't be tidied:" +for exclude in "${excludeDirs[@]}"; do + for i in "${!terraformModules[@]}"; do + if [[ ${terraformModules[i]} == "${BUILD_WORKSPACE_DIRECTORY}/${exclude}"* ]]; then + echo " ${terraformModules[i]}" + unset 'terraformModules[i]' + fi + done +done + +case ${mode} in +"check") + echo "Checking validity and format of the following Terraform modules:" + for script in "${terraformModules[@]}"; do + echo " ${script}" + done + echo "This may take a minute..." + for module in "${terraformModules[@]}"; do + ${terraform} -chdir="${module}" init > /dev/null + ${terraform} -chdir="${module}" fmt -check -recursive > /dev/null + ${terraform} -chdir="${module}" validate > /dev/null + rm -rf "${module}/.terraform" + done + ;; + +"generate") + echo "Formatting and generating lock files for the following Terraform modules:" + for script in "${terraformModules[@]}"; do + echo " ${script}" + done + echo "This may take 5-10 min..." + for module in "${terraformModules[@]}"; do + ${terraform} -chdir="${module}" init > /dev/null + ${terraform} -chdir="${module}" providers lock -platform=linux_arm64 > /dev/null + ${terraform} -chdir="${module}" providers lock -platform=linux_amd64 > /dev/null + ${terraform} -chdir="${module}" providers lock -platform=darwin_arm64 > /dev/null + ${terraform} -chdir="${module}" providers lock -platform=darwin_amd64 > /dev/null + ${terraform} -chdir="${module}" providers lock -platform=windows_amd64 > /dev/null + ${terraform} -chdir="${module}" fmt -recursive > /dev/null + rm -rf "${module}/.terraform" + done + ;; + +*) + echo "Error: unknown mode \"${mode}\"" + exit 1 + ;; +esac diff --git a/bazel/toolchains/BUILD.terraform.bazel b/bazel/toolchains/BUILD.terraform.bazel new file mode 100644 index 000000000..b406963d4 --- /dev/null +++ b/bazel/toolchains/BUILD.terraform.bazel @@ -0,0 +1,8 @@ +genrule( + name = "terraform_bin", + srcs = ["terraform"], + outs = ["terraform_bin_out"], + cmd = "cp $< $@", # Copy the binary to the output directory. + executable = True, + visibility = ["//visibility:public"], +) diff --git a/bazel/toolchains/ci_deps.bzl b/bazel/toolchains/ci_deps.bzl index f270b68b1..b77ae39b4 100644 --- a/bazel/toolchains/ci_deps.bzl +++ b/bazel/toolchains/ci_deps.bzl @@ -3,7 +3,9 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def ci_deps(): + """Install CI dependencies""" _shellcheck_deps() + _terraform_deps() def _shellcheck_deps(): http_archive( @@ -31,3 +33,37 @@ def _shellcheck_deps(): strip_prefix = "shellcheck-v0.9.0", build_file = "//bazel/toolchains:BUILD.shellcheck.bazel", ) + +def _terraform_deps(): + http_archive( + name = "com_github_hashicorp_terraform_linux_amd64", + build_file = "//bazel/toolchains:BUILD.terraform.bazel", + urls = [ + "https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_amd64.zip", + ], + sha256 = "9f3ca33d04f5335472829d1df7785115b60176d610ae6f1583343b0a2221a931", + ) + http_archive( + name = "com_github_hashicorp_terraform_linux_arm64", + build_file = "//bazel/toolchains:BUILD.terraform.bazel", + urls = [ + "https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_arm64.zip", + ], + sha256 = "39c182670c4e63e918e0a16080b1cc47bb16e158d7da96333d682d6a9cb8eb91", + ) + http_archive( + name = "com_github_hashicorp_terraform_darwin_amd64", + build_file = "//bazel/toolchains:BUILD.terraform.bazel", + urls = [ + "https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_darwin_amd64.zip", + ], + sha256 = "c218a6c0ef6692b25af16995c8c7bdf6739e9638fef9235c6aced3cd84afaf66", + ) + http_archive( + name = "com_github_hashicorp_terraform_darwin_arm64", + build_file = "//bazel/toolchains:BUILD.terraform.bazel", + urls = [ + "https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_darwin_arm64.zip", + ], + sha256 = "af8ff7576c8fc41496fdf97e9199b00d8d81729a6a0e821eaf4dfd08aa763540", + )