From 607ced98db3abf1c3769ed9034aae2b77a533df2 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:42:18 +0200 Subject: [PATCH] bazel: rewrite pseudo-version stamping in bash (#2020) * bazel: simplify workspace_status command to only depend on bash and git * bazel: remove pseudo-version freshness code --- .../workflows/pseudo-version-freshness.yml | 24 --- .gitignore | 1 - bazel/ci/BUILD.bazel | 6 - hack/go.mod | 2 - hack/go.sum | 4 - hack/pseudo-version/BUILD.bazel | 15 -- hack/pseudo-version/check/BUILD.bazel | 27 --- hack/pseudo-version/check/check.go | 176 ------------------ .../pseudo_version_tool_freshness.sh.in | 43 ----- tools/pseudo_version_darwin_amd64.sha256 | 1 - tools/pseudo_version_darwin_arm64.sha256 | 1 - tools/pseudo_version_linux_amd64.sha256 | 1 - tools/pseudo_version_linux_arm64.sha256 | 1 - tools/workspace_status.sh | 88 ++++----- 14 files changed, 36 insertions(+), 354 deletions(-) delete mode 100644 .github/workflows/pseudo-version-freshness.yml delete mode 100644 hack/pseudo-version/check/BUILD.bazel delete mode 100644 hack/pseudo-version/check/check.go delete mode 100644 hack/pseudo-version/pseudo_version_tool_freshness.sh.in delete mode 100644 tools/pseudo_version_darwin_amd64.sha256 delete mode 100644 tools/pseudo_version_darwin_arm64.sha256 delete mode 100644 tools/pseudo_version_linux_amd64.sha256 delete mode 100644 tools/pseudo_version_linux_arm64.sha256 diff --git a/.github/workflows/pseudo-version-freshness.yml b/.github/workflows/pseudo-version-freshness.yml deleted file mode 100644 index 55b80fe82..000000000 --- a/.github/workflows/pseudo-version-freshness.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Pseudo-Version tool freshness check - -on: - workflow_dispatch: - pull_request: - branches: - - renovate/* - - -jobs: - check-freshness: - name: Pseudo-Version tool freshness check - runs-on: ubuntu-22.04 - - steps: - - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - persist-credentials: false - - - name: Check freshness - shell: bash - run: | - bazel run //bazel/ci:pseudo_version_tool_freshness diff --git a/.gitignore b/.gitignore index 6db029bba..74954ae52 100644 --- a/.gitignore +++ b/.gitignore @@ -59,7 +59,6 @@ __pycache__/ # Bazel /bazel-* -tools/pseudo-version .bazeloverwriterc # bazeldnf diff --git a/bazel/ci/BUILD.bazel b/bazel/ci/BUILD.bazel index cb09fcac8..a38e8ecdb 100644 --- a/bazel/ci/BUILD.bazel +++ b/bazel/ci/BUILD.bazel @@ -461,7 +461,6 @@ multirun( ":govulncheck", ":deps_mirror_check", ":proto_targets_check", - "//hack/pseudo-version:pseudo_version_tool_freshness", ] + select({ "@io_bazel_rules_go//go/platform:darwin_arm64": [ ":shellcheck_noop_warning", @@ -489,8 +488,3 @@ multirun( jobs = 0, # execute concurrently visibility = ["//visibility:public"], ) - -repo_command( - name = "pseudo_version_tool_freshness", - command = "//hack/pseudo-version/check", -) diff --git a/hack/go.mod b/hack/go.mod index ffb3c4383..92a772382 100644 --- a/hack/go.mod +++ b/hack/go.mod @@ -37,12 +37,10 @@ replace ( ) require ( - github.com/aws/aws-sdk-go v1.44.257 github.com/aws/aws-sdk-go-v2/config v1.18.27 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e - github.com/bazelbuild/rules_go v0.39.1 github.com/edgelesssys/constellation/v2 v2.6.0 github.com/go-git/go-git/v5 v5.6.1 github.com/hexops/gotextdiff v1.0.3 diff --git a/hack/go.sum b/hack/go.sum index 53cb4917c..548ec0550 100644 --- a/hack/go.sum +++ b/hack/go.sum @@ -214,8 +214,6 @@ github.com/aws/aws-sdk-go v1.25.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.44.257 h1:HwelXYZZ8c34uFFhgVw3ybu2gB5fkk8KLj2idTvzZb8= -github.com/aws/aws-sdk-go v1.44.257/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= @@ -262,8 +260,6 @@ github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e h1:XmPu4mXICgdGnC5dXGjUGbwUD/kUmS0l5Aop3LaevBM= github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo= -github.com/bazelbuild/rules_go v0.39.1 h1:wkJLUDx59dntWMghuL8++GteoU1To6sRoKJXuyFtmf8= -github.com/bazelbuild/rules_go v0.39.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= diff --git a/hack/pseudo-version/BUILD.bazel b/hack/pseudo-version/BUILD.bazel index 4e44fdce6..fe4dd5665 100644 --- a/hack/pseudo-version/BUILD.bazel +++ b/hack/pseudo-version/BUILD.bazel @@ -1,5 +1,4 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library") -load("//bazel/sh:def.bzl", "sh_template") load("//hack/pseudo-version:platforms.bzl", "platforms") go_library( @@ -33,17 +32,3 @@ go_binary( ) for platform in platforms() ] - -sh_template( - name = "pseudo_version_tool_freshness", - data = [ - ":pseudo_version_" + platform - for platform in platforms() - ], - substitutions = { - "@@PSEUDO_VERSION_%s@@" % platform: "$(rootpath :pseudo_version_%s)" % platform - for platform in platforms() - }, - template = "pseudo_version_tool_freshness.sh.in", - visibility = ["//visibility:public"], -) diff --git a/hack/pseudo-version/check/BUILD.bazel b/hack/pseudo-version/check/BUILD.bazel deleted file mode 100644 index d1949b80a..000000000 --- a/hack/pseudo-version/check/BUILD.bazel +++ /dev/null @@ -1,27 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") -load("//hack/pseudo-version:platforms.bzl", "platforms") - -go_library( - name = "check_lib", - srcs = ["check.go"], - importpath = "github.com/edgelesssys/constellation/v2/hack/pseudo-version/check", - visibility = ["//visibility:private"], - deps = [ - "@com_github_aws_aws_sdk_go//aws", - "@com_github_aws_aws_sdk_go//aws/awserr", - "@com_github_aws_aws_sdk_go//aws/session", - "@com_github_aws_aws_sdk_go//service/s3", - "@com_github_aws_aws_sdk_go//service/s3/s3manager", - "@io_bazel_rules_go//go/runfiles:go_default_library", - ], -) - -go_binary( - name = "check", - data = [ - "//hack/pseudo-version:pseudo_version_" + platform - for platform in platforms() - ], - embed = [":check_lib"], - visibility = ["//visibility:public"], -) diff --git a/hack/pseudo-version/check/check.go b/hack/pseudo-version/check/check.go deleted file mode 100644 index b697ab784..000000000 --- a/hack/pseudo-version/check/check.go +++ /dev/null @@ -1,176 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ -package main - -import ( - "bytes" - "crypto/sha256" - "fmt" - "log" - "os" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/bazelbuild/rules_go/go/runfiles" -) - -const ( - darwinArm64Filename = "pseudo_version_darwin_arm64" - darwinAmd64Filename = "pseudo_version_darwin_amd64" - linuxArm64Filename = "pseudo_version_linux_arm64" - linuxAmd64Filename = "pseudo_version_linux_amd64" - bucket = "cdn-constellation-backend" - keyPrefix = "constellation/cas/sha256/" -) - -func main() { - checker, err := newChecker() - if err != nil { - log.Fatalf("failed to create checker: %v", err) - } - - if err := checker.checkAll(); err != nil { - log.Fatalf("failed to check pseudo-version tools: %v", err) - } - - log.Println("All pseudo-version tools are up-to-date") -} - -// a checker checks if the pseudo-version tool with the specified hash exists in S3. -type checker struct { - files *runfiles.Runfiles - downloader *s3manager.Downloader - uploader *s3manager.Uploader - pseudoVersionToolFilenames []string -} - -// newChecker creates a new checker. -func newChecker() (*checker, error) { - files, err := runfiles.New() - if err != nil { - return nil, fmt.Errorf("Failed to create runfiles: %v", err) - } - - sess := session.Must(session.NewSession(&aws.Config{ - Region: aws.String("eu-central-1"), - })) - - return &checker{ - files: files, - downloader: s3manager.NewDownloader(sess), - uploader: s3manager.NewUploader(sess), - pseudoVersionToolFilenames: []string{ - darwinArm64Filename, - darwinAmd64Filename, - linuxArm64Filename, - linuxAmd64Filename, - }, - }, nil -} - -// checkAll checks all embedded pseudo-version tools. -func (c *checker) checkAll() error { - for _, filename := range c.pseudoVersionToolFilenames { - if err := c.check(filename); err != nil { - return fmt.Errorf("failed to check pseudo-version tool (%s): %v", filename, err) - } - } - return nil -} - -// check checks if the pseudo-version tool with the specified hash exists in S3 and -// uploads it if it doesn't. -func (c *checker) check(filename string) error { - log.Println("Checking pseudo-version tool:", filename) - hash, err := c.hashPseudoVersionTool(filename) - if err != nil { - return fmt.Errorf("failed to hash pseudo-version tool (%s): %v", filename, err) - } - log.Printf("Hash: %x\n", hash) - - exists, err := c.matchesS3Hash(filename, hash) - if err != nil { - return fmt.Errorf("failed to check if pseudo-version tool (%s) exists in S3: %v", filename, err) - } - log.Println("Exists in S3:", exists) - - if !exists { - log.Println("Uploading pseudo-version tool:", filename) - if err := c.uploadToS3(filename, hash); err != nil { - return fmt.Errorf("failed to upload pseudo-version tool (%s) to S3: %v", filename, err) - } - } - - return nil -} - -// uploadToS3 uploads the pseudo-version tool with the specified hash to S3. -func (c *checker) uploadToS3(filename string, hash [32]byte) error { - contents, err := c.files.ReadFile(fmt.Sprintf("__main__/hack/pseudo-version/%s", filename)) - if err != nil { - return fmt.Errorf("failed to read pseudo-version tool (%s): %v", filename, err) - } - - key := keyPrefix + fmt.Sprintf("%x", hash) - _, err = c.uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - Body: bytes.NewReader(contents), - }) - if err != nil { - return fmt.Errorf("failed to upload %x to S3: %v", filename, err) - } - - return nil -} - -// matchesS3Hash checks the pseudo-version tool with the specified hash exists in S3. -func (c *checker) matchesS3Hash(filename string, hash [32]byte) (bool, error) { - tmpfileName := filename + "-tmp" - tmpfile, err := os.Create(tmpfileName) - if err != nil { - return false, fmt.Errorf("failed to create temporary file %s: %v", tmpfileName, err) - } - defer os.Remove(tmpfileName) - - key := keyPrefix + fmt.Sprintf("%x", hash) - _, err = c.downloader.Download(tmpfile, &s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) - if err != nil { - if isNoSuchKeyErr(err) { - return false, nil - } - return false, fmt.Errorf("failed to download %x from S3: %v", filename, err) - } - - // A file with the hash exists in S3 - tmpfile.Close() - return true, nil -} - -// hashPseudoVersionTool hashes the specified embedded pseudo-version tool. -func (c *checker) hashPseudoVersionTool(filename string) ([32]byte, error) { - contents, err := c.files.ReadFile(fmt.Sprintf("__main__/hack/pseudo-version/%s", filename)) - if err != nil { - return [32]byte{}, fmt.Errorf("failed to read pseudo-version tool (%s): %v", filename, err) - } - - return sha256.Sum256(contents), nil -} - -func isNoSuchKeyErr(err error) bool { - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == s3.ErrCodeNoSuchKey { - return true - } - } - return false -} diff --git a/hack/pseudo-version/pseudo_version_tool_freshness.sh.in b/hack/pseudo-version/pseudo_version_tool_freshness.sh.in deleted file mode 100644 index cebae2d3e..000000000 --- a/hack/pseudo-version/pseudo_version_tool_freshness.sh.in +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# This script checks if the pseudo-version tool hashes are up-to-date. - -###### script header ###### - -lib=$(realpath @@BASE_LIB@@) || exit 1 -stat "${lib}" >> /dev/null || exit 1 - -# shellcheck source=../../bazel/sh/lib.bash -if ! source "${lib}"; then - echo "Error: could not find import" - exit 1 -fi - -declare -A pseudo_version_tools -pseudo_version_tools["darwin_amd64"]="$(realpath @@PSEUDO_VERSION_darwin_amd64@@)" -pseudo_version_tools["darwin_arm64"]="$(realpath @@PSEUDO_VERSION_darwin_arm64@@)" -pseudo_version_tools["linux_amd64"]="$(realpath @@PSEUDO_VERSION_linux_amd64@@)" -pseudo_version_tools["linux_arm64"]="$(realpath @@PSEUDO_VERSION_linux_arm64@@)" - -cd "${BUILD_WORKING_DIRECTORY}" - -###### script body ###### - -platforms=( - darwin_amd64 - darwin_arm64 - linux_amd64 - linux_arm64 -) - -for platform in "${platforms[@]}"; do - computed_hash=$(sha256sum "${pseudo_version_tools[$platform]}" | cut -d' ' -f1) - # compare hash to saved hash in ${BUILD_WORKSPACE_DIRECTORY}/tools/pseudo_version_${platform}.sha256 - saved_hash=$(cat "${BUILD_WORKSPACE_DIRECTORY}/tools/pseudo_version_${platform}.sha256") - if [[ ${computed_hash} != "${saved_hash}" ]]; then - echo "Error: pseudo-version tool hash for ${platform} does not match saved hash" - echo "Computed hash: ${computed_hash}" - echo "Saved hash: ${saved_hash}" - exit 1 - fi -done diff --git a/tools/pseudo_version_darwin_amd64.sha256 b/tools/pseudo_version_darwin_amd64.sha256 deleted file mode 100644 index 697be53c5..000000000 --- a/tools/pseudo_version_darwin_amd64.sha256 +++ /dev/null @@ -1 +0,0 @@ -2879113c2619cc93292826a7cc6acc59bec8f92cc5e7e0e38a5374e28a3efe54 diff --git a/tools/pseudo_version_darwin_arm64.sha256 b/tools/pseudo_version_darwin_arm64.sha256 deleted file mode 100644 index fbd5ad7d6..000000000 --- a/tools/pseudo_version_darwin_arm64.sha256 +++ /dev/null @@ -1 +0,0 @@ -9b2a42d9887b2bc22e11cd3a7893edc14493b5091fa9085289a604415f905465 diff --git a/tools/pseudo_version_linux_amd64.sha256 b/tools/pseudo_version_linux_amd64.sha256 deleted file mode 100644 index 9abd08950..000000000 --- a/tools/pseudo_version_linux_amd64.sha256 +++ /dev/null @@ -1 +0,0 @@ -1cb39f3092de360491229b25d16dd148698c21776c3c787fc29a4f6a0a6b0ea3 diff --git a/tools/pseudo_version_linux_arm64.sha256 b/tools/pseudo_version_linux_arm64.sha256 deleted file mode 100644 index b7d2936ad..000000000 --- a/tools/pseudo_version_linux_arm64.sha256 +++ /dev/null @@ -1 +0,0 @@ -8aef11a420348cbcf4f3f451a61c413e59f9c3c8ea1f36ad66c8625528a222c5 diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh index 1bc83e375..8f1518953 100755 --- a/tools/workspace_status.sh +++ b/tools/workspace_status.sh @@ -29,62 +29,46 @@ goarch() { esac } -need_pseudo_version_tool() { - if [[ ! -f "${REPOSITORY_ROOT}/tools/pseudo-version" ]]; then - return 1 - fi - - expected=$(cat "${REPOSITORY_ROOT}/tools/pseudo_version_$(goos)_$(goarch).sha256") - local need_pseudo_version_tool=0 - if type sha256sum > /dev/null 2>&1; then - need_pseudo_version_tool=$(sha256sum -c --status <(echo "${expected} ${REPOSITORY_ROOT}/tools/pseudo-version") && echo 0 || echo 1) - elif type shasum > /dev/null 2>&1; then - need_pseudo_version_tool=$(shasum -a 256 -c --status <(echo "${expected} ${REPOSITORY_ROOT}/tools/pseudo-version") && echo 0 || echo 1) - else - echo "sha256sum or shasum is required to verify the pseudo-version tool" >&2 - exit 1 - fi - - return "${need_pseudo_version_tool}" -} - -# shellcheck disable=SC2310 -ensure_pseudo_version_tool() { - local should_download=0 - should_download=$(need_pseudo_version_tool && echo 0 || echo 1) - - if [[ ${should_download} -ne 0 ]]; then - get_pseudo_version_tool - fi -} - -get_pseudo_version_tool() { - out="${REPOSITORY_ROOT}/tools/pseudo-version" - hash=$(cat "${REPOSITORY_ROOT}/tools/pseudo_version_$(goos)_$(goarch).sha256") - url=https://cdn.confidential.cloud/constellation/cas/sha256/${hash} - if command -v curl &> /dev/null; then - curl -fsSL "${url}" -o "${out}" - elif command -v wget &> /dev/null; then - wget -q -O "${out}" "${url}" - else - echo "curl or wget is required to download the pseudo-version tool" >&2 - exit 1 - fi - chmod +x "${out}" -} - -pseudo_version() { - ensure_pseudo_version_tool - "${REPOSITORY_ROOT}/tools/pseudo-version" -skip-v -} - timestamp() { - ensure_pseudo_version_tool - "${REPOSITORY_ROOT}/tools/pseudo-version" -print-timestamp -timestamp-format '2006-01-02T15:04:05Z07:00' + git show -s --date=format:'%Y-%m-%dT%H:%M:%S' --format=%cd HEAD +} + +stamp_version() { + local version + version=$(fixed_version) + # shellcheck disable=SC2310 + if is_pre_version; then + version=$(pseudo_version) + fi + remove_v_prefix "${version}" +} + +is_pre_version() { + local version + version=$(cat "${REPOSITORY_ROOT}/version.txt") + [[ ${version} =~ ^.*-pre.*$ ]] +} + +remove_v_prefix() { + local version=$1 + echo "${version#v}" +} + +# pseudo_version is a bash implementation of the go pseudo version format +# We only care about pre-release versions, so we can simplify the implementation +# See https://pkg.go.dev/golang.org/x/mod/module#PseudoVersion +pseudo_version() { + local prefix + prefix=$(fixed_version) + echo "${prefix}.0.$(git show -s --date=format:'%Y%m%d%H%M%S' --format=%cd HEAD)-$(git rev-parse --short=12 HEAD)" +} + +fixed_version() { + cat "${REPOSITORY_ROOT}/version.txt" } echo "REPO_URL https://github.com/edgelesssys/constellation.git" echo "STABLE_STAMP_COMMIT $(git rev-parse HEAD)" echo "STABLE_STAMP_STATE $(git update-index -q --really-refresh && git diff-index --quiet HEAD -- && echo "clean" || echo "dirty")" -echo "STABLE_STAMP_VERSION $(pseudo_version)" +echo "STABLE_STAMP_VERSION $(stamp_version)" echo "STABLE_STAMP_TIME $(timestamp)"