bazel: pseudo version tool freshness check (#1869)

* switch to darwin compatible shasum

* add bazel rule

* update shellscript for in-place updates

* Revert "update shellscript for in-place updates"

This reverts commit 87d39b06f7.

* add version tool freshness check

* remove pseudo-version file

* revert to `sha256sum`

* fix workflow indentation

.github/workflows/pseudo-version-freshness.yml

@@ -0,0 +1,24 @@
+name: Pseudo-Version tool freshness check
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - renovate/*
+
+
+jobs:
+  check-freshness:
+    name: Pseudo-Version tool freshness check
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        with:
+          persist-credentials: false
+
+      - name: Check freshness
+        shell: bash
+        run: |
+          bazel run //bazel/ci:pseudo_version_tool_freshness

bazel/ci/BUILD.bazel

@@ -489,3 +489,8 @@ multirun(
     jobs = 0,  # execute concurrently
     visibility = ["//visibility:public"],
 )
+
+repo_command(
+    name = "pseudo_version_tool_freshness",
+    command = "//hack/pseudo-version/check",
+)

hack/go.mod

@@ -37,10 +37,12 @@ replace (
 )
 
 require (
+	github.com/aws/aws-sdk-go v1.44.257
 	github.com/aws/aws-sdk-go-v2/config v1.18.23
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.60
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1
 	github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e
+	github.com/bazelbuild/rules_go v0.39.1
 	github.com/edgelesssys/constellation/v2 v2.6.0
 	github.com/go-git/go-git/v5 v5.6.1
 	github.com/hexops/gotextdiff v1.0.3

hack/go.sum

@@ -212,6 +212,8 @@ github.com/aws/aws-sdk-go v1.25.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi
 github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/aws/aws-sdk-go v1.44.257 h1:HwelXYZZ8c34uFFhgVw3ybu2gB5fkk8KLj2idTvzZb8=
+github.com/aws/aws-sdk-go v1.44.257/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
 github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY=
@@ -269,6 +271,8 @@ github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e h1:XmPu4mXICgdGnC5dXGjUGbwUD/kUmS0l5Aop3LaevBM=
 github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo=
+github.com/bazelbuild/rules_go v0.39.1 h1:wkJLUDx59dntWMghuL8++GteoU1To6sRoKJXuyFtmf8=
+github.com/bazelbuild/rules_go v0.39.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=

hack/pseudo-version/BUILD.bazel

@@ -1,12 +1,6 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library")
 load("//bazel/sh:def.bzl", "sh_template")
-
-platforms = [
-    "darwin_amd64",
-    "darwin_arm64",
-    "linux_amd64",
-    "linux_arm64",
-]
+load("//hack/pseudo-version:platforms.bzl", "platforms")
 
 go_library(
     name = "pseudo-version_lib",
@@ -37,18 +31,18 @@ go_binary(
         target = ":pseudo-version",
         visibility = ["//visibility:public"],
     )
-    for platform in platforms
+    for platform in platforms()
 ]
 
 sh_template(
     name = "pseudo_version_tool_freshness",
     data = [
         ":pseudo_version_" + platform
-        for platform in platforms
+        for platform in platforms()
     ],
     substitutions = {
         "@@PSEUDO_VERSION_%s@@" % platform: "$(rootpath :pseudo_version_%s)" % platform
-        for platform in platforms
+        for platform in platforms()
     },
     template = "pseudo_version_tool_freshness.sh.in",
     visibility = ["//visibility:public"],

hack/pseudo-version/check/BUILD.bazel

@@ -0,0 +1,27 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("//hack/pseudo-version:platforms.bzl", "platforms")
+
+go_library(
+    name = "check_lib",
+    srcs = ["check.go"],
+    importpath = "github.com/edgelesssys/constellation/v2/hack/pseudo-version/check",
+    visibility = ["//visibility:private"],
+    deps = [
+        "@com_github_aws_aws_sdk_go//aws",
+        "@com_github_aws_aws_sdk_go//aws/awserr",
+        "@com_github_aws_aws_sdk_go//aws/session",
+        "@com_github_aws_aws_sdk_go//service/s3",
+        "@com_github_aws_aws_sdk_go//service/s3/s3manager",
+        "@io_bazel_rules_go//go/runfiles:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "check",
+    data = [
+        "//hack/pseudo-version:pseudo_version_" + platform
+        for platform in platforms()
+    ],
+    embed = [":check_lib"],
+    visibility = ["//visibility:public"],
+)

hack/pseudo-version/check/check.go

@@ -0,0 +1,176 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/bazelbuild/rules_go/go/runfiles"
+)
+
+const (
+	darwinArm64Filename = "pseudo_version_darwin_arm64"
+	darwinAmd64Filename = "pseudo_version_darwin_amd64"
+	linuxArm64Filename  = "pseudo_version_linux_arm64"
+	linuxAmd64Filename  = "pseudo_version_linux_amd64"
+	bucket              = "cdn-constellation-backend"
+	keyPrefix           = "constellation/cas/sha256/"
+)
+
+func main() {
+	checker, err := newChecker()
+	if err != nil {
+		log.Fatalf("failed to create checker: %v", err)
+	}
+
+	if err := checker.checkAll(); err != nil {
+		log.Fatalf("failed to check pseudo-version tools: %v", err)
+	}
+
+	log.Println("All pseudo-version tools are up-to-date")
+}
+
+// a checker checks if the pseudo-version tool with the specified hash exists in S3.
+type checker struct {
+	files                      *runfiles.Runfiles
+	downloader                 *s3manager.Downloader
+	uploader                   *s3manager.Uploader
+	pseudoVersionToolFilenames []string
+}
+
+// newChecker creates a new checker.
+func newChecker() (*checker, error) {
+	files, err := runfiles.New()
+	if err != nil {
+		return nil, fmt.Errorf("Failed to create runfiles: %v", err)
+	}
+
+	sess := session.Must(session.NewSession(&aws.Config{
+		Region: aws.String("eu-central-1"),
+	}))
+
+	return &checker{
+		files:      files,
+		downloader: s3manager.NewDownloader(sess),
+		uploader:   s3manager.NewUploader(sess),
+		pseudoVersionToolFilenames: []string{
+			darwinArm64Filename,
+			darwinAmd64Filename,
+			linuxArm64Filename,
+			linuxAmd64Filename,
+		},
+	}, nil
+}
+
+// checkAll checks all embedded pseudo-version tools.
+func (c *checker) checkAll() error {
+	for _, filename := range c.pseudoVersionToolFilenames {
+		if err := c.check(filename); err != nil {
+			return fmt.Errorf("failed to check pseudo-version tool (%s): %v", filename, err)
+		}
+	}
+	return nil
+}
+
+// check checks if the pseudo-version tool with the specified hash exists in S3 and
+// uploads it if it doesn't.
+func (c *checker) check(filename string) error {
+	log.Println("Checking pseudo-version tool:", filename)
+	hash, err := c.hashPseudoVersionTool(filename)
+	if err != nil {
+		return fmt.Errorf("failed to hash pseudo-version tool (%s): %v", filename, err)
+	}
+	log.Printf("Hash: %x\n", hash)
+
+	exists, err := c.matchesS3Hash(filename, hash)
+	if err != nil {
+		return fmt.Errorf("failed to check if pseudo-version tool (%s) exists in S3: %v", filename, err)
+	}
+	log.Println("Exists in S3:", exists)
+
+	if !exists {
+		log.Println("Uploading pseudo-version tool:", filename)
+		if err := c.uploadToS3(filename, hash); err != nil {
+			return fmt.Errorf("failed to upload pseudo-version tool (%s) to S3: %v", filename, err)
+		}
+	}
+
+	return nil
+}
+
+// uploadToS3 uploads the pseudo-version tool with the specified hash to S3.
+func (c *checker) uploadToS3(filename string, hash [32]byte) error {
+	contents, err := c.files.ReadFile(fmt.Sprintf("__main__/hack/pseudo-version/%s", filename))
+	if err != nil {
+		return fmt.Errorf("failed to read pseudo-version tool (%s): %v", filename, err)
+	}
+
+	key := keyPrefix + fmt.Sprintf("%x", hash)
+	_, err = c.uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+		Body:   bytes.NewReader(contents),
+	})
+	if err != nil {
+		return fmt.Errorf("failed to upload %x to S3: %v", filename, err)
+	}
+
+	return nil
+}
+
+// matchesS3Hash checks the pseudo-version tool with the specified hash exists in S3.
+func (c *checker) matchesS3Hash(filename string, hash [32]byte) (bool, error) {
+	tmpfileName := filename + "-tmp"
+	tmpfile, err := os.Create(tmpfileName)
+	if err != nil {
+		return false, fmt.Errorf("failed to create temporary file %s: %v", tmpfileName, err)
+	}
+	defer os.Remove(tmpfileName)
+
+	key := keyPrefix + fmt.Sprintf("%x", hash)
+	_, err = c.downloader.Download(tmpfile, &s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		if isNoSuchKeyErr(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("failed to download %x from S3: %v", filename, err)
+	}
+
+	// A file with the hash exists in S3
+	tmpfile.Close()
+	return true, nil
+}
+
+// hashPseudoVersionTool hashes the specified embedded pseudo-version tool.
+func (c *checker) hashPseudoVersionTool(filename string) ([32]byte, error) {
+	contents, err := c.files.ReadFile(fmt.Sprintf("__main__/hack/pseudo-version/%s", filename))
+	if err != nil {
+		return [32]byte{}, fmt.Errorf("failed to read pseudo-version tool (%s): %v", filename, err)
+	}
+
+	return sha256.Sum256(contents), nil
+}
+
+func isNoSuchKeyErr(err error) bool {
+	if aerr, ok := err.(awserr.Error); ok {
+		if aerr.Code() == s3.ErrCodeNoSuchKey {
+			return true
+		}
+	}
+	return false
+}

hack/pseudo-version/platforms.bzl

@@ -0,0 +1,9 @@
+"""Platforms"""
+
+def platforms():
+    return [
+        "darwin_amd64",
+        "darwin_arm64",
+        "linux_amd64",
+        "linux_arm64",
+    ]

tools/update-pseudo-version-tool.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-shopt -s inherit_errexit
-
-platforms=(
-  darwin_amd64
-  darwin_arm64
-  linux_amd64
-  linux_arm64
-)
-bucket=cdn-constellation-backend
-
-dir=$(mktemp -d -t constellation-XXXXXXXXXX)
-trap 'rm -rf "${dir}"' EXIT
-
-bazel build --config nostamp "//hack/pseudo-version:all"
-workspace_dir=$(git rev-parse --show-toplevel)
-
-for platform in "${platforms[@]}"; do
-  echo "Building for ${platform}..."
-  target="//hack/pseudo-version:pseudo_version_${platform}"
-  cp "$(bazel cquery --config nostamp --output=files "${target}")" "${dir}/pseudo_version_${platform}"
-  sha256="$(sha256sum "${dir}/pseudo_version_${platform}" | cut -d ' ' -f 1)"
-  echo "${platform} ${sha256}" | tee -a "${dir}/checksums.txt"
-  aws s3 cp "${dir}/pseudo_version_${platform}" "s3://${bucket}/constellation/cas/sha256/${sha256}"
-  echo "${sha256}" > "${workspace_dir}/tools/pseudo_version_${platform}.sha256"
-done
-
-cat "${dir}/checksums.txt"