bazel: pseudo version tool freshness check (#1869)

* switch to darwin compatible shasum

* add bazel rule

* update shellscript for in-place updates

* Revert "update shellscript for in-place updates"

This reverts commit 87d39b06f7.

* add version tool freshness check

* remove pseudo-version file

* revert to `sha256sum`

* fix workflow indentation
This commit is contained in:
Moritz Sanft 2023-06-09 11:50:51 +02:00 committed by GitHub
parent 892752a1f8
commit 72e168e653
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 251 additions and 40 deletions

View File

@ -0,0 +1,24 @@
name: Pseudo-Version tool freshness check
on:
workflow_dispatch:
pull_request:
branches:
- renovate/*
jobs:
check-freshness:
name: Pseudo-Version tool freshness check
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
with:
persist-credentials: false
- name: Check freshness
shell: bash
run: |
bazel run //bazel/ci:pseudo_version_tool_freshness

View File

@ -489,3 +489,8 @@ multirun(
jobs = 0, # execute concurrently jobs = 0, # execute concurrently
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
repo_command(
name = "pseudo_version_tool_freshness",
command = "//hack/pseudo-version/check",
)

View File

@ -37,10 +37,12 @@ replace (
) )
require ( require (
github.com/aws/aws-sdk-go v1.44.257
github.com/aws/aws-sdk-go-v2/config v1.18.23 github.com/aws/aws-sdk-go-v2/config v1.18.23
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.60 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.60
github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1 github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1
github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e
github.com/bazelbuild/rules_go v0.39.1
github.com/edgelesssys/constellation/v2 v2.6.0 github.com/edgelesssys/constellation/v2 v2.6.0
github.com/go-git/go-git/v5 v5.6.1 github.com/go-git/go-git/v5 v5.6.1
github.com/hexops/gotextdiff v1.0.3 github.com/hexops/gotextdiff v1.0.3

View File

@ -212,6 +212,8 @@ github.com/aws/aws-sdk-go v1.25.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi
github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.44.257 h1:HwelXYZZ8c34uFFhgVw3ybu2gB5fkk8KLj2idTvzZb8=
github.com/aws/aws-sdk-go v1.44.257/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY=
@ -269,6 +271,8 @@ github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e h1:XmPu4mXICgdGnC5dXGjUGbwUD/kUmS0l5Aop3LaevBM= github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e h1:XmPu4mXICgdGnC5dXGjUGbwUD/kUmS0l5Aop3LaevBM=
github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo= github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo=
github.com/bazelbuild/rules_go v0.39.1 h1:wkJLUDx59dntWMghuL8++GteoU1To6sRoKJXuyFtmf8=
github.com/bazelbuild/rules_go v0.39.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=

View File

@ -1,12 +1,6 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library") load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library")
load("//bazel/sh:def.bzl", "sh_template") load("//bazel/sh:def.bzl", "sh_template")
load("//hack/pseudo-version:platforms.bzl", "platforms")
platforms = [
"darwin_amd64",
"darwin_arm64",
"linux_amd64",
"linux_arm64",
]
go_library( go_library(
name = "pseudo-version_lib", name = "pseudo-version_lib",
@ -37,18 +31,18 @@ go_binary(
target = ":pseudo-version", target = ":pseudo-version",
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
for platform in platforms for platform in platforms()
] ]
sh_template( sh_template(
name = "pseudo_version_tool_freshness", name = "pseudo_version_tool_freshness",
data = [ data = [
":pseudo_version_" + platform ":pseudo_version_" + platform
for platform in platforms for platform in platforms()
], ],
substitutions = { substitutions = {
"@@PSEUDO_VERSION_%s@@" % platform: "$(rootpath :pseudo_version_%s)" % platform "@@PSEUDO_VERSION_%s@@" % platform: "$(rootpath :pseudo_version_%s)" % platform
for platform in platforms for platform in platforms()
}, },
template = "pseudo_version_tool_freshness.sh.in", template = "pseudo_version_tool_freshness.sh.in",
visibility = ["//visibility:public"], visibility = ["//visibility:public"],

View File

@ -0,0 +1,27 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
load("//hack/pseudo-version:platforms.bzl", "platforms")
go_library(
name = "check_lib",
srcs = ["check.go"],
importpath = "github.com/edgelesssys/constellation/v2/hack/pseudo-version/check",
visibility = ["//visibility:private"],
deps = [
"@com_github_aws_aws_sdk_go//aws",
"@com_github_aws_aws_sdk_go//aws/awserr",
"@com_github_aws_aws_sdk_go//aws/session",
"@com_github_aws_aws_sdk_go//service/s3",
"@com_github_aws_aws_sdk_go//service/s3/s3manager",
"@io_bazel_rules_go//go/runfiles:go_default_library",
],
)
go_binary(
name = "check",
data = [
"//hack/pseudo-version:pseudo_version_" + platform
for platform in platforms()
],
embed = [":check_lib"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,176 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package main
import (
"bytes"
"crypto/sha256"
"fmt"
"log"
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/bazelbuild/rules_go/go/runfiles"
)
const (
darwinArm64Filename = "pseudo_version_darwin_arm64"
darwinAmd64Filename = "pseudo_version_darwin_amd64"
linuxArm64Filename = "pseudo_version_linux_arm64"
linuxAmd64Filename = "pseudo_version_linux_amd64"
bucket = "cdn-constellation-backend"
keyPrefix = "constellation/cas/sha256/"
)
func main() {
checker, err := newChecker()
if err != nil {
log.Fatalf("failed to create checker: %v", err)
}
if err := checker.checkAll(); err != nil {
log.Fatalf("failed to check pseudo-version tools: %v", err)
}
log.Println("All pseudo-version tools are up-to-date")
}
// a checker checks if the pseudo-version tool with the specified hash exists in S3.
type checker struct {
files *runfiles.Runfiles
downloader *s3manager.Downloader
uploader *s3manager.Uploader
pseudoVersionToolFilenames []string
}
// newChecker creates a new checker.
func newChecker() (*checker, error) {
files, err := runfiles.New()
if err != nil {
return nil, fmt.Errorf("Failed to create runfiles: %v", err)
}
sess := session.Must(session.NewSession(&aws.Config{
Region: aws.String("eu-central-1"),
}))
return &checker{
files: files,
downloader: s3manager.NewDownloader(sess),
uploader: s3manager.NewUploader(sess),
pseudoVersionToolFilenames: []string{
darwinArm64Filename,
darwinAmd64Filename,
linuxArm64Filename,
linuxAmd64Filename,
},
}, nil
}
// checkAll checks all embedded pseudo-version tools.
func (c *checker) checkAll() error {
for _, filename := range c.pseudoVersionToolFilenames {
if err := c.check(filename); err != nil {
return fmt.Errorf("failed to check pseudo-version tool (%s): %v", filename, err)
}
}
return nil
}
// check checks if the pseudo-version tool with the specified hash exists in S3 and
// uploads it if it doesn't.
func (c *checker) check(filename string) error {
log.Println("Checking pseudo-version tool:", filename)
hash, err := c.hashPseudoVersionTool(filename)
if err != nil {
return fmt.Errorf("failed to hash pseudo-version tool (%s): %v", filename, err)
}
log.Printf("Hash: %x\n", hash)
exists, err := c.matchesS3Hash(filename, hash)
if err != nil {
return fmt.Errorf("failed to check if pseudo-version tool (%s) exists in S3: %v", filename, err)
}
log.Println("Exists in S3:", exists)
if !exists {
log.Println("Uploading pseudo-version tool:", filename)
if err := c.uploadToS3(filename, hash); err != nil {
return fmt.Errorf("failed to upload pseudo-version tool (%s) to S3: %v", filename, err)
}
}
return nil
}
// uploadToS3 uploads the pseudo-version tool with the specified hash to S3.
func (c *checker) uploadToS3(filename string, hash [32]byte) error {
contents, err := c.files.ReadFile(fmt.Sprintf("__main__/hack/pseudo-version/%s", filename))
if err != nil {
return fmt.Errorf("failed to read pseudo-version tool (%s): %v", filename, err)
}
key := keyPrefix + fmt.Sprintf("%x", hash)
_, err = c.uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: bytes.NewReader(contents),
})
if err != nil {
return fmt.Errorf("failed to upload %x to S3: %v", filename, err)
}
return nil
}
// matchesS3Hash checks the pseudo-version tool with the specified hash exists in S3.
func (c *checker) matchesS3Hash(filename string, hash [32]byte) (bool, error) {
tmpfileName := filename + "-tmp"
tmpfile, err := os.Create(tmpfileName)
if err != nil {
return false, fmt.Errorf("failed to create temporary file %s: %v", tmpfileName, err)
}
defer os.Remove(tmpfileName)
key := keyPrefix + fmt.Sprintf("%x", hash)
_, err = c.downloader.Download(tmpfile, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
if isNoSuchKeyErr(err) {
return false, nil
}
return false, fmt.Errorf("failed to download %x from S3: %v", filename, err)
}
// A file with the hash exists in S3
tmpfile.Close()
return true, nil
}
// hashPseudoVersionTool hashes the specified embedded pseudo-version tool.
func (c *checker) hashPseudoVersionTool(filename string) ([32]byte, error) {
contents, err := c.files.ReadFile(fmt.Sprintf("__main__/hack/pseudo-version/%s", filename))
if err != nil {
return [32]byte{}, fmt.Errorf("failed to read pseudo-version tool (%s): %v", filename, err)
}
return sha256.Sum256(contents), nil
}
func isNoSuchKeyErr(err error) bool {
if aerr, ok := err.(awserr.Error); ok {
if aerr.Code() == s3.ErrCodeNoSuchKey {
return true
}
}
return false
}

View File

@ -0,0 +1,9 @@
"""Platforms"""
def platforms():
return [
"darwin_amd64",
"darwin_arm64",
"linux_amd64",
"linux_arm64",
]

View File

@ -1,30 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
shopt -s inherit_errexit
platforms=(
darwin_amd64
darwin_arm64
linux_amd64
linux_arm64
)
bucket=cdn-constellation-backend
dir=$(mktemp -d -t constellation-XXXXXXXXXX)
trap 'rm -rf "${dir}"' EXIT
bazel build --config nostamp "//hack/pseudo-version:all"
workspace_dir=$(git rev-parse --show-toplevel)
for platform in "${platforms[@]}"; do
echo "Building for ${platform}..."
target="//hack/pseudo-version:pseudo_version_${platform}"
cp "$(bazel cquery --config nostamp --output=files "${target}")" "${dir}/pseudo_version_${platform}"
sha256="$(sha256sum "${dir}/pseudo_version_${platform}" | cut -d ' ' -f 1)"
echo "${platform} ${sha256}" | tee -a "${dir}/checksums.txt"
aws s3 cp "${dir}/pseudo_version_${platform}" "s3://${bucket}/constellation/cas/sha256/${sha256}"
echo "${sha256}" > "${workspace_dir}/tools/pseudo_version_${platform}.sha256"
done
cat "${dir}/checksums.txt"