mirror of
https://github.com/edgelesssys/constellation.git
synced 2024-10-01 01:36:09 -04:00
bazel-deps-mirror: upgrade command (#1617)
* bazel-deps-mirror: upgrade command This command can be used to upgrade a dependency. Users are supposed to replace any upstream URLs and run the upgrade command. It replaces the expected hash and uploads the new dep to the mirror.
This commit is contained in:
parent
69de06dd1f
commit
0ece41c146
4
.github/workflows/test-tidy.yml
vendored
4
.github/workflows/test-tidy.yml
vendored
@ -50,7 +50,9 @@ jobs:
|
||||
- name: Upload Bazel dependencies to the mirror
|
||||
if: startsWith(github.head_ref, 'renovate/')
|
||||
shell: bash
|
||||
run: bazelisk run //bazel/ci:deps_mirror_upload
|
||||
run: |
|
||||
bazelisk run //bazel/ci:deps_mirror_upgrade
|
||||
bazelisk run //bazel/ci:deps_mirror_upload
|
||||
|
||||
- name: Run Bazel tidy
|
||||
shell: bash
|
||||
|
@ -17,6 +17,7 @@
|
||||
/e2e @katexochen
|
||||
/hack/azure-jump-host @malt3
|
||||
/hack/azure-snp-report-verify @derpsteb
|
||||
/hack/bazel-deps-mirror @malt3
|
||||
/hack/check-licenses.sh @thomasten
|
||||
/hack/clidocgen @thomasten
|
||||
/hack/fetch-broken-e2e @katexochen
|
||||
|
@ -332,6 +332,10 @@ sh_template(
|
||||
template = "go_generate.sh.in",
|
||||
)
|
||||
|
||||
# deps_mirror_fix fixes bazel workspace rules for external dependencies.
|
||||
# It normalizes the rules and rewrites WORKSPACE and bzl files.
|
||||
# If files are not in the mirror, it will fail.
|
||||
# Use deps_mirror_upload to upload missing files.
|
||||
repo_command(
|
||||
name = "deps_mirror_fix",
|
||||
args = [
|
||||
@ -341,6 +345,8 @@ repo_command(
|
||||
command = "//hack/bazel-deps-mirror",
|
||||
)
|
||||
|
||||
# deps_mirror_upload fixes bazel workspace rules for external dependencies.
|
||||
# It uploads all dependencies to the mirror, normalizes the rules and rewrites WORKSPACE and bzl files.
|
||||
repo_command(
|
||||
name = "deps_mirror_upload",
|
||||
args = [
|
||||
@ -349,6 +355,21 @@ repo_command(
|
||||
command = "//hack/bazel-deps-mirror",
|
||||
)
|
||||
|
||||
# deps_mirror_upgrade upgrades bazel workspace rules for external dependencies.
|
||||
# Users are supposed to replace any upstream URLs.
|
||||
# It replaces the expected hash and uploads the new dep to the mirror.
|
||||
repo_command(
|
||||
name = "deps_mirror_upgrade",
|
||||
args = [
|
||||
"upgrade",
|
||||
],
|
||||
command = "//hack/bazel-deps-mirror",
|
||||
)
|
||||
|
||||
# deps_mirror_check checks bazel workspace rules for external dependencies.
|
||||
# It checks if all dependency rules have mirror urls and are properly formatted.
|
||||
# It doesn't check if the mirror has the files.
|
||||
# Use deps_mirror_check_mirror to check if the mirror has the files.
|
||||
repo_command(
|
||||
name = "deps_mirror_check",
|
||||
args = [
|
||||
@ -357,6 +378,8 @@ repo_command(
|
||||
command = "//hack/bazel-deps-mirror",
|
||||
)
|
||||
|
||||
# deps_mirror_check_mirror checks bazel workspace rules for external dependencies.
|
||||
# It checks if all dependency rules are correctly mirrored and checks that the rules are properly formatted.
|
||||
repo_command(
|
||||
name = "deps_mirror_check_mirror",
|
||||
args = [
|
||||
|
@ -6,6 +6,7 @@ go_library(
|
||||
"bazel-deps-mirror.go",
|
||||
"check.go",
|
||||
"fix.go",
|
||||
"upgrade.go",
|
||||
],
|
||||
importpath = "github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror",
|
||||
visibility = ["//visibility:private"],
|
||||
|
@ -46,6 +46,7 @@ func newRootCmd() *cobra.Command {
|
||||
|
||||
rootCmd.AddCommand(newCheckCmd())
|
||||
rootCmd.AddCommand(newFixCmd())
|
||||
rootCmd.AddCommand(newUpgradeCmd())
|
||||
|
||||
return rootCmd
|
||||
}
|
||||
|
@ -97,7 +97,7 @@ func checkBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorCh
|
||||
found := rules.Rules(buildfile, rules.SupportedRules)
|
||||
if len(found) == 0 {
|
||||
log.Debugf("No rules found in file: %s", bazelFile.RelPath)
|
||||
return
|
||||
return issByFile, nil
|
||||
}
|
||||
log.Debugf("Found %d rules in file: %s", len(found), bazelFile.RelPath)
|
||||
for _, rule := range found {
|
||||
@ -121,7 +121,7 @@ func checkBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorCh
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
return issByFile, nil
|
||||
}
|
||||
|
||||
type checkFlags struct {
|
||||
|
@ -131,41 +131,72 @@ func fixBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorUplo
|
||||
return iss, nil
|
||||
}
|
||||
|
||||
func learnHashForRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.Rule, log *logger.Logger) error {
|
||||
upstreamURLs, err := rules.UpstreamURLs(rule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
learnedHash, learnErr := mirrorUpload.Learn(ctx, upstreamURLs)
|
||||
if learnErr != nil {
|
||||
return err
|
||||
}
|
||||
rules.SetHash(rule, learnedHash)
|
||||
log.Debugf("Learned hash for rule %s: %s", rule.Name(), learnedHash)
|
||||
return nil
|
||||
}
|
||||
|
||||
func fixRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.Rule, log *logger.Logger) (changed bool, iss []error) {
|
||||
log.Debugf("Fixing rule: %s", rule.Name())
|
||||
|
||||
// try to learn the hash
|
||||
if hash, err := rules.GetHash(rule); err != nil || hash == "" {
|
||||
if err := learnHashForRule(ctx, mirrorUpload, rule, log); err != nil {
|
||||
// don't try to fix the rule if the hash is missing and we can't learn it
|
||||
iss = append(iss,
|
||||
errors.New("hash attribute is missing and can't learn it from upstream."+
|
||||
"Unable to check if the artifact is already mirrored or upload it"))
|
||||
return false, iss
|
||||
}
|
||||
changed = true
|
||||
}
|
||||
|
||||
// check if the rule is a valid pinned dependency rule (has all required attributes)
|
||||
issue := rules.ValidatePinned(rule)
|
||||
if issue != nil {
|
||||
// don't try to fix the rule if it's invalid
|
||||
iss = append(iss, issue...)
|
||||
return
|
||||
return false, iss
|
||||
}
|
||||
|
||||
// check if the referenced CAS object exists in the mirror and is consistent
|
||||
expectedHash, expectedHashErr := rules.GetHash(rule)
|
||||
if expectedHashErr != nil {
|
||||
// don't try to fix the rule if the hash is missing
|
||||
iss = append(iss,
|
||||
errors.New("hash attribute is missing. unable to check if the artifact is already mirrored or upload it"))
|
||||
return
|
||||
iss = append(iss, expectedHashErr)
|
||||
return false, iss
|
||||
}
|
||||
|
||||
if rules.HasMirrorURL(rule) {
|
||||
changed = rules.Normalize(rule)
|
||||
return
|
||||
changed = rules.Normalize(rule) || changed
|
||||
return changed, iss
|
||||
}
|
||||
|
||||
log.Infof("Artifact %s with hash %s is not yet mirrored. Uploading...", rule.Name(), expectedHash)
|
||||
if uploadErr := mirrorUpload.Mirror(ctx, expectedHash, rules.GetURLs(rule)); uploadErr != nil {
|
||||
// don't try to fix the rule if the upload failed
|
||||
iss = append(iss, uploadErr)
|
||||
return
|
||||
if checkErr := mirrorUpload.Check(ctx, expectedHash); checkErr != nil {
|
||||
log.Infof("Artifact %s with hash %s is not yet mirrored. Uploading...", rule.Name(), expectedHash)
|
||||
if uploadErr := mirrorUpload.Mirror(ctx, expectedHash, rules.GetURLs(rule)); uploadErr != nil {
|
||||
// don't try to fix the rule if the upload failed
|
||||
iss = append(iss, uploadErr)
|
||||
return changed, iss
|
||||
}
|
||||
} else {
|
||||
log.Infof("Artifact %s with hash %s was already uploaded before. Adding to rule...", rule.Name(), expectedHash)
|
||||
}
|
||||
|
||||
// now the artifact is mirrored (if it wasn't already) and we can fix the rule
|
||||
mirrorURL, err := mirrorUpload.MirrorURL(expectedHash)
|
||||
if err != nil {
|
||||
iss = append(iss, err)
|
||||
return
|
||||
return changed, iss
|
||||
}
|
||||
rules.AddURLs(rule, []string{mirrorURL})
|
||||
|
||||
@ -224,6 +255,7 @@ func parseFixFlags(cmd *cobra.Command) (fixFlags, error) {
|
||||
}
|
||||
|
||||
type mirrorUploader interface {
|
||||
Learn(ctx context.Context, urls []string) (string, error)
|
||||
Check(ctx context.Context, expectedHash string) error
|
||||
Mirror(ctx context.Context, hash string, urls []string) error
|
||||
MirrorURL(hash string) (string, error)
|
||||
|
@ -25,6 +25,7 @@ go_test(
|
||||
"@com_github_aws_aws_sdk_go_v2_service_s3//:s3",
|
||||
"@com_github_aws_aws_sdk_go_v2_service_s3//types",
|
||||
"@com_github_stretchr_testify//assert",
|
||||
"@com_github_stretchr_testify//require",
|
||||
"@org_uber_go_goleak//:goleak",
|
||||
],
|
||||
)
|
||||
|
@ -123,6 +123,27 @@ func (m *Maintainer) Mirror(ctx context.Context, hash string, urls []string) err
|
||||
return fmt.Errorf("failed to download / reupload file with hash %v from any of the urls: %v", hash, urls)
|
||||
}
|
||||
|
||||
// Learn downloads a file from one of the existing (non-mirror) urls, hashes it and returns the hash.
|
||||
func (m *Maintainer) Learn(ctx context.Context, urls []string) (string, error) {
|
||||
for _, url := range urls {
|
||||
m.log.Debugf("Learning new hash from %q", url)
|
||||
body, err := m.downloadFromUpstream(ctx, url)
|
||||
if err != nil {
|
||||
m.log.Debugf("Failed to download file from %q: %v", url, err)
|
||||
continue
|
||||
}
|
||||
defer body.Close()
|
||||
streamedHash := sha256.New()
|
||||
if _, err := io.Copy(streamedHash, body); err != nil {
|
||||
m.log.Debugf("Failed to stream file from %q: %v", url, err)
|
||||
}
|
||||
learnedHash := hex.EncodeToString(streamedHash.Sum(nil))
|
||||
m.log.Debugf("File successfully downloaded from %q with %q", url, learnedHash)
|
||||
return learnedHash, nil
|
||||
}
|
||||
return "", fmt.Errorf("failed to download file / learn hash from any of the urls: %v", urls)
|
||||
}
|
||||
|
||||
// Check checks if a file is present and has the correct hash in the CAS mirror.
|
||||
func (m *Maintainer) Check(ctx context.Context, expectedHash string) error {
|
||||
m.log.Debugf("Checking consistency of object with hash %v", expectedHash)
|
||||
|
@ -19,6 +19,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
@ -146,6 +147,51 @@ func TestMirror(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLearn(t *testing.T) {
|
||||
testCases := map[string]struct {
|
||||
wantHash string
|
||||
upstreamResponse []byte
|
||||
upstreamStatusCode int
|
||||
wantErr bool
|
||||
}{
|
||||
"http error": {
|
||||
upstreamResponse: []byte("foo"), // ignored
|
||||
upstreamStatusCode: http.StatusNotFound,
|
||||
wantErr: true,
|
||||
},
|
||||
"success": {
|
||||
wantHash: "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",
|
||||
upstreamResponse: []byte("foo"),
|
||||
upstreamStatusCode: http.StatusOK,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range testCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
m := Maintainer{
|
||||
unauthenticated: true,
|
||||
httpClient: &http.Client{
|
||||
Transport: &stubUpstream{
|
||||
statusCode: tc.upstreamStatusCode,
|
||||
body: tc.upstreamResponse,
|
||||
},
|
||||
},
|
||||
log: logger.NewTest(t),
|
||||
}
|
||||
gotHash, err := m.Learn(context.Background(), []string{"https://example.com/foo"})
|
||||
if tc.wantErr {
|
||||
assert.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(err)
|
||||
assert.Equal(tc.wantHash, gotHash)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheck(t *testing.T) {
|
||||
testCases := map[string]struct {
|
||||
hash string
|
||||
|
@ -34,7 +34,7 @@ ruleLoop:
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
return rules
|
||||
}
|
||||
|
||||
// ValidatePinned checks if the given rule is a pinned dependency rule.
|
||||
@ -78,7 +78,7 @@ func ValidatePinned(rule *build.Rule) (validationErrs []error) {
|
||||
validationErrs = append(validationErrs, errors.New("rule has empty sha256 attribute"))
|
||||
}
|
||||
}
|
||||
return
|
||||
return validationErrs
|
||||
}
|
||||
|
||||
// Check checks if a dependency rule is normalized and contains a mirror url.
|
||||
@ -107,7 +107,7 @@ func Check(rule *build.Rule) (validationErrs []error) {
|
||||
if rule.Kind() == "rpm" && len(urls) != 1 {
|
||||
validationErrs = append(validationErrs, errors.New("rpm rule has unstable urls that are not the edgeless mirror"))
|
||||
}
|
||||
return
|
||||
return validationErrs
|
||||
}
|
||||
|
||||
// Normalize normalizes a rule and returns true if the rule was changed.
|
||||
@ -122,11 +122,10 @@ func Normalize(rule *build.Rule) (changed bool) {
|
||||
sortURLs(normalizedURLS)
|
||||
normalizedURLS = deduplicateURLs(normalizedURLS)
|
||||
if slices.Equal(urls, normalizedURLS) && rule.Attr("url") == nil {
|
||||
return
|
||||
return changed
|
||||
}
|
||||
setURLs(rule, normalizedURLS)
|
||||
changed = true
|
||||
return
|
||||
return true
|
||||
}
|
||||
|
||||
// AddURLs adds a url to a rule.
|
||||
@ -147,6 +146,11 @@ func GetHash(rule *build.Rule) (string, error) {
|
||||
return hash, nil
|
||||
}
|
||||
|
||||
// SetHash sets the sha256 hash of a rule.
|
||||
func SetHash(rule *build.Rule, hash string) {
|
||||
rule.SetAttr("sha256", &build.StringExpr{Value: hash})
|
||||
}
|
||||
|
||||
// GetURLs returns the sorted urls of a rule.
|
||||
func GetURLs(rule *build.Rule) []string {
|
||||
urls := rule.AttrStrings("urls")
|
||||
@ -157,12 +161,42 @@ func GetURLs(rule *build.Rule) []string {
|
||||
return urls
|
||||
}
|
||||
|
||||
// HasMirrorURL returns true if the rule has a url from the Edgeless mirror.
|
||||
// HasMirrorURL returns true if the rule has a url from the Edgeless mirror
|
||||
// with the correct hash.
|
||||
func HasMirrorURL(rule *build.Rule) bool {
|
||||
_, err := mirrorURL(rule)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// PrepareUpgrade prepares a rule for an upgrade
|
||||
// by removing all urls that are not upstream urls.
|
||||
// and removing the hash attribute.
|
||||
// it returns true if the rule was changed.
|
||||
func PrepareUpgrade(rule *build.Rule) (changed bool, err error) {
|
||||
upstreamURLs, err := UpstreamURLs(rule)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
setURLs(rule, upstreamURLs)
|
||||
rule.DelAttr("sha256")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// UpstreamURLs returns the upstream urls (non-mirror urls) of a rule.
|
||||
func UpstreamURLs(rule *build.Rule) (urls []string, err error) {
|
||||
urls = GetURLs(rule)
|
||||
var upstreamURLs []string
|
||||
for _, url := range urls {
|
||||
if isUpstreamURL(url) {
|
||||
upstreamURLs = append(upstreamURLs, url)
|
||||
}
|
||||
}
|
||||
if len(upstreamURLs) == 0 {
|
||||
return nil, ErrNoUpstreamURL
|
||||
}
|
||||
return upstreamURLs, nil
|
||||
}
|
||||
|
||||
func deduplicateURLs(urls []string) (deduplicated []string) {
|
||||
seen := make(map[string]bool)
|
||||
for _, url := range urls {
|
||||
@ -171,7 +205,7 @@ func deduplicateURLs(urls []string) (deduplicated []string) {
|
||||
seen[url] = true
|
||||
}
|
||||
}
|
||||
return
|
||||
return deduplicated
|
||||
}
|
||||
|
||||
// addTypeAttribute adds the type attribute to http_archive rules if it is missing.
|
||||
@ -243,9 +277,13 @@ urlLoop:
|
||||
|
||||
// mirrorURL returns the first mirror URL for a rule.
|
||||
func mirrorURL(rule *build.Rule) (string, error) {
|
||||
hash, err := GetHash(rule)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
urls := GetURLs(rule)
|
||||
for _, url := range urls {
|
||||
if strings.HasPrefix(url, edgelessMirrorPrefix) {
|
||||
if strings.HasPrefix(url, edgelessMirrorPrefix) && strings.HasSuffix(url, hash) {
|
||||
return url, nil
|
||||
}
|
||||
}
|
||||
@ -284,13 +322,22 @@ func sortURLs(urls []string) {
|
||||
})
|
||||
}
|
||||
|
||||
// SupportedRules is a list of all rules that can be mirrored.
|
||||
var SupportedRules = []string{
|
||||
"http_archive",
|
||||
"http_file",
|
||||
"rpm",
|
||||
func isUpstreamURL(url string) bool {
|
||||
return !strings.HasPrefix(url, bazelMirrorPrefix) && !strings.HasPrefix(url, edgelessMirrorPrefix)
|
||||
}
|
||||
|
||||
var (
|
||||
// SupportedRules is a list of all rules that can be mirrored.
|
||||
SupportedRules = []string{
|
||||
"http_archive",
|
||||
"http_file",
|
||||
"rpm",
|
||||
}
|
||||
|
||||
// ErrNoUpstreamURL is returned when a rule has no upstream URL.
|
||||
ErrNoUpstreamURL = errors.New("rule has no upstream URL")
|
||||
)
|
||||
|
||||
const (
|
||||
bazelMirrorPrefix = "https://mirror.bazel.build/"
|
||||
edgelessMirrorPrefix = "https://cdn.confidential.cloud/constellation/cas/sha256/"
|
||||
|
@ -448,3 +448,54 @@ http_archive(
|
||||
_, err = GetHash(rules[1])
|
||||
assert.Error(err)
|
||||
}
|
||||
|
||||
func TestPrepareUpgrade(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
require := require.New(t)
|
||||
|
||||
rule := `
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
|
||||
http_archive(
|
||||
name = "foo_archive",
|
||||
sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",
|
||||
urls = [
|
||||
"https://cdn.confidential.cloud/constellation/cas/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",
|
||||
"https://mirror.bazel.build/example.com/foo.tar.gz",
|
||||
"https://example.com/foo.tar.gz",
|
||||
],
|
||||
type = "tar.gz",
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "bar_archive",
|
||||
sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",
|
||||
urls = [
|
||||
"https://cdn.confidential.cloud/constellation/cas/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",
|
||||
"https://mirror.bazel.build/example.com/foo.tar.gz",
|
||||
],
|
||||
type = "tar.gz",
|
||||
)
|
||||
`
|
||||
bf, err := build.Parse("foo.bzl", []byte(rule))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rules := Rules(bf, SupportedRules)
|
||||
require.Len(rules, 2)
|
||||
|
||||
changed, err := PrepareUpgrade(rules[0])
|
||||
assert.NoError(err)
|
||||
assert.True(changed)
|
||||
|
||||
urls := GetURLs(rules[0])
|
||||
assert.Equal(1, len(urls))
|
||||
assert.Equal("https://example.com/foo.tar.gz", urls[0])
|
||||
hash, err := GetHash(rules[0])
|
||||
assert.Empty(hash)
|
||||
assert.Error(err)
|
||||
|
||||
changed, err = PrepareUpgrade(rules[1])
|
||||
assert.ErrorIs(err, ErrNoUpstreamURL)
|
||||
assert.False(changed)
|
||||
}
|
||||
|
221
hack/bazel-deps-mirror/upgrade.go
Normal file
221
hack/bazel-deps-mirror/upgrade.go
Normal file
@ -0,0 +1,221 @@
|
||||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/bazelbuild/buildtools/build"
|
||||
"github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/bazelfiles"
|
||||
"github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/issues"
|
||||
"github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/mirror"
|
||||
"github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/rules"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
func newUpgradeCmd() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "upgrade",
|
||||
Short: "upgrade all Bazel dependency rules by recalculating expected hashes, uploading artifacts to the mirror (if needed) and formatting the rules.",
|
||||
RunE: runUpgrade,
|
||||
}
|
||||
|
||||
cmd.Flags().Bool("unauthenticated", false, "Doesn't require authentication to the mirror but cannot upload files.")
|
||||
cmd.Flags().Bool("dry-run", false, "Don't actually change files or upload anything.")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runUpgrade(cmd *cobra.Command, _ []string) error {
|
||||
flags, err := parseUpgradeFlags(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log := logger.New(logger.PlainLog, flags.logLevel)
|
||||
log.Debugf("Parsed flags: %+v", flags)
|
||||
|
||||
fileHelper, err := bazelfiles.New()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("Searching for Bazel files in the current WORKSPACE and all subdirectories...")
|
||||
bazelFiles, err := fileHelper.FindFiles()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var mirrorUpload mirrorUploader
|
||||
switch {
|
||||
case flags.unauthenticated:
|
||||
log.Warnf("Upgrading rules without authentication for AWS S3. If artifacts are not yet mirrored, this will fail.")
|
||||
mirrorUpload = mirror.NewUnauthenticated(flags.mirrorBaseURL, flags.dryRun, log)
|
||||
default:
|
||||
log.Debugf("Upgrading rules with authentication for AWS S3.")
|
||||
mirrorUpload, err = mirror.New(cmd.Context(), flags.region, flags.bucket, flags.mirrorBaseURL, flags.dryRun, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
issues := issues.New()
|
||||
for _, bazelFile := range bazelFiles {
|
||||
fileIssues, err := upgradeBazelFile(cmd.Context(), fileHelper, mirrorUpload, bazelFile, flags.dryRun, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(fileIssues) > 0 {
|
||||
issues.Set(bazelFile.AbsPath, fileIssues)
|
||||
}
|
||||
}
|
||||
if len(issues) > 0 {
|
||||
log.Warnf("Found %d issues in rules", len(issues))
|
||||
issues.Report(cmd.OutOrStdout())
|
||||
return errors.New("found issues in rules")
|
||||
}
|
||||
|
||||
log.Infof("No issues found")
|
||||
return nil
|
||||
}
|
||||
|
||||
func upgradeBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorUpload mirrorUploader, bazelFile bazelfiles.BazelFile, dryRun bool, log *logger.Logger) (iss issues.ByFile, err error) {
|
||||
iss = issues.NewByFile()
|
||||
var changed bool // true if any rule in this file was changed
|
||||
log.Infof("Checking file: %s", bazelFile.RelPath)
|
||||
buildfile, err := fileHelper.LoadFile(bazelFile)
|
||||
if err != nil {
|
||||
return iss, err
|
||||
}
|
||||
found := rules.Rules(buildfile, rules.SupportedRules)
|
||||
if len(found) == 0 {
|
||||
log.Debugf("No rules found in file: %s", bazelFile.RelPath)
|
||||
return iss, nil
|
||||
}
|
||||
log.Debugf("Found %d rules in file: %s", len(found), bazelFile.RelPath)
|
||||
for _, rule := range found {
|
||||
changedRule, ruleIssues := upgradeRule(ctx, mirrorUpload, rule, log)
|
||||
if len(ruleIssues) > 0 {
|
||||
iss.Add(rule.Name(), ruleIssues...)
|
||||
}
|
||||
changed = changed || changedRule
|
||||
}
|
||||
|
||||
if len(iss) > 0 {
|
||||
log.Warnf("File %s has issues. Not saving!", bazelFile.RelPath)
|
||||
return iss, nil
|
||||
}
|
||||
if !changed {
|
||||
log.Debugf("No changes to file: %s", bazelFile.RelPath)
|
||||
return iss, nil
|
||||
}
|
||||
if dryRun {
|
||||
diff, err := fileHelper.Diff(bazelFile, buildfile)
|
||||
if err != nil {
|
||||
return iss, err
|
||||
}
|
||||
log.Infof("Dry run: would save updated file %s with diff:\n%s", bazelFile.RelPath, diff)
|
||||
return iss, nil
|
||||
}
|
||||
log.Infof("Saving updated file: %s", bazelFile.RelPath)
|
||||
if err := fileHelper.WriteFile(bazelFile, buildfile); err != nil {
|
||||
return iss, err
|
||||
}
|
||||
|
||||
return iss, nil
|
||||
}
|
||||
|
||||
func upgradeRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.Rule, log *logger.Logger) (changed bool, iss []error) {
|
||||
log.Debugf("Upgrading rule: %s", rule.Name())
|
||||
|
||||
upstreamURLs, err := rules.UpstreamURLs(rule)
|
||||
if errors.Is(err, rules.ErrNoUpstreamURL) {
|
||||
log.Debugf("Rule has no upstream URL. Skipping.")
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
iss = append(iss, err)
|
||||
return false, iss
|
||||
}
|
||||
|
||||
// learn the hash of the upstream artifact
|
||||
learnedHash, learnErr := mirrorUpload.Learn(ctx, upstreamURLs)
|
||||
if learnErr != nil {
|
||||
iss = append(iss, learnErr)
|
||||
return false, iss
|
||||
}
|
||||
|
||||
existingHash, err := rules.GetHash(rule)
|
||||
if err == nil && learnedHash == existingHash {
|
||||
log.Debugf("Rule already upgraded. Skipping.")
|
||||
return false, nil
|
||||
}
|
||||
|
||||
changed, err = rules.PrepareUpgrade(rule)
|
||||
if err != nil {
|
||||
iss = append(iss, err)
|
||||
return changed, iss
|
||||
}
|
||||
rules.SetHash(rule, learnedHash)
|
||||
changed = true
|
||||
|
||||
if _, fixErr := fixRule(ctx, mirrorUpload, rule, log); fixErr != nil {
|
||||
iss = append(iss, fixErr...)
|
||||
return changed, iss
|
||||
}
|
||||
return changed, iss
|
||||
}
|
||||
|
||||
type upgradeFlags struct {
|
||||
unauthenticated bool
|
||||
dryRun bool
|
||||
region string
|
||||
bucket string
|
||||
mirrorBaseURL string
|
||||
logLevel zapcore.Level
|
||||
}
|
||||
|
||||
func parseUpgradeFlags(cmd *cobra.Command) (upgradeFlags, error) {
|
||||
unauthenticated, err := cmd.Flags().GetBool("unauthenticated")
|
||||
if err != nil {
|
||||
return upgradeFlags{}, err
|
||||
}
|
||||
dryRun, err := cmd.Flags().GetBool("dry-run")
|
||||
if err != nil {
|
||||
return upgradeFlags{}, err
|
||||
}
|
||||
verbose, err := cmd.Flags().GetBool("verbose")
|
||||
if err != nil {
|
||||
return upgradeFlags{}, err
|
||||
}
|
||||
logLevel := zapcore.InfoLevel
|
||||
if verbose {
|
||||
logLevel = zapcore.DebugLevel
|
||||
}
|
||||
region, err := cmd.Flags().GetString("region")
|
||||
if err != nil {
|
||||
return upgradeFlags{}, err
|
||||
}
|
||||
bucket, err := cmd.Flags().GetString("bucket")
|
||||
if err != nil {
|
||||
return upgradeFlags{}, err
|
||||
}
|
||||
mirrorBaseURL, err := cmd.Flags().GetString("mirror-base-url")
|
||||
if err != nil {
|
||||
return upgradeFlags{}, err
|
||||
}
|
||||
|
||||
return upgradeFlags{
|
||||
unauthenticated: unauthenticated,
|
||||
dryRun: dryRun,
|
||||
region: region,
|
||||
bucket: bucket,
|
||||
mirrorBaseURL: mirrorBaseURL,
|
||||
logLevel: logLevel,
|
||||
}, nil
|
||||
}
|
Loading…
Reference in New Issue
Block a user