AB#2158 publish measurements (#268)

* cleaned up actions and new measure action to generate, sign and upload measurements
* improve constellation ip fetching to support multiple control nodes
Signed-off-by: Fabian Kammel <fk@edgeless.systems>

.github/actions/azure_login/action.yml

@@ -1,4 +1,4 @@
-name: azure_login
+name: Azure login
 description: "Login to Azure & configure az CLI."
 inputs:
   azure_credentials:

.github/actions/build_cli/action.yml

@@ -1,17 +1,18 @@
-name: build
+name: Build CLI
 description: |
-  Runs cmake & default make target in build folder. Additionally, Sigstore tools
-  are used to sign CLI and publish a release when run on v* tag.
+  Runs cmake and cli make target in build folder. Optionally, Sigstore tools
+  are used to sign CLI when inputs are provided. A draft release is published
+  when run on v* tag.
 inputs:
-  cosign-public-key:
+  cosignPublicKey:
     description: 'Cosign public key'
     required: false
     default: ''
-  cosign-private-key:
+  cosignPrivateKey:
     description: 'Cosign private key'
     required: false
     default: ''
-  cosign-password:
+  cosignPassword:
     description: 'Password for Cosign private key'
     required: false
     default: ''
@@ -82,10 +83,10 @@ runs:
       shell: bash
       working-directory: build
       env:
-        COSIGN_PUBLIC_KEY: ${{ inputs.cosign-public-key }}
-        COSIGN_PRIVATE_KEY: ${{ inputs.cosign-private-key }}
-        COSIGN_PASSWORD: ${{ inputs.cosign-password }}
-      if: ${{ inputs.cosign-public-key != '' && inputs.cosign-private-key != '' && inputs.cosign-password != '' }}
+        COSIGN_PUBLIC_KEY: ${{ inputs.cosignPublicKey }}
+        COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }}
+        COSIGN_PASSWORD: ${{ inputs.cosignPassword }}
+      if: ${{ inputs.cosignPublicKey != '' && inputs.cosignPrivateKey != '' && inputs.cosignPassword != '' }}
 
     - name: Release CLI
       # GitHub endorsed release project. See: https://github.com/actions/create-release

.github/actions/build_micro_service/action.yml

@@ -1,4 +1,4 @@
-name: build micro-service image
+name: Build micro service
 description: Build and upload a container image for a Constellation micro-service
 inputs:
   name:

.github/actions/constellation_create/action.yml

@@ -1,5 +1,6 @@
-name: constellation_create
-description: "Create a new Constellation cluster."
+name: Constellation create
+description: |
+  Create a new Constellation cluster using latest CoreOS image.
 inputs:
   workerNodesCount:
     description: "Number of worker nodes to spawn."
@@ -40,18 +41,21 @@ runs:
       constellation config generate ${{ inputs.cloudProvider }}
     shell: bash
 
-  - name: Set latest Azure image
+  - name: Set latest image
     run: |
-      LATEST_AZURE_IMAGE=$(az sig image-version list --resource-group constellation-images --gallery-name Constellation --gallery-image-definition constellation-coreos --query "sort_by([], &publishingProfile.publishedDate)[].id" -o table | tail -n 1)
-      yq eval -i "(.provider.azure.image) = \"${LATEST_AZURE_IMAGE}\"" constellation-conf.yaml
+      case $CSP in
+        azure)
+          LATEST_AZURE_IMAGE=$(az sig image-version list --resource-group constellation-images --gallery-name Constellation --gallery-image-definition constellation-coreos --query "sort_by([], &publishingProfile.publishedDate)[].id" -o table | tail -n 1)
+          yq eval -i "(.provider.azure.image) = \"${LATEST_AZURE_IMAGE}\"" constellation-conf.yaml
+          ;;
+        gcp)
+          LATEST_GCP_IMAGE_TIMESTAMP=$(gcloud compute images list --filter="name~'constellation-coreos-\d{10}'" --sort-by=creationTimestamp --project constellation-images --format="table(name)" | tail -n 1 | cut -d '-' -f3)
+          yq eval -i "(.provider.gcp.image) = \"projects/constellation-images/global/images/constellation-coreos-${LATEST_GCP_IMAGE_TIMESTAMP}\"" constellation-conf.yaml
+          ;;
+      esac
     shell: bash
-    if: ${{ inputs.cloudProvider == 'azure' }}
-  - name: Set latest GCP image
-    run: |
-      LATEST_GCP_IMAGE_TIMESTAMP=$(gcloud compute images list --filter="name~'constellation-coreos-\d{10}'" --sort-by=creationTimestamp --project constellation-images --format="table(name)" | tail -n 1 | cut -d '-' -f3)
-      yq eval -i "(.provider.gcp.image) = \"projects/constellation-images/global/images/constellation-coreos-${LATEST_GCP_IMAGE_TIMESTAMP}\"" constellation-conf.yaml
-    shell: bash
-    if: ${{ inputs.cloudProvider == 'gcp' }}
+    env:
+      CSP: ${{ inputs.cloudProvider }}
 
   - name: Constellation create
     run: |
@@ -66,34 +70,12 @@ runs:
       path: constellation-state.json
     if: ${{ always() && !env.ACT }}
 
-  - name: Read Coordinator IP (Azure)
-    run: |
-      echo CONSTELL_IP=$(jq -r .azurecoordinators[].PublicIP constellation-state.json) >> $GITHUB_ENV
-    shell: bash
-    if: ${{ inputs.cloudProvider == 'azure' }}
-  - name: Read Coordinator IP (GCP)
-    run: |
-      echo CONSTELL_IP=$(jq -r .gcpcoordinators[].PublicIP constellation-state.json) >> $GITHUB_ENV
-    shell: bash
-    if: ${{ inputs.cloudProvider == 'gcp' }}
-
   - name: Constellation init
     run: |
       if [ ${{ inputs.autoscale }} = true ]; then autoscale=--autoscale; fi
       constellation init ${autoscale}
     shell: bash
 
-  - name: Fetch PCRs
-    run: |
-      pcr-reader --constell-ip ${{ env.CONSTELL_IP }} -o measurements.go
-    shell: bash
-  - name: Upload measurements
-    uses: actions/upload-artifact@v3
-    with:
-      name: measurements.go
-      path: measurements.go
-    if: ${{ !env.ACT }}
-
   - name: Configure VPN connection
     run: wg-quick up ./wg0.conf
     shell: bash

.github/actions/constellation_destroy/action.yml

@@ -1,4 +1,4 @@
-name: constellation_destroy
+name: Constellation destroy
 description: "Destroy a running Constellation cluster."
 runs:
   using: 'composite'

.github/actions/constellation_measure/action.yml

@@ -0,0 +1,114 @@
+name: Constellation measure
+description: |
+  Create measurements of a Constellation cluster and print to stdout.
+  Optionally sign and/or upload to S3, if corresponding inputs are provided.
+inputs:
+  cloudProvider:
+    description: "Either 'gcp' or 'azure'."
+    required: true
+  cosignPublicKey:
+    description: 'Cosign public key'
+    required: false
+    default: ''
+  cosignPrivateKey:
+    description: 'Cosign private key'
+    required: false
+    default: ''
+  cosignPassword:
+    description: 'Password for Cosign private key'
+    required: false
+    default: ''
+  awsAccessKeyID:
+    description: 'AWS access key ID to upload measurements'
+    required: false
+    default: ''
+  awsSecretAccessKey:
+    description: 'AWS secret access key to upload measurements'
+    required: false
+    default: ''
+  awsDefaultRegion:
+    description: 'AWS region of S3 bucket to upload measurements'
+    required: false
+    default: ''
+  awsBucketName:
+    description: 'S3 bucket name to upload measurements to'
+    required: false
+    default: ''
+runs:
+  using: 'composite'
+  steps:
+    # Check /docs/secure_software_distribution.md#sign-measurements
+    # for why we ignore certain measurement values.
+    - name: Fetch PCRs
+      run: |
+        case $CSP in
+          azure)
+            FIRST_NODE=$(jq -r ".azurecoordinators | keys | first" constellation-state.json)
+            CONSTELL_IP=$(jq -r ".azurecoordinators.\"${FIRST_NODE}\".PublicIP" constellation-state.json)
+            pcr-reader --constell-ip ${CONSTELL_IP} -format yaml > measurements.yaml
+            yq e 'del(.[0,6,10,11,12,13,14,15,16,17,18,19,20,21,22,23])' -i measurements.yaml
+            ;;
+          gcp)
+            FIRST_NODE=$(jq -r ".gcpcoordinators | keys | first" constellation-state.json)
+            CONSTELL_IP=$(jq -r ".gcpcoordinators.\"${FIRST_NODE}\".PublicIP" constellation-state.json)
+            pcr-reader --constell-ip ${CONSTELL_IP} -format yaml > measurements.yaml
+            yq e 'del(.[11,12,13,14,15,16,17,18,19,20,21,22,23])' -i measurements.yaml
+            ;;
+        esac
+        cat measurements.yaml
+      shell: bash
+      env:
+        CSP: ${{ inputs.cloudProvider }}
+
+    # TODO: Replace with https://github.com/sigstore/sigstore-installer/tree/initial
+    # once it has the functionality
+    - name: Install Cosign
+      uses: sigstore/cosign-installer@main
+      if: ${{ inputs.cosignPublicKey != '' && inputs.cosignPrivateKey != '' && inputs.cosignPassword != '' }}
+    - name: Install Rekor
+      run: |
+        curl -LO https://github.com/sigstore/rekor/releases/download/v0.9.0/rekor-cli-linux-amd64
+        sudo install rekor-cli-linux-amd64 /usr/local/bin/rekor-cli
+      shell: bash
+      if: ${{ inputs.cosignPublicKey != '' && inputs.cosignPrivateKey != '' && inputs.cosignPassword != '' }}
+    - name: Sign measurements
+      run: |
+        set -e
+        set -o pipefail
+        echo "$COSIGN_PUBLIC_KEY" > cosign.pub
+        # Enabling experimental mode also publishes signature to Rekor
+        COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY measurements.yaml > measurements.yaml.sig
+        # Verify - As documentation & check
+        # Local Signature (input: artifact, key, signature)
+        cosign verify-blob --key cosign.pub --signature measurements.yaml.sig measurements.yaml
+        # Transparency Log Signature (input: artifact, key)
+        uuid=$(rekor-cli search --artifact measurements.yaml | tail -n 1)
+        sig=$(rekor-cli get --uuid=$uuid --format=json | jq -r .Body.HashedRekordObj.signature.content)
+        cosign verify-blob --key cosign.pub --signature <(echo $sig) measurements.yaml
+      shell: bash
+      env:
+        COSIGN_PUBLIC_KEY: ${{ inputs.cosignPublicKey }}
+        COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }}
+        COSIGN_PASSWORD: ${{ inputs.cosignPassword }}
+      if: ${{ inputs.cosignPublicKey != '' && inputs.cosignPrivateKey != '' && inputs.cosignPassword != '' }}
+
+    - name: Install AWS CLI
+      run: sudo apt-get update && sudo apt-get -y install awscli
+      shell: bash
+      if: ${{ inputs.awsAccessKeyID != '' && inputs.awsSecretAccessKey != '' && inputs.awsDefaultRegion != '' && inputs.awsBucketName != '' }}
+    - name: Upload to S3
+      run: |
+        IMAGE=$(yq e ".provider.${CSP}.image" constellation-conf.yaml)
+        S3_PATH=s3://${PUBLIC_BUCKET_NAME}/${IMAGE}
+        aws s3 cp measurements.yaml ${S3_PATH}/measurements.yaml
+        if test -f measurements.yaml.sig; then
+          aws s3 cp measurements.yaml.sig ${S3_PATH}/measurements.yaml.sig
+        fi
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.awsAccessKeyID }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.awsSecretAccessKey }}
+        AWS_DEFAULT_REGION: ${{ inputs.awsDefaultRegion }}
+        PUBLIC_BUCKET_NAME: ${{ inputs.awsBucketName }}
+        CSP: ${{ inputs.cloudProvider }}
+      if: ${{ inputs.awsAccessKeyID != '' && inputs.awsSecretAccessKey != '' && inputs.awsDefaultRegion != '' && inputs.awsBucketName != '' }}

.github/actions/e2e_test/action.yml

@@ -1,4 +1,4 @@
-name: e2e_test
+name: E2E test
 description: "Run Constellation e2e test."
 inputs:
   workerNodesCount:
@@ -26,8 +26,29 @@ inputs:
     description: 'Which tests should be run? Check README for guidance!'
     required: true
   msTeamsWebhook:
-    description: 'WebHook used to notify of failure'
+    description: 'WebHook used to notify of failure.'
     required: true
+  cosignPublicKey:
+    description: 'Cosign public key to sign measurements.'
+    required: false
+  cosignPrivateKey:
+    description: 'Cosign private key to sign measurements.'
+    required: false
+  cosignPassword:
+    description: 'Cosign password for private key.'
+    required: false
+  awsAccessKeyID:
+    description: 'AWS access key ID to upload measurements.'
+    required: false
+  awsSecretAccessKey:
+    description: 'AWS secrets access key to upload measurements.'
+    required: false
+  awsDefaultRegion:
+    description: 'AWS region of S3 bucket. to upload measurements.'
+    required: false
+  awsBucketName:
+    description: 'AWS S3 bucket name to upload measurements.'
+    required: false
 runs:
   using: 'composite'
   steps:
@@ -53,6 +74,17 @@ runs:
       workerNodesCount: ${{ inputs.workerNodesCount }}
       controlNodesCount: ${{ inputs.controlNodesCount }}
       machineType: ${{ inputs.machineType }}
+  - name: Measure cluster
+    uses: ./.github/actions/constellation_measure
+    with:
+      cloudProvider: ${{ inputs.cloudProvider }}
+      cosignPublicKey: ${{ inputs.cosignPublicKey }}
+      cosignPrivateKey: ${{ inputs.cosignPrivateKey }}
+      cosignPassword: ${{ inputs.cosignPassword }}
+      awsAccessKeyID: ${{ inputs.awsAccessKeyID }}
+      awsSecretAccessKey: ${{ inputs.awsSecretAccessKey }}
+      awsDefaultRegion: ${{ inputs.awsDefaultRegion }}
+      awsBucketName: ${{ inputs.awsBucketName }}
   - name: Run e2e tests
     uses: ./.github/actions/sonobuoy
     with:

.github/actions/gcp_login/action.yml

@@ -1,4 +1,4 @@
-name: gcp_login
+name: GCP login
 description: "Login to GCP & configure gcloud CLI."
 inputs:
   gcp_service_account_json:

.github/workflows/build-access-manager-image.yml

@@ -30,7 +30,7 @@ jobs:
 
       - name: Build and upload access-manager container image
         id: build-and-upload
-        uses: ./.github/actions/build_micro-service
+        uses: ./.github/actions/build_micro_service
         with:
           name: access-manager
           projectVersion: "0.0.0"

.github/workflows/build-activation-image.yml

@@ -32,7 +32,7 @@ jobs:
 
       - name: Build and upload activation-service container image
         id: build-and-upload
-        uses: ./.github/actions/build_micro-service
+        uses: ./.github/actions/build_micro_service
         with:
           name: activation-service
           projectVersion: "0.0.0"

.github/workflows/build-cli.yml

@@ -24,6 +24,6 @@ jobs:
       - name: Build cli
         uses: ./.github/actions/build_cli
         with:
-          cosign-public-key: ${{ secrets.COSIGN_PUBLIC_KEY }}
-          cosign-private-key: ${{ secrets.COSIGN_PRIVATE_KEY }}
-          cosign-password: ${{ secrets.COSIGN_PASSWORD }}
+          cosignPublicKey: ${{ secrets.COSIGN_PUBLIC_KEY }}
+          cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
+          cosignPassword: ${{ secrets.COSIGN_PASSWORD }}

.github/workflows/build-kms-image.yml

@@ -31,7 +31,7 @@ jobs:
 
       - name: Build and upload KMS server container image
         id: build-and-upload
-        uses: ./.github/actions/build_micro-service
+        uses: ./.github/actions/build_micro_service
         with:
           name: kmsserver
           projectVersion: '0.0.0'

.github/workflows/build-micro-service-manual.yml

@@ -50,7 +50,7 @@ jobs:
 
       - name: Build and upload activation-service container image
         id: build-and-upload
-        uses: ./.github/actions/build_micro-service
+        uses: ./.github/actions/build_micro_service
         with:
           name: ${{ inputs.microService }}
           projectVersion: ${{ inputs.version }}

.github/workflows/build-verification-service.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Build and upload verification-service container image
         id: build-and-upload
-        uses: ./.github/actions/build_micro-service
+        uses: ./.github/actions/build_micro_service
         with:
           name: verification-service
           projectVersion: '0.0.0'

.github/workflows/e2e-test-azure.yml

@@ -27,3 +27,10 @@ jobs:
         # TODO: Remove E2E_SKIP once AB#2174 is resolved
         sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
         msTeamsWebhook: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}
+        cosignPublicKey: ${{ secrets.COSIGN_PUBLIC_KEY }}
+        cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
+        cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+        awsAccessKeyID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        awsSecretAccessKey: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        awsDefaultRegion: ${{ secrets.AWS_DEFAULT_REGION }}
+        awsBucketName: ${{ secrets.PUBLIC_BUCKET_NAME }}

.github/workflows/e2e-test-gcp.yml

@@ -27,3 +27,10 @@ jobs:
         # TODO: Remove E2E_SKIP once AB#2174 is resolved
         sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
         msTeamsWebhook: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}
+        cosignPublicKey: ${{ secrets.COSIGN_PUBLIC_KEY }}
+        cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }}
+        cosignPassword: ${{ secrets.COSIGN_PASSWORD }}
+        awsAccessKeyID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        awsSecretAccessKey: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        awsDefaultRegion: ${{ secrets.AWS_DEFAULT_REGION }}
+        awsBucketName: ${{ secrets.PUBLIC_BUCKET_NAME }}