OS images: use "ref", "stream" and "version"

Switch azure default region to west us
Update find-image script to work with new API spec
Add version for every os image build
generate measurements: Use new API paths
CLI: config fetch measurements: Use image short versions to fetch measurements
CLI: allows shortnames to specify image in config
Image build pipeline: Change paths to contain "ref" and "stream"

.github/actions/constellation_create/action.yml

@@ -73,7 +73,7 @@ runs:
         yq eval -i \
           "(.provider | select(. | has(\"azure\")).azure.subscription) = \"${{ inputs.azureSubscription }}\" |
             (.provider | select(. | has(\"azure\")).azure.tenant) = \"${{ inputs.azureTenant }}\" |
-            (.provider | select(. | has(\"azure\")).azure.location) = \"North Europe\" |
+            (.provider | select(. | has(\"azure\")).azure.location) = \"West US\" |
             (.provider | select(. | has(\"azure\")).azure.userAssignedIdentity) = \"${{ inputs.azureUserAssignedIdentity }}\" |
             (.provider | select(. | has(\"azure\")).azure.resourceGroup) = \"${{ inputs.azureResourceGroup }}\" |
             (.provider | select(. | has(\"azure\")).azure.appClientID) = \"${{ inputs.azureClientID }}\" |
@@ -94,7 +94,7 @@ runs:
             (.provider | select(. | has(\"aws\")).aws.iamProfileWorkerNodes) = \"e2e_test_worker_node_instance_profile\"" \
           constellation-conf.yaml
 
-        if [ ${{ inputs.kubernetesVersion != '' }} = true ]; then
+        if [[ ${{ inputs.kubernetesVersion != '' }} = true ]]; then
           yq eval -i "(.kubernetesVersion) = ${{ inputs.kubernetesVersion }}" constellation-conf.yaml
         fi
 
@@ -187,13 +187,13 @@ runs:
         echo "::group::Wait for nodes"
         NODES_COUNT=$((${{ inputs.controlNodesCount }} + ${{ inputs.workerNodesCount }}))
         JOINWAIT=0
-        until [ "$(kubectl get nodes -o json | jq '.items | length')" == "${NODES_COUNT}" ] || [ $JOINWAIT -gt $JOINTIMEOUT ];
+        until [[ "$(kubectl get nodes -o json | jq '.items | length')" == "${NODES_COUNT}" ]] || [[ $JOINWAIT -gt $JOINTIMEOUT ]];
         do
             echo "$(kubectl get nodes -o json | jq '.items | length')/"${NODES_COUNT}" nodes have joined.. waiting.."
             JOINWAIT=$((JOINWAIT+30))
             sleep 30
         done
-        if [ $JOINWAIT -gt $JOINTIMEOUT ]; then
+        if [[ $JOINWAIT -gt $JOINTIMEOUT ]]; then
             echo "Timed out waiting for nodes to join"
             exit 1
         fi

.github/actions/constellation_destroy/action.yml

@@ -22,18 +22,18 @@ runs:
 
         # Scrap namespaces that contain PVCs
         for namespace in `kubectl get namespace --no-headers=true -o custom-columns=":metadata.name"`; do
-          if [ `kubectl get pvc -n $namespace --no-headers=true -o custom-columns=":metadata.name" | wc -l` -gt 0 ]; then
+          if [[ `kubectl get pvc -n $namespace --no-headers=true -o custom-columns=":metadata.name" | wc -l` -gt 0 ]]; then
             kubectl delete namespace $namespace --wait
           fi
         done
 
-        until [ "$(kubectl get pv -o json | jq '.items | length')" == "0" ] || [ $ELAPSED -gt $PV_DELETION_TIMEOUT ];
+        until [[ "$(kubectl get pv -o json | jq '.items | length')" == "0" ]] || [[ $ELAPSED -gt $PV_DELETION_TIMEOUT ]];
         do
           echo $(kubectl get pv -o json | jq '.items | length') PV remaining..
           sleep 1
           ELAPSED=$((ELAPSED+1))
         done
-        if [ $ELAPSED -gt $PV_DELETION_TIMEOUT ]; then
+        if [[ $ELAPSED -gt $PV_DELETION_TIMEOUT ]]; then
           echo "Timed out waiting for PV deletion.."
           exit 1
         fi

.github/actions/find_latest_image/action.yaml

@@ -2,13 +2,18 @@ name: Find latest OS image
 description: Finds the latest OS image of a given type.
 
 inputs:
-  imageType:
+  ref:
-    description: "Type of image to find. Can be one of [debug, release] or a custom prefix (branch name)."
+    description: 'Branch to search on. Can be "-" for releases or a branch name.'
     required: true
+    default: "main"
+  stream:
+    description: 'Type of image to find. Can be one of "stable", "nightly", "debug".'
+    required: true
+    default: "debug"
 
 outputs:
   image:
-    description: "The latest image of the given type."
+    description: "The latest image of the given ref and stream."
     value: ${{ steps.find-latest-image.outputs.image }}
 
 runs:
@@ -20,31 +25,13 @@ runs:
         role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationImageFinder
         aws-region: eu-central-1
 
-    - name: Set search prefix
-      id: set-search-prefix
-      shell: bash
-      env:
-        image_type: ${{ inputs.imageType }}
-      run: |
-        if [[ "${image_type}" == "debug" ]]; then
-          echo "prefix=debug-v" >> "${GITHUB_OUTPUT}"
-        elif [[ "${image_type}" == "release" ]]; then
-          echo "prefix=v" >> "${GITHUB_OUTPUT}"
-        else
-          echo "prefix=${image_type}" >> "${GITHUB_OUTPUT}"
-        fi
-
     - name: Find latest image
       id: find-latest-image
       shell: bash
       env:
-        bucket: cdn-constellation-backend
+        ref: ${{ inputs.ref }}
-        prefix: constellation/v1/images/${{ steps.set-search-prefix.outputs.prefix }}
+        stream: ${{ inputs.stream }}
       run: |
-        newest_debug_image_path=$(aws s3api list-objects-v2 \
+        image=$(./find-image.sh --ref "${ref}" --stream "${stream}")
-          --output text \
-          --bucket "${bucket}" \
-          --prefix "${prefix}" \
-          --query "reverse(sort_by(Contents, &LastModified))[0].Key")
-        image=$(basename "${newest_debug_image_path}" .json)
         echo "image=${image}" >> "${GITHUB_OUTPUT}"
+      working-directory: hack/find-image

.github/actions/os_build_variables/action.yml

@@ -10,8 +10,14 @@ inputs:
   basePath:
     description: "Base path to the image build directory"
     required: true
+  ref:
+    description: "Branch of the image to be built (or '-' for releases)"
+    required: true
+  stream:
+    description: "Image stream / type. (Use 'stable' for releases, 'nightly' for regular non-release images and 'debug' for debug builds)"
+    required: true
   imageVersion:
-    description: "Semantic version including patch e.g. v<major>.<minor>.<patch> (only used for releases)"
+    description: "Semantic version including patch e.g. v<major>.<minor>.<patch> or pseudo version"
     required: false
   imageType:
     description: "Type of image to build"
@@ -145,15 +151,16 @@ runs:
 
     - name: Configure AWS input variables
       id: aws
-      if: ${{ inputs.csp == 'aws' }}
+      if: inputs.csp == 'aws'
       shell: bash
       env:
         basePath: ${{ inputs.basePath }}
+        ref: ${{ inputs.ref }}
+        stream: ${{ inputs.stream }}
         imageVersion: ${{ inputs.imageVersion }}
         imageType: ${{ inputs.imageType }}
         timestamp: ${{ steps.version.outputs.timestamp }}
         semver: ${{ steps.version.outputs.semanticVersion }}
-        branchName: ${{ steps.version.outputs.branchName }}
       run: |
         echo "region=eu-central-1" >> $GITHUB_OUTPUT
         echo "replicationRegions=us-east-2 ap-south-1" >> $GITHUB_OUTPUT
@@ -162,16 +169,12 @@ runs:
         echo "imagePath=${basePath}/mkosi.output.aws/fedora~37/image.raw" >> $GITHUB_OUTPUT
         echo "jsonOutput=${basePath}/mkosi.output.aws/fedora~37/image-upload.json" >> $GITHUB_OUTPUT
         echo "imageFilename=image-$(date +%s).raw" >> $GITHUB_OUTPUT
-        if [ "${imageType}" = release ]
+        if [[ "${stream}" = "stable" ]]
         then
           echo "imageName=constellation-${imageVersion}" >> $GITHUB_OUTPUT
           echo "publish=true" >> $GITHUB_OUTPUT
-        elif [ "${imageType}" = debug ]
-        then
-          echo "imageName=constellation-debug-${semver}-${timestamp}" >> $GITHUB_OUTPUT
-          echo "publish=false" >> $GITHUB_OUTPUT
         else
-          echo "imageName=constellation-${branchName}-${timestamp}" >> $GITHUB_OUTPUT
+          echo "imageName=constellation-${ref}-${stream}-${semver}-${timestamp}" >> $GITHUB_OUTPUT
           echo "publish=false" >> $GITHUB_OUTPUT
         fi
 
@@ -180,10 +183,12 @@ runs:
     #    image version has to be semantic version in the form <uint>.<uint>.<uint> . uint may not be larger than 2,147,483,647
     - name: Configure Azure input variables
       id: azure
-      if: ${{ inputs.csp == 'azure' }}
+      if: inputs.csp == 'azure'
       shell: bash
       env:
         basePath: ${{ inputs.basePath }}
+        ref: ${{ inputs.ref }}
+        stream: ${{ inputs.stream }}
         imageVersion: ${{ inputs.imageVersion }}
         imageType: ${{ inputs.imageType }}
         timestamp: ${{ steps.version.outputs.timestamp }}
@@ -203,31 +208,31 @@ runs:
         echo "jsonOutput=${basePath}/mkosi.output.azure/fedora~37/image-upload${uploadVariant}.json" >> $GITHUB_OUTPUT
         # TODO: set default security type to "ConfidentialVM" once replication is possible
         securityType=${{ inputs.uploadVariant }}
-        if [ -z "${securityType}" ]; then
+        if [[ -z "${securityType}" ]]; then
           securityType=ConfidentialVMSupported
         fi
         echo "securityType=${securityType}" >> $GITHUB_OUTPUT
-        echo "diskName=constellation-${pseudover//./-}-${securityType,,}" >> $GITHUB_OUTPUT
+        echo "diskName=constellation-${stream}-${timestamp}-${securityType,,}" >> $GITHUB_OUTPUT
-        if [ "${imageType}" = release ]
+        if [[ "${stream}" = "release" ]]
         then
           echo "imageDefinition=constellation" >> $GITHUB_OUTPUT
           echo "imageOffer=constellation" >> $GITHUB_OUTPUT
           echo "imageVersion=${imageVersion:1}" >> $GITHUB_OUTPUT
           galleryName=Constellation
-        elif [ "${imageType}" = debug ]
+        elif [[ "${imageType}" = "debug" ]] && [[ [[ "${ref}" = "-" ] || [[ "${ref}" = "main" ]] ]]
         then
           echo "imageDefinition=${semver}" >> $GITHUB_OUTPUT
           echo "imageOffer=${semver}" >> $GITHUB_OUTPUT
           echo "imageVersion=${timestamp:0:4}.${timestamp:4:4}.${timestamp:8}" >> $GITHUB_OUTPUT
           galleryName=Constellation_Debug
         else
-          echo "imageDefinition=${branchName}" >> $GITHUB_OUTPUT
+          echo "imageDefinition=${ref}-${stream}" >> $GITHUB_OUTPUT
-          echo "imageOffer=${branchName}" >> $GITHUB_OUTPUT
+          echo "imageOffer=${ref}-${stream}" >> $GITHUB_OUTPUT
           echo "imageVersion=${timestamp:0:4}.${timestamp:4:4}.${timestamp:8}" >> $GITHUB_OUTPUT
           galleryName=Constellation_Testing
         fi
         # TODO: enable VMGS upload for ConfidentialVM images once replication is possible
-        if [ "${securityType}" == "ConfidentialVMSupported" ]; then
+        if [[ "${securityType}" == "ConfidentialVMSupported" ]]; then
           echo "galleryName=${galleryName}_CVM" >> $GITHUB_OUTPUT
           echo "vmgsPath=" >> $GITHUB_OUTPUT
         else
@@ -239,15 +244,16 @@ runs:
     #    Must not end or begin with a dash
     - name: Configure GCP input variables
       id: gcp
-      if: ${{ inputs.csp == 'gcp' }}
+      if: inputs.csp == 'gcp'
       shell: bash
       env:
         basePath: ${{ inputs.basePath }}
+        ref: ${{ inputs.ref }}
+        stream: ${{ inputs.stream }}
         imageVersion: ${{ inputs.imageVersion }}
         imageType: ${{ inputs.imageType }}
         timestamp: ${{ steps.version.outputs.timestamp }}
         semver: ${{ steps.version.outputs.semanticVersion }}
-        branchName: ${{ steps.version.outputs.branchName }}
       run: |
         echo "project=constellation-images" >> $GITHUB_OUTPUT
         echo "bucket=constellation-images" >> $GITHUB_OUTPUT
@@ -255,25 +261,25 @@ runs:
         echo "rawImagePath=${basePath}/mkosi.output.gcp/fedora~37/image.raw" >> $GITHUB_OUTPUT
         echo "imagePath=${basePath}/mkosi.output.gcp/fedora~37/image.tar.gz" >> $GITHUB_OUTPUT
         echo "jsonOutput=${basePath}/mkosi.output.gcp/fedora~37/image-upload.json" >> $GITHUB_OUTPUT
-        if [ "${imageType}" = release ]
+        if [[ "${stream}" = "stable" ]]
         then
           echo "imageName=constellation-${imageVersion//./-}" >> $GITHUB_OUTPUT
           echo "imageFilename=constellation-${imageVersion//./-}.tar.gz" >> $GITHUB_OUTPUT
           echo "imageFamily=constellation" >> $GITHUB_OUTPUT
-        elif [ "${imageType}" = debug ]
+        elif [[ "${imageType}" = "debug" ]] && [[ [[ "${ref}" = "-" ]] || [[ "${ref}" = "main" ]] ]]
         then
-          echo "imageName=constellation-${timestamp}" >> $GITHUB_OUTPUT
+          echo "imageName=constellation-${ref}-${stream}-${timestamp}" >> $GITHUB_OUTPUT
-          echo "imageFilename=constellation-${timestamp}.tar.gz" >> $GITHUB_OUTPUT
+          echo "imageFilename=constellation-${ref}-${stream}-${timestamp}.tar.gz" >> $GITHUB_OUTPUT
           echo "imageFamily=constellation-debug-${semver//./-}" >> $GITHUB_OUTPUT
         else
-          echo "imageName=constellation-${timestamp}" >> $GITHUB_OUTPUT
+          echo "imageName=constellation-${ref}-${stream}-${timestamp}" >> $GITHUB_OUTPUT
-          echo "imageFilename=constellation-${timestamp}.tar.gz" >> $GITHUB_OUTPUT
+          echo "imageFilename=constellation-${ref}-${stream}-${timestamp}.tar.gz" >> $GITHUB_OUTPUT
-          echo "imageFamily=constellation-${branchName}" >> $GITHUB_OUTPUT
+          echo "imageFamily=constellation-${ref}" >> $GITHUB_OUTPUT
         fi
 
     - name: Configure QEMU input variables
       id: qemu
-      if: ${{ inputs.csp == 'qemu' }}
+      if: inputs.csp == 'qemu'
       shell: bash
       env:
         basePath: ${{ inputs.basePath }}

.github/actions/shortname/action.yaml

@@ -0,0 +1,45 @@
+name: Determine parts for shortname
+description: "Determine ref, stream and version for a shortname."
+
+inputs:
+  shortname:
+    description: "Shortname of the image to build"
+    required: true
+
+outputs:
+  ref:
+    description: "Branch name that the resource is built from (or '-' for releases)"
+    value: ${{ steps.extract.outputs.ref }}
+  stream:
+    description: "Stream that the resource belongs to"
+    value: ${{ steps.extract.outputs.stream }}
+  version:
+    description: "Resource version"
+    value: ${{ steps.extract.outputs.version }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Extract ref, stream and version
+      shell: bash
+      id: extract
+      run: |
+        ref="-"
+        stream="stable"
+        version=""
+        IFS="/" read -r -a string_array <<< "${{ inputs.shortname }}"
+        for ((i=0; i<${#string_array[@]}; i++)); do
+          echo "${string_array[i]}"
+          if [[ ${string_array[i]} == "ref" ]]; then
+            ref=${string_array[i+1]}
+          elif [[ ${string_array[i]} == "stream" ]]; then
+            stream=${string_array[i+1]}
+          else
+            version=${string_array[i]}
+          fi
+        done
+        {
+          echo "ref=$ref"
+          echo "stream=$stream"
+          echo "version=$version"
+        }  | tee "$GITHUB_OUTPUT" "$GITHUB_ENV"

.github/docs/README.md

@@ -104,48 +104,3 @@ The following secrets need to be defined:
 * `AZURE_E2E_CLIENT_SECRET`: The client secret value for the registered app on Azure (which is defined as `appClientID`).
 
 For information on how to achieve this, refer to the [First steps](https://docs.edgeless.systems/constellation/getting-started/first-steps) in the documentation for Constellation.
-
-## Image versions
-
-The [build-os-image](../workflows/build-os-image.yml) workflow can be used to trigger an image build.
-
-The workflow can be used to build debug or release images.
-A debug image uses [`debugd`](../../debugd/) as its bootstrapper binary, while release images use the actual [`bootstrapper`](../../bootstrapper/)
-Workflows for the main branch will always build debug images.
-
-The image will be named and categorized depending on the branch the build is triggered from.
-In the following, __Release__ refers to non debug images build from a release branch, e.g. `release/v1.4.0`,
-__Debug__ refers to debug images build from either main or a release branch,
-and __Branch__ refers to any image build from a branch that is not main or a release branch.
-Non debug images built from main follow the __Branch__ image naming scheme.
-
-### GCP
-
-Type | Image Family | Image Name
--|-|-
-Release | constellation | constellation-v\<major\>-\<minor\>-\<patch\>
-Debug | constellation-debug-v\<major\>-\<minor\>-\<patch\> | constellation-\<commit-timestamp\>
-Branch | constellation-\<branch-name\> | constellation-\<commit-timestamp\>
-
-Example:
-Type | Image Family | Image Name | List command
--|-|-|-
-Release | constellation | constellation-v1-5-0 | `gcloud compute images list --filter="family~'^constellation$'" --sort-by=creationTimestamp --project constellation-images --uri \| sed 's#https://www.googleapis.com/compute/v1/##'`
-Debug | constellation-debug-v1-5-0 | constellation-20220912123456 | `gcloud compute images list --filter="family~'constellation-debug-v.+'" --sort-by=creationTimestamp --project constellation-images --uri \| sed 's#https://www.googleapis.com/compute/v1/##'`
-Branch | constellation-ref-cli | constellation-20220912123456 | `gcloud compute images list --filter="family~'constellation-$(go run $(git rev-parse --show-toplevel)/hack/pseudo-version/pseudo-version.go -print-branch)'" --sort-by=creationTimestamp --project constellation-images --uri \| sed 's#https://www.googleapis.com/compute/v1/##'`
-
-### Azure
-
-Type | Gallery | Image Definition | Image Version
--|-|-|-
-Release | Constellation | constellation | \<major\>.\<minor\>.\<patch\>
-Debug | Constellation_Debug | v\<major\>.\<minor\>.\<patch\> | \<commit-timestamp\>
-Branch | Constellation_Testing | \<branch-name\> | \<commit-timestamp\>
-
-Example:
-
-Type | Gallery | Image Definition | Image Version | List command | Community list command
--|-|-|-|-|-
-Release | Constellation | constellation | 1.5.0 | `az sig image-version list --resource-group constellation-images --gallery-name Constellation_CVM --gallery-image-definition constellation --query "sort_by([], &publishingProfile.publishedDate)[].id" -o table` | `az sig image-version list-community --public-gallery-name ConstellationCVM-b3782fa0-0df7-4f2f-963e-fc7fc42663df --gallery-image-definition constellation --location northeurope`
-Debug | Constellation_Debug | v1.5.0 | 2022.0912.123456 | `az sig image-version list --resource-group constellation-images --gallery-name Constellation_Debug_CVM --gallery-image-definition v1.5.0 --query "sort_by([], &publishingProfile.publishedDate)[].id" -o table` | `az sig image-version list-community --public-gallery-name ConstellationCVM-d1905bb0-a66c-497e-a9e6-4410ca7e3701 --gallery-image-definition v1.5.0 --location northeurope`
-Branch | Constellation_Testing | ref-cli | 2022.0912.123456 | `az sig image-version list --resource-group constellation-images --gallery-name Constellation_Testing_CVM --gallery-image-definition $(go run $(git rev-parse --show-toplevel)/hack/pseudo-version/pseudo-version.go -print-branch) --query "sort_by([], &publishingProfile.publishedDate)[].id" -o table` | `az sig image-version list-community --public-gallery-name ConstellationCVM-d1905bb0-a66c-497e-a9e6-4410ca7e3701 --gallery-image-definition $(go run $(git rev-parse --show-toplevel)/hack/pseudo-version/pseudo-version.go -print-branch) --location northeurope`

.github/workflows/build-os-image.yml

@@ -5,11 +5,21 @@ on:
       imageVersion:
         description: "Semantic version including patch e.g. v<major>.<minor>.<patch> (only used for releases)"
         required: false
-      debug:
+      isRelease:
-        description: "Build debug image"
+        description: 'Is this a release? (sets "ref" to special value "-")'
         type: boolean
-        default: false
         required: false
+        default: false
+      stream:
+        description: "Image stream / type. (Use 'stable' for releases, 'nightly' for regular non-release images and 'debug' for debug builds)"
+        type: choice
+        required: true
+        options:
+          - "stable"
+          - "nightly"
+          - "debug"
+          # TODO: implement console access enabled image
+          # - "console"
 
 jobs:
   build-dependencies:
@@ -28,13 +38,13 @@ jobs:
           ref: ${{ github.head_ref }}
 
       - name: Build bootstrapper
-        if: ${{ inputs.debug == false }}
+        if: inputs.stream != 'debug'
         uses: ./.github/actions/build_bootstrapper
         with:
           outputPath: ${{ github.workspace }}/build/bootstrapper
 
       - name: Build debugd
-        if: ${{ inputs.debug == true }}
+        if: inputs.stream == 'debug'
         uses: ./.github/actions/build_debugd
         with:
           outputPath: ${{ github.workspace }}/build/bootstrapper
@@ -65,9 +75,13 @@ jobs:
     name: "Determine build settings"
     runs-on: ubuntu-22.04
     outputs:
+      ref: ${{ steps.ref.outputs.ref }}
       imageType: ${{ steps.image-type.outputs.imageType }}
       pkiSet: ${{ steps.pki-set.outputs.pkiSet }}
-      imageVersionUid: ${{ steps.image-version-uid.outputs.imageVersionUid }}
+      imageVersion: ${{ steps.image-version.outputs.imageVersion }}
+      imageName: ${{ steps.image-version.outputs.imageName }}
+      imageNameShort: ${{ steps.image-version.outputs.imageNameShort }}
+      imageApiBasePath: ${{ steps.image-version.outputs.imageApiBasePath }}
 
     steps:
       - name: Checkout
@@ -79,41 +93,65 @@ jobs:
         id: version
         uses: ./.github/actions/pseudo_version
 
+      - name: Determine ref
+        id: ref
+        run: |
+          if [[ "${{ inputs.isRelease }}" = "true" ]]; then
+            echo "ref=-" >> "$GITHUB_OUTPUT"
+          else
+            echo "ref=${{ steps.version.outputs.branchName }}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Validate stream
+        run: |
+          if [[ "${{ inputs.isRelease }}" == "true" ]] && [[ "${{ inputs.stream }}" == "nightly" ]]; then
+            echo "Nightly builds are not allowed for releases"
+            exit 1
+          elif [[ "${{ inputs.isRelease }}" != "true" ]] && [[ "${{ inputs.stream }}" == "stable" ]]; then
+            echo "Stable builds are only allowed for releases"
+            exit 1
+          fi
+
       - name: Determine type of image build
         shell: bash
         id: image-type
         run: |
-          if [ "${{ startsWith(github.ref, 'refs/heads/release/') && (inputs.debug == false) }}" = true ]
+          if  [[ "${{ inputs.stream }}" == "debug" ]]; then
-          then
-            echo "imageType=release" >> "$GITHUB_OUTPUT"
-          elif [ "${{ ((github.ref == 'refs/heads/main') || startsWith(github.ref, 'refs/heads/release/')) && (inputs.debug == true) }}" = true ]
-          then
             echo "imageType=debug" >> "$GITHUB_OUTPUT"
           else
-            echo "imageType=branch" >> "$GITHUB_OUTPUT"
+            echo "imageType=default" >> "$GITHUB_OUTPUT"
           fi
 
       - name: Determine PKI set
         id: pki-set
         shell: bash
         run: |
-          if [ "${{ steps.image-type.outputs.imageType }}" = "release" ]
+          if [[ "${{ inputs.isRelease }}" != "true" ]] && [[ "${{ inputs.stream }}" == "stable" ]]; then
-          then
+             echo "pkiSet=pki_prod" >> "$GITHUB_OUTPUT"
-            echo "pkiSet=pki_prod" >> "$GITHUB_OUTPUT"
+           else
-          else
+             echo "pkiSet=pki_testing" >> "$GITHUB_OUTPUT"
-            echo "pkiSet=pki_testing" >> "$GITHUB_OUTPUT"
+           fi
-          fi
 
-      - name: Determine image version uid
+      - name: Determine image version
-        id: image-version-uid
+        id: image-version
         shell: bash
+        env:
+          REF: ${{ steps.ref.outputs.ref }}
+          STREAM: ${{ inputs.stream }}
+          IMAGE_VERSION: ${{ inputs.imageVersion || steps.version.outputs.pseudoVersion }}
         run: |
-          if [ "${{ steps.image-type.outputs.imageType }}" = "release" ]; then
+          {
-            echo "imageVersionUid=${{ github.event.inputs.imageVersion }}" >> "$GITHUB_OUTPUT"
+            echo "imageVersion=${IMAGE_VERSION}"
-          elif [ "${{ steps.image-type.outputs.imageType }}" = "debug" ]; then
+            echo "imageName=ref/${REF}/stream/${STREAM}/${IMAGE_VERSION}"
-            echo "imageVersionUid=debug-${{ steps.version.outputs.pseudoVersion }}" >> "$GITHUB_OUTPUT"
+            echo "imageApiBasePath=constellation/v1/ref/${REF}/stream/${STREAM}/image/${IMAGE_VERSION}"
+          } >> "$GITHUB_OUTPUT"
+
+          if [[ "${REF}" = "-" ]] && [[ "${stream}" = "stable" ]]; then
+            echo "imageNameShort=${IMAGE_VERSION}" >> "$GITHUB_OUTPUT"
+          elif [[ "${REF}" = "-" ]]; then
+            echo "imageNameShort=stream/${STREAM}/${IMAGE_VERSION}" >> "$GITHUB_OUTPUT"
           else
-            echo "imageVersionUid=${{ steps.version.outputs.branchName }}-${{ steps.version.outputs.pseudoVersion }}" >> "$GITHUB_OUTPUT"
+            echo "imageNameShort=ref/${REF}/stream/${STREAM}/${IMAGE_VERSION}" >> "$GITHUB_OUTPUT"
           fi
 
   make-os-image:
@@ -202,7 +240,7 @@ jobs:
         env:
           BOOTSTRAPPER_BINARY: ${{ github.workspace }}/build/bootstrapper
           DISK_MAPPER_BINARY: ${{ github.workspace }}/build/disk-mapper
-          IMAGE_VERSION: ${{ needs.build-settings.outputs.imageVersionUid }}
+          IMAGE_VERSION: ${{ needs.build-settings.outputs.imageVersion }}
           CSP: ${{ matrix.csp }}
 
       - name: Collect hashes
@@ -288,9 +326,11 @@ jobs:
           csp: ${{ matrix.csp }}
           uploadVariant: ${{ matrix.upload-variant }}
           basePath: ${{ github.workspace }}/image
-          imageVersion: ${{ inputs.imageVersion }}
+          ref: ${{ needs.build-settings.outputs.ref }}
+          stream: ${{ inputs.stream }}
+          imageVersion: ${{ needs.build-settings.outputs.imageVersion }}
           imageType: ${{ needs.build-settings.outputs.imageType }}
-          debug: ${{ inputs.debug }}
+          debug: ${{ needs.build-settings.outputs.imageType == 'debug' }}
 
       - name: Install tools
         shell: bash
@@ -308,7 +348,7 @@ jobs:
         # on AWS, login is required to upload the image as AMI
         # on Azure, login is done to download the VMGS from S3
         # on QEMU, login is done to upload the image to S3
-        if: ${{ matrix.csp == 'aws' || matrix.csp == 'azure' || matrix.csp == 'qemu' }}
+        if: matrix.csp == 'aws' || matrix.csp == 'azure' || matrix.csp == 'qemu'
         with:
           role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline
           aws-region: eu-central-1
@@ -321,7 +361,7 @@ jobs:
 
       - name: Login to GCP
         uses: ./.github/actions/login_gcp
-        if: ${{ matrix.csp == 'gcp' }}
+        if: matrix.csp == 'gcp'
         with:
           gcp_service_account_json: ${{ secrets.GCP_IMAGE_UPLOAD_SERVICE_ACCOUNT }}
 
@@ -340,7 +380,7 @@ jobs:
             "${PKI_SET}/${AZURE_SECURITY_TYPE}.vmgs" \
             --no-progress
         working-directory: ${{ github.workspace }}/image
-        if: ${{ matrix.csp == 'azure' && !endsWith(env.AZURE_SECURITY_TYPE, 'Supported') }}
+        if: matrix.csp == 'azure' && !endsWith(env.AZURE_SECURITY_TYPE, 'Supported')
         env:
           PKI_SET: ${{ needs.build-settings.outputs.pkiSet }}
           AZURE_VMGS_REGION: ${{ steps.vars.outputs.azureVmgsRegion }}
@@ -355,7 +395,7 @@ jobs:
           echo -e "Uploaded AWS image: \n\n\`\`\`\n$(jq < "${AWS_JSON_OUTPUT}")\n\`\`\`\n" >> "$GITHUB_STEP_SUMMARY"
           echo "::endgroup::"
         working-directory: ${{ github.workspace }}/image
-        if: ${{ matrix.csp == 'aws' }}
+        if: matrix.csp == 'aws'
         env:
           PKI: ${{ github.workspace }}/image/pki
           AWS_JSON_OUTPUT: ${{ steps.vars.outputs.awsJsonOutput }}
@@ -377,7 +417,7 @@ jobs:
           echo -e "Uploaded GCP image: \n\n\`\`\`\n$(jq < "${GCP_JSON_OUTPUT}")\n\`\`\`\n" >> "$GITHUB_STEP_SUMMARY"
           echo "::endgroup::"
         working-directory: ${{ github.workspace }}/image
-        if: ${{ matrix.csp == 'gcp' }}
+        if: matrix.csp == 'gcp'
         env:
           PKI: ${{ github.workspace }}/image/pki
           GCP_JSON_OUTPUT: ${{ steps.vars.outputs.gcpJsonOutput }}
@@ -399,7 +439,7 @@ jobs:
           echo -e "Uploaded Azure ${AZURE_SECURITY_TYPE} image: \n\n\`\`\`\n$(jq < "${AZURE_JSON_OUTPUT}")\n\`\`\`\n" >> "$GITHUB_STEP_SUMMARY"
           echo "::endgroup::"
         working-directory: ${{ github.workspace }}/image
-        if: ${{ matrix.csp == 'azure' }}
+        if: matrix.csp == 'azure'
         env:
           PKI: ${{ github.workspace }}/image/pki
           AZURE_JSON_OUTPUT: ${{ steps.vars.outputs.azureJsonOutput }}
@@ -421,7 +461,7 @@ jobs:
 
       - name: Upload QEMU image
         shell: bash
-        if: ${{ matrix.csp == 'qemu' }}
+        if: matrix.csp == 'qemu'
         run: |
           echo "::group::Upload QEMU image"
           upload/upload_qemu.sh
@@ -433,7 +473,9 @@ jobs:
           QEMU_BUCKET: ${{ steps.vars.outputs.qemuBucket }}
           QEMU_BASE_URL: ${{ steps.vars.outputs.qemuBaseUrl }}
           QEMU_IMAGE_PATH: ${{ steps.vars.outputs.qemuImagePath }}
-          IMAGE_VERSION_UID: ${{ needs.build-settings.outputs.imageVersionUid }}
+          REF: ${{needs.build-settings.outputs.ref }}
+          STREAM: ${{inputs.stream }}
+          IMAGE_VERSION: ${{needs.build-settings.outputs.imageVersion }}
 
       - name: Upload image lookup table as artifact
         uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # tag=v3.1.1
@@ -501,7 +543,7 @@ jobs:
         run: |
           aws s3 cp \
             "pcrs-${{ matrix.csp }}.json" \
-            "s3://cdn-constellation-backend/constellation/v1/measurements/${{ needs.build-settings.outputs.imageVersionUid }}/${{ matrix.csp }}/measurements.image.json" \
+            "s3://cdn-constellation-backend/${{needs.build-settings.outputs.imageApiBasePath}}/csp/${{ matrix.csp }}/measurements.image.json" \
             --no-progress
 
   generate-sbom:
@@ -616,7 +658,7 @@ jobs:
           for file in ${sboms} ${manifests} ${hashes}; do
             aws s3 cp \
               "${file}" \
-              "s3://cdn-constellation-backend/constellation/v1/sbom/${{ needs.build-settings.outputs.imageVersionUid }}/${file}" \
+              "s3://cdn-constellation-backend/${{needs.build-settings.outputs.imageApiBasePath}}/${file}" \
               --no-progress
           done
 
@@ -633,15 +675,24 @@ jobs:
         with:
           name: lookup-table
 
+      - name: Login to AWS
+        uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # tag=v1.7.0
+        with:
+          role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline
+          aws-region: eu-central-1
+
       - name: Combine lookup tables for CSPs
         shell: bash
         run: |
           echo '{}' > intermediate.json
 
-          jq '.version = "${{ needs.build-settings.outputs.imageVersionUid }}"' intermediate.json > lookup-table.json
+          jq '.ref = "${{  needs.build-settings.outputs.ref }}"' intermediate.json > lookup-table.json
           cp lookup-table.json intermediate.json
 
-          jq '.debug = ${{ inputs.debug }}' intermediate.json > lookup-table.json
+          jq '.stream = "${{ inputs.stream }}"' intermediate.json > lookup-table.json
+          cp lookup-table.json intermediate.json
+
+          jq '.version = "${{ needs.build-settings.outputs.imageVersion }}"' intermediate.json > lookup-table.json
           cp lookup-table.json intermediate.json
 
           for lut in mkosi.output.*/*/image-upload*.json; do
@@ -651,25 +702,38 @@ jobs:
 
           rm -f intermediate.json
 
-          mv lookup-table.json "${{ needs.build-settings.outputs.imageVersionUid }}.json"
-
-      - name: Login to AWS
-        uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # tag=v1.7.0
-        with:
-          role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline
-          aws-region: eu-central-1
-
       - name: Upload lookup table to S3
         shell: bash
         run: |
           aws s3 cp \
-            "${{ needs.build-settings.outputs.imageVersionUid }}.json" \
+            "lookup-table.json" \
-            "s3://cdn-constellation-backend/constellation/v1/images/${{ needs.build-settings.outputs.imageVersionUid }}.json" \
+            "s3://cdn-constellation-backend/${{ needs.build-settings.outputs.imageApiBasePath }}/info.json" \
             --no-progress
           (
-            echo -e "Image version ([Lookup table](https://cdn.confidential.cloud/constellation/v1/images/${{ needs.build-settings.outputs.imageVersionUid }}.json)):"
+            echo -e "Image version ([Lookup table](https://cdn.confidential.cloud/${{ needs.build-settings.outputs.imageApiBasePath }}/info.json)):"
             echo
             echo -e "\`\`\`"
-            echo "${{ needs.build-settings.outputs.imageVersionUid }}"
+            echo "${{ needs.build-settings.outputs.imageNameShort }}"
             echo -e "\`\`\`"
           ) >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Checkout
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Setup Go environment
+        uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 # v3.4.0
+        with:
+          go-version: "1.19.4"
+          cache: true
+
+      - name: Update list of available OS image versions
+        if: needs.build-settings.outputs.ref != '-'
+        run: |
+          go run main.go \
+            --version "${{ needs.build-settings.outputs.imageVersion }}" \
+            --stream "${{ inputs.stream }}" \
+            --ref "${{ needs.build-settings.outputs.ref }}" \
+            --latest
+        working-directory: hack/add-version

.github/workflows/e2e-test-daily.yml

@@ -30,7 +30,8 @@ jobs:
         id: find-latest-image
         uses: ./.github/actions/find_latest_image
         with:
-          imageType: debug
+          ref: main
+          stream: debug
 
   e2e-daily:
     strategy:

.github/workflows/e2e-test-manual.yml

@@ -40,10 +40,10 @@ on:
         default: false
         required: false
       osImage:
-        description: "Full name of OS image (CSP independent image version UID)."
+        description: "Full name of OS image (CSP independent image version UID). Leave empty for latest debug image on main."
         type: string
-        default: "main latest"
+        default: ""
-        required: true
+        required: false
       isDebugImage:
         description: "Is OS image a debug image?"
         type: boolean
@@ -63,7 +63,7 @@ env:
 
 jobs:
   find-latest-image:
-    name: Find latest debug image
+    name: Select image
     runs-on: ubuntu-22.04
     permissions:
       id-token: write
@@ -75,7 +75,7 @@ jobs:
         id: check-input
         shell: bash
         run: |
-          if [[ "${{ inputs.osImage }}" == "main latest" ]]; then
+          if [[ -z "${{ inputs.osImage }}" ]]; then
             echo "Using latest debug image from main."
           else
             echo "Using image '${{ inputs.osImage }}'."
@@ -90,9 +90,11 @@ jobs:
 
       - name: Find latest image
         id: find-latest-image
+        if: steps.check-input.outputs.image == ''
         uses: ./.github/actions/find_latest_image
         with:
-          imageType: debug
+          ref: main
+          stream: debug
 
   e2e-test-manual:
     runs-on: ubuntu-22.04

.github/workflows/e2e-test-weekly.yml

@@ -30,7 +30,8 @@ jobs:
         id: find-latest-image
         uses: ./.github/actions/find_latest_image
         with:
-          imageType: debug
+          ref: main
+          stream: debug
 
   e2e-weekly:
     strategy:

.github/workflows/generate-measurements.yml

@@ -32,15 +32,25 @@ jobs:
       ARM_CLIENT_SECRET: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
       ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
       ARM_TENANT_ID: ${{ secrets.AZURE_E2E_TENANT_ID }}
+    outputs:
+      ref: ${{ steps.extract.outputs.ref }}
+      stream: ${{ steps.extract.outputs.stream }}
+      version: ${{ steps.extract.outputs.version }}
     steps:
       - name: Check out repository
         uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # tag=v3.1.0
         with:
           ref: ${{ github.head_ref }}
 
+      - name: Extract ref, stream and version
+        id: extract
+        uses: ./.github/actions/shortname
+        with:
+          shortname: ${{ github.event.inputs.osImage }}
+
       - name: Check if image definition from build pipeline exists
         run: |
-          wget -O /dev/null https://cdn.confidential.cloud/constellation/v1/images/${{ github.event.inputs.osImage }}.json
+          wget -O /dev/null "https://cdn.confidential.cloud/constellation/v1/ref/${{ steps.extract.outputs.ref }}/stream/${{ steps.extract.outputs.stream }}/image/${{ steps.extract.outputs.version }}/info.json"
         shell: bash
 
       - name: Setup Go environment
@@ -207,10 +217,15 @@ jobs:
 
       - name: Download expected measurements from build pipeline for image
         run: |
+          path="constellation/v1/ref/${ref}/stream/${stream}/image/${version}/csp/${{ matrix.provider }}/measurements.image.json"
           mkdir -p ${{ github.workspace }}/expected-measurements
-          wget -O ${{ github.workspace }}/expected-measurements/measurements.image.json https://cdn.confidential.cloud/constellation/v1/measurements/${{ github.event.inputs.osImage }}/${{ matrix.provider }}/measurements.image.json
+          wget -O ${{ github.workspace }}/expected-measurements/measurements.image.json "https://cdn.confidential.cloud/${path}"
           cat ${{ github.workspace }}/expected-measurements/measurements.image.json
         shell: bash
+        env:
+          ref: ${{ needs.calculate-measurements-on-csp.outputs.ref }}
+          stream: ${{ needs.calculate-measurements-on-csp.outputs.stream }}
+          version: ${{ needs.calculate-measurements-on-csp.outputs.version }}
 
       - name: Check if expected measurements == actual measurements from running cluster
         run: |
@@ -304,7 +319,7 @@ jobs:
 
       - name: Upload to S3
         run: |
-          S3_PATH=s3://cdn-constellation-backend/constellation/v1/measurements/${IMAGE_UID,,}/${{ matrix.provider }}
+          S3_PATH=s3://cdn-constellation-backend/constellation/v1/ref/${ref}/stream/${stream}/image/${version}/csp/${{ matrix.provider }}
           aws s3 cp "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json" "${S3_PATH}/measurements.json"
           aws s3 cp "${{ github.workspace }}/generated-measurements/measurements-${{ matrix.provider }}.json.sig" "${S3_PATH}/measurements.json.sig"
         shell: bash
@@ -312,3 +327,6 @@ jobs:
           IMAGE_UID: ${{ inputs.osImage }}
           PUBLIC_BUCKET_NAME: ${{ secrets.PUBLIC_BUCKET_NAME }}
           CSP: ${{ matrix.provider }}
+          ref: ${{ needs.calculate-measurements-on-csp.outputs.ref }}
+          stream: ${{ needs.calculate-measurements-on-csp.outputs.stream }}
+          version: ${{ needs.calculate-measurements-on-csp.outputs.version }}

.github/workflows/on-release.yml

@@ -8,6 +8,10 @@ on:
       tag:
         description: "Semantic version tag of the release (vX.Y.Z)."
         required: true
+      latest:
+        description: "Whether to update the latest tag."
+        type: boolean
+        default: false
 
 jobs:
   update:
@@ -35,5 +39,10 @@ jobs:
 
       - name: Update OS images
         run: |
-          go run main.go --version "${{ github.event.release.tag_name }}${{ github.event.inputs.tag }}"
+          latest=$( [[ "${{ inputs.latest }}" = "true" ]] && echo "--latest")
+          go run main.go \
+            --version "${{ github.event.release.tag_name }}${{ github.event.inputs.tag }}" \
+            --stream stable \
+            --release \
+            "${latest}"
         working-directory: hack/add-version

.github/workflows/update-cli-reference.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Check if action branch exists
         run: |
           ex="$(git ls-remote --heads origin action/constellation/update-cli-reference)"
-          echo "EXISTS=$(if [ -z "$ex" ]; then echo 0; else echo 1; fi)" >> "$GITHUB_ENV"
+          echo "EXISTS=$(if [[ -z "$ex" ]]; then echo 0; else echo 1; fi)" >> "$GITHUB_ENV"
 
       - name: Publish new reference (create new branch)
         if: ${{ env.EXISTS == 0 }}

cli/internal/cmd/configfetchmeasurements.go

@@ -20,6 +20,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/config"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/file"
+	"github.com/edgelesssys/constellation/v2/internal/shortname"
 	"github.com/edgelesssys/constellation/v2/internal/sigstore"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
@@ -163,11 +164,14 @@ func (f *fetchMeasurementsFlags) updateURLs(conf *config.Config) error {
 }
 
 func measurementURL(provider cloudprovider.Provider, image, file string) (*url.URL, error) {
+	ref, stream, version, err := shortname.ToParts(image)
+	if err != nil {
+		return nil, fmt.Errorf("parsing image name: %w", err)
+	}
 	url, err := url.Parse(constants.CDNRepositoryURL)
 	if err != nil {
 		return nil, fmt.Errorf("parsing image version repository URL: %w", err)
 	}
-	url.Path = path.Join(constants.CDNMeasurementsPath, image, strings.ToLower(provider.String()), file)
+	url.Path = path.Join(constants.CDNAPIPrefix, "ref", ref, "stream", stream, "image", version, "csp", strings.ToLower(provider.String()), file)
-
 	return url, nil
 }

cli/internal/cmd/configfetchmeasurements_test.go

@@ -109,8 +109,8 @@ func TestUpdateURLs(t *testing.T) {
 				},
 			},
 			flags:                  &fetchMeasurementsFlags{},
-			wantMeasurementsURL:    constants.CDNRepositoryURL + "/" + constants.CDNMeasurementsPath + "/someImageVersion/gcp/measurements.json",
+			wantMeasurementsURL:    constants.CDNRepositoryURL + "/" + constants.CDNAPIPrefix + "/ref/-/stream/stable/image/someImageVersion/csp/gcp/measurements.json",
-			wantMeasurementsSigURL: constants.CDNRepositoryURL + "/" + constants.CDNMeasurementsPath + "/someImageVersion/gcp/measurements.json.sig",
+			wantMeasurementsSigURL: constants.CDNRepositoryURL + "/" + constants.CDNAPIPrefix + "/ref/-/stream/stable/image/someImageVersion/csp/gcp/measurements.json.sig",
 		},
 		"both set by user": {
 			conf: &config.Config{},
@@ -185,14 +185,14 @@ func TestConfigFetchMeasurements(t *testing.T) {
 	signature := "MEYCIQDRAQNK2NjHJBGrnw3HQAyBsXMCmVCptBdgA6VZ3IlyiAIhAPG42waF1aFZq7dnjP3b2jsMNUtaKYDQQSazW1AX8jgF"
 
 	client := newTestClient(func(req *http.Request) *http.Response {
-		if req.URL.String() == "https://cdn.confidential.cloud/constellation/v1/measurements/v999.999.999/gcp/measurements.json" {
+		if req.URL.Path == "/constellation/v1/ref/-/stream/stable/image/v999.999.999/csp/gcp/measurements.json" {
 			return &http.Response{
 				StatusCode: http.StatusOK,
 				Body:       io.NopCloser(bytes.NewBufferString(measurements)),
 				Header:     make(http.Header),
 			}
 		}
-		if req.URL.String() == "https://cdn.confidential.cloud/constellation/v1/measurements/v999.999.999/gcp/measurements.json.sig" {
+		if req.URL.Path == "/constellation/v1/ref/-/stream/stable/image/v999.999.999/csp/gcp/measurements.json.sig" {
 			return &http.Response{
 				StatusCode: http.StatusOK,
 				Body:       io.NopCloser(bytes.NewBufferString(signature)),

cli/internal/cmd/upgradeplan.go

@@ -11,8 +11,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"net/url"
-	"path"
 	"strings"
 
 	"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
@@ -175,12 +173,12 @@ func getCompatibleImageMeasurements(ctx context.Context, cmd *cobra.Command, cli
 ) (map[string]config.UpgradeConfig, error) {
 	upgrades := make(map[string]config.UpgradeConfig)
 	for _, img := range images {
-		measurementsURL, err := url.Parse(constants.CDNRepositoryURL + path.Join("/", constants.CDNMeasurementsPath, img, strings.ToLower(csp.String()), "measurements.json"))
+		measurementsURL, err := measurementURL(csp, img, "measurements.json")
 		if err != nil {
 			return nil, err
 		}
 
-		signatureURL, err := url.Parse(constants.CDNRepositoryURL + path.Join("/", constants.CDNMeasurementsPath, img, strings.ToLower(csp.String()), "measurements.json.sig"))
+		signatureURL, err := measurementURL(csp, img, "measurements.json.sig")
 		if err != nil {
 			return nil, err
 		}

debugd/README.md

@@ -22,7 +22,7 @@ With `cdbg` and `yq` installed in your path:
 
 1. Run `constellation config generate` to create a new default configuration
 
-2. Locate the latest debugd images by running `hack/find-image/find-image.sh`
+2. Locate the latest debugd images by running `hack/find-image/find-image.sh --ref main --stream debug`
 
 3. Modify the `constellation-conf.yaml` to use an image with the debugd already included and add required firewall rules:
 
@@ -83,7 +83,3 @@ For QEMU, the credentials for Opensearch must be parsed via the info flag as wel
 Remember to use single quotes for the password.
 
 You will also need to increase the memory size of QEMU to 4GB.
-
-### debugd images
-
-For a full list of image naming conventions and how to retreive them check [image version documentation](/.github/docs/README.md#image-versions)

hack/find-image/find-image.sh

@@ -8,19 +8,55 @@
 set -euo pipefail
 shopt -s inherit_errexit
 
+ref="-"
+stream="stable"
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+  -r | --ref)
+    ref="$2"
+    shift # past argument
+    shift # past value
+    ;;
+  -s | --stream)
+    stream="$2"
+    shift # past argument
+    shift # past value
+    ;;
+  -*)
+    echo "Unknown option $1"
+    exit 1
+    ;;
+  *)
+    POSITIONAL_ARGS+=("$1") # save positional arg
+    shift                   # past argument
+    ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
+
+ref=$(echo -n "${ref}" | tr -c '[:alnum:]' '-')
+
 base_url="https://cdn.confidential.cloud"
-bucket="cdn-constellation-backend"
+latest_path="constellation/v1/ref/${ref}/stream/${stream}/versions/latest/image.json"
-newest_debug_image_path=$(aws s3api list-objects-v2 \
+latest_url="${base_url}/${latest_path}"
-  --output text \
+latest_status=$(curl -s -o /dev/null -w "%{http_code}" "${latest_url}")
-  --bucket "${bucket}" \
+if [[ ${latest_status} != "200" ]]; then
-  --prefix constellation/v1/images/debug-v \
+  echo "No image found for ref ${ref} and stream ${stream} (${latest_status})"
-  --query "reverse(sort_by(Contents, &LastModified))[0].Key")
+  exit 1
+fi
+latest_version=$(curl -sL "${latest_url}" | jq -r '.version')
 
-image_version_uid=$(basename "${newest_debug_image_path}" .json)
+shortname=""
-url="${base_url}/${newest_debug_image_path}"
+if [[ ${ref} != "-" ]]; then
-echo "Found image version UID:"
+  shortname+="ref/${ref}/"
-echo "${image_version_uid}"
+fi
+if [[ ${stream} != "stable" ]]; then
+  shortname+="stream/${stream}/"
+fi
+shortname+="${latest_version}"
 
-echo "Containing the following images:"
+echo "${shortname}"
-echo "${url}"
-curl -sL "${url}" | jq

image/README.md

@@ -245,7 +245,9 @@ upload/upload_azure.sh -g --disk-name "${AZURE_DISK_NAME}" "${AZURE_VMGS_PATH}"
 
 ```sh
 # set these variables
-export IMAGE_VERSION_UID= # e.g. "test123" or "v2.1.0"
+export REF= # e.g. feat-xyz (branch name encoded with dashes)
+export STREAM= # e.g. "nightly", "debug", "stable" (depends on the type of image and if it is a release)
+export IMAGE_VERSION= # e.g. v2.1.0" or output of pseudo-version tool
 export QEMU_BUCKET=cdn-constellation-backend
 export QEMU_BASE_URL="https://cdn.confidential.cloud"
 export QEMU_IMAGE_PATH=${PWD}/mkosi.output.qemu/fedora~37/image.raw

image/upload/upload_qemu.sh

@@ -11,9 +11,10 @@ if [[ -z ${CONFIG_FILE-} ]] && [[ -f ${CONFIG_FILE-} ]]; then
   . "${CONFIG_FILE}"
 fi
 
-aws s3 cp "${QEMU_IMAGE_PATH}" "s3://${QEMU_BUCKET}/constellation/v1/raw/${IMAGE_VERSION_UID}/qemu/image.raw" --no-progress
+path="constellation/v1/ref/${REF}/stream/${STREAM}/image/${IMAGE_VERSION}/csp/qemu/image.raw"
+aws s3 cp "${QEMU_IMAGE_PATH}" "s3://${QEMU_BUCKET}/${path}" --no-progress
 
-image_url="${QEMU_BASE_URL}/constellation/v1/raw/${IMAGE_VERSION_UID}/qemu/image.raw"
+image_url="${QEMU_BASE_URL}/${path}"
 
 json=$(jq -ncS \
   --arg image_url "${image_url}" \

internal/config/config.go

@@ -45,7 +45,7 @@ type Config struct {
 	Version string `yaml:"version" validate:"eq=v2"`
 	// description: |
 	//   Machine image used to create Constellation nodes.
-	Image string `yaml:"image" validate:"required,safe_image"`
+	Image string `yaml:"image" validate:"required"`
 	// description: |
 	//   Size (in GB) of a node's disk to store the non-volatile state.
 	StateDiskSizeGB int `yaml:"stateDiskSizeGB" validate:"min=0"`
@@ -472,10 +472,6 @@ func (c *Config) Validate() error {
 		return err
 	}
 
-	if err := validate.RegisterValidation("safe_image", validateImage); err != nil {
-		return err
-	}
-
 	// register custom validator with label supported_k8s_version to validate version based on available versionConfigs.
 	if err := validate.RegisterValidation("supported_k8s_version", validateK8sVersion); err != nil {
 		return err

internal/config/validation.go

@@ -19,19 +19,6 @@ import (
 	"github.com/go-playground/validator/v10"
 )
 
-func validateImage(fl validator.FieldLevel) bool {
-	image := fl.Field().String()
-	switch {
-	case image == "":
-		return false
-	case image == "..":
-		return false
-	case strings.Contains(image, "/"):
-		return false
-	}
-	return true
-}
-
 func validateK8sVersion(fl validator.FieldLevel) bool {
 	return versions.IsSupportedK8sVersion(fl.Field().String())
 }

internal/constants/constants.go

@@ -170,11 +170,7 @@ const (
 
 	// CDNRepositoryURL is the base URL of the Constellation CDN artifact repository.
 	CDNRepositoryURL = "https://cdn.confidential.cloud"
-	// CDNImagePath is the default path to image references in the CDN repository.
+	// CDNAPIPrefix is the prefix of the Constellation API.
-	CDNImagePath = "constellation/v1/images"
-	// CDNMeasurementsPath is the default path to image measurements in the CDN repository.
-	CDNMeasurementsPath = "constellation/v1/measurements"
-	// CDNAPIPrefix is the prefix for the Constellation CDN API.
 	CDNAPIPrefix = "constellation/v1"
 )
 

internal/image/image.go

@@ -15,15 +15,55 @@ import (
 	"net/url"
 	"os"
 	"path"
-	"path/filepath"
 	"strings"
 
 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/v2/internal/config"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
+	"github.com/edgelesssys/constellation/v2/internal/shortname"
 	"github.com/spf13/afero"
 )
 
+// imageName is a struct that describes a Constellation OS imageName name.
+type imageName struct {
+	Ref     string
+	Stream  string
+	Version string
+}
+
+func newImageName(name string) (*imageName, error) {
+	ref, stream, version, err := shortname.ToParts(name)
+	if err != nil {
+		return nil, err
+	}
+	return &imageName{
+		Ref:     ref,
+		Stream:  stream,
+		Version: version,
+	}, nil
+}
+
+func (i *imageName) infoPath() string {
+	return path.Join(constants.CDNAPIPrefix, "ref", i.Ref, "stream", i.Stream, "image", i.Version, "info.json")
+}
+
+func (i *imageName) shortname() string {
+	return shortname.FromParts(i.Ref, i.Stream, i.Version)
+}
+
+// filename is the override file name for the image info file.
+func (i *imageName) filename() string {
+	name := i.shortname()
+	// replace all non-alphanumeric characters with an underscore
+	name = strings.Map(func(r rune) rune {
+		if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '.' {
+			return r
+		}
+		return '_'
+	}, name)
+	return name + ".json"
+}
+
 // imageInfo is a lookup table for image references.
 //
 // Example:
@@ -106,14 +146,18 @@ func (f *Fetcher) FetchReference(ctx context.Context, config *config.Config) (st
 	if err != nil {
 		return "", err
 	}
-	return f.fetch(ctx, provider, config.Image, variant)
+	image, err := newImageName(config.Image)
+	if err != nil {
+		return "", err
+	}
+	return f.fetch(ctx, provider, image, variant)
 }
 
-// fetch fetches the image reference for a given image version uid, CSP and image variant.
+// fetch fetches the image reference for a given image name, uid, CSP and image variant.
-func (f *Fetcher) fetch(ctx context.Context, csp cloudprovider.Provider, version, variant string) (string, error) {
+func (f *Fetcher) fetch(ctx context.Context, csp cloudprovider.Provider, img *imageName, variant string) (string, error) {
-	raw, err := getFromFile(f.fs, version)
+	raw, err := getFromFile(f.fs, img)
 	if err != nil && os.IsNotExist(err) {
-		raw, err = getFromURL(ctx, f.httpc, version)
+		raw, err = getFromURL(ctx, f.httpc, img)
 	}
 	if err != nil {
 		return "", fmt.Errorf("fetching image reference: %w", err)
@@ -145,19 +189,17 @@ func variant(provider cloudprovider.Provider, config *config.Config) (string, er
 	}
 }
 
-func getFromFile(fs *afero.Afero, version string) ([]byte, error) {
+func getFromFile(fs *afero.Afero, img *imageName) ([]byte, error) {
-	version = filepath.Base(version)
+	return fs.ReadFile(img.filename())
-	return fs.ReadFile(version + ".json")
 }
 
 // getFromURL fetches the image lookup table from a URL.
-func getFromURL(ctx context.Context, client httpc, version string) ([]byte, error) {
+func getFromURL(ctx context.Context, client httpc, img *imageName) ([]byte, error) {
 	url, err := url.Parse(constants.CDNRepositoryURL)
 	if err != nil {
 		return nil, fmt.Errorf("parsing image version repository URL: %w", err)
 	}
-	versionFilename := path.Base(version) + ".json"
+	url.Path = img.infoPath()
-	url.Path = path.Join(constants.CDNImagePath, versionFilename)
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), http.NoBody)
 	if err != nil {
 		return nil, err
@@ -171,7 +213,7 @@ func getFromURL(ctx context.Context, client httpc, version string) ([]byte, erro
 	if resp.StatusCode != http.StatusOK {
 		switch resp.StatusCode {
 		case http.StatusNotFound:
-			return nil, fmt.Errorf("image %q does not exist", version)
+			return nil, fmt.Errorf("image %q does not exist", img.shortname())
 		default:
 			return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode)
 		}

internal/image/image_test.go

@@ -160,9 +160,9 @@ func TestVariant(t *testing.T) {
 }
 
 func TestFetchReference(t *testing.T) {
-	imageVersionUID := "someImageVersionUID"
+	img := "ref/abc/stream/nightly/v1.2.3"
 	client := newTestClient(func(req *http.Request) *http.Response {
-		if strings.HasSuffix(req.URL.String(), "/constellation/v1/images/someImageVersionUID.json") {
+		if strings.HasSuffix(req.URL.String(), "/constellation/v1/ref/abc/stream/nightly/image/v1.2.3/info.json") {
 			return &http.Response{
 				StatusCode: http.StatusOK,
 				Body:       io.NopCloser(bytes.NewBufferString(lut)),
@@ -183,27 +183,27 @@ func TestFetchReference(t *testing.T) {
 		wantErr       bool
 	}{
 		"reference fetched remotely": {
-			config: &config.Config{Image: imageVersionUID, Provider: config.ProviderConfig{
+			config: &config.Config{Image: img, Provider: config.ProviderConfig{
 				QEMU: &config.QEMUConfig{},
 			}},
 			wantReference: "someReference",
 		},
 		"reference fetched locally": {
-			config: &config.Config{Image: imageVersionUID, Provider: config.ProviderConfig{
+			config: &config.Config{Image: img, Provider: config.ProviderConfig{
 				QEMU: &config.QEMUConfig{},
 			}},
 			overrideFile:  `{"qemu":{"default":"localOverrideReference"}}`,
 			wantReference: "localOverrideReference",
 		},
 		"lut is invalid": {
-			config: &config.Config{Image: imageVersionUID, Provider: config.ProviderConfig{
+			config: &config.Config{Image: img, Provider: config.ProviderConfig{
 				QEMU: &config.QEMUConfig{},
 			}},
 			overrideFile: `{`,
 			wantErr:      true,
 		},
 		"image version does not exist": {
-			config: &config.Config{Image: "nonExistingImageVersionUID", Provider: config.ProviderConfig{
+			config: &config.Config{Image: "nonExistingImageName", Provider: config.ProviderConfig{
 				QEMU: &config.QEMUConfig{},
 			}},
 			wantErr: true,
@@ -221,7 +221,7 @@ func TestFetchReference(t *testing.T) {
 
 			fetcher := &Fetcher{
 				httpc: client,
-				fs:    newImageVersionStubFs(t, imageVersionUID, tc.overrideFile),
+				fs:    newImageVersionStubFs(t, img, tc.overrideFile),
 			}
 			reference, err := fetcher.FetchReference(context.Background(), tc.config)
 			if tc.wantErr {
@@ -256,10 +256,12 @@ func newTestClient(fn roundTripFunc) *http.Client {
 	}
 }
 
-func newImageVersionStubFs(t *testing.T, imageVersionUID string, overrideFile string) *afero.Afero {
+func newImageVersionStubFs(t *testing.T, image string, overrideFile string) *afero.Afero {
 	fs := afero.NewMemMapFs()
+	img, err := newImageName(image)
+	must(t, err)
 	if overrideFile != "" {
-		must(t, afero.WriteFile(fs, imageVersionUID+".json", []byte(overrideFile), os.ModePerm))
+		must(t, afero.WriteFile(fs, img.filename(), []byte(overrideFile), os.ModePerm))
 	}
 	return &afero.Afero{Fs: fs}
 }

internal/image/raw.go

@@ -36,12 +36,12 @@ func NewDownloader() *Downloader {
 }
 
 // Download downloads the raw image from source.
-func (d *Downloader) Download(ctx context.Context, errWriter io.Writer, showBar bool, source, version string) (string, error) {
+func (d *Downloader) Download(ctx context.Context, errWriter io.Writer, showBar bool, source, imageName string) (string, error) {
 	url, err := url.Parse(source)
 	if err != nil {
 		return "", fmt.Errorf("parsing image source URL: %w", err)
 	}
-	version = filepath.Base(version)
+	imageName = filepath.Base(imageName)
 	var partfile, destination string
 	switch url.Scheme {
 	case "http", "https":
@@ -49,8 +49,8 @@ func (d *Downloader) Download(ctx context.Context, errWriter io.Writer, showBar
 		if err != nil {
 			return "", fmt.Errorf("getting current working directory: %w", err)
 		}
-		partfile = filepath.Join(cwd, version+".raw.part")
+		partfile = filepath.Join(cwd, imageName+".raw.part")
-		destination = filepath.Join(cwd, version+".raw")
+		destination = filepath.Join(cwd, imageName+".raw")
 	case "file":
 		return url.Path, nil
 	default:

internal/shortname/shortname.go

@@ -0,0 +1,55 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package shortname
+
+import (
+	"fmt"
+	"strings"
+)
+
+// ToParts splits an config shortname into its parts.
+// The shortname is expected to be in the format of one of:
+// ref/<ref>/stream/<stream>/<version>
+// stream/<stream>/<version>
+// version/<version>.
+func ToParts(shortname string) (string, string, string, error) {
+	parts := strings.Split(shortname, "/")
+	ref := "-"
+	stream := "stable"
+	var version string
+	switch len(parts) {
+	case 1:
+		version = parts[0]
+	case 3:
+		if parts[0] != "stream" {
+			return "", "", "", fmt.Errorf("invalid shortname: expected \"stream/<stream>/<version>\", got %q", shortname)
+		}
+		stream = parts[1]
+		version = parts[2]
+	case 5:
+		if parts[0] != "ref" || parts[2] != "stream" {
+			return "", "", "", fmt.Errorf("invalid shortname: expected \"ref/<ref>/stream/<stream>/<version>\", got %q", shortname)
+		}
+		ref = parts[1]
+		stream = parts[3]
+		version = parts[4]
+	default:
+		return "", "", "", fmt.Errorf("invalid shortname reference %q", shortname)
+	}
+	return ref, stream, version, nil
+}
+
+// FromParts joins the parts of a config shortname into a shortname.
+func FromParts(ref, stream, version string) string {
+	switch {
+	case ref == "-" && stream == "stable":
+		return version
+	case ref == "-":
+		return "stream/" + stream + "/" + version
+	}
+	return "ref/" + ref + "/stream/" + stream + "/" + version
+}

internal/shortname/shortname_test.go

@@ -0,0 +1,102 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package shortname
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
+func TestToParts(t *testing.T) {
+	testCases := map[string]struct {
+		shortname string
+		wantParts [3]string
+		wantErr   bool
+	}{
+		"only version": {
+			shortname: "v1.2.3",
+			wantParts: [3]string{"-", "stable", "v1.2.3"},
+		},
+		"stream and version": {
+			shortname: "stream/nightly/v1.2.3",
+			wantParts: [3]string{"-", "nightly", "v1.2.3"},
+		},
+		"full name": {
+			shortname: "ref/feat-xyz/stream/nightly/v1.2.3",
+			wantParts: [3]string{"feat-xyz", "nightly", "v1.2.3"},
+		},
+		"full name with extra slashes": {
+			shortname: "ref/feat-xyz//stream/nightly/v1.2.3",
+			wantErr:   true,
+		},
+		"invalid three part path": {
+			shortname: "invalid/invalid/invalid",
+			wantErr:   true,
+		},
+		"five part path": {
+			shortname: "invalid/invalid/invalid/invalid/invalid",
+			wantErr:   true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			ref, stream, version, err := ToParts(tc.shortname)
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			require.NoError(err)
+			assert.Equal(tc.wantParts, [3]string{ref, stream, version})
+		})
+	}
+}
+
+func TestFromParts(t *testing.T) {
+	testCases := map[string]struct {
+		ref, stream, version string
+		wantShortname        string
+	}{
+		"only version": {
+			ref:           "-",
+			stream:        "stable",
+			version:       "v1.2.3",
+			wantShortname: "v1.2.3",
+		},
+		"stream and version": {
+			ref:           "-",
+			stream:        "nightly",
+			version:       "v1.2.3",
+			wantShortname: "stream/nightly/v1.2.3",
+		},
+		"full name": {
+			ref:           "feat-xyz",
+			stream:        "nightly",
+			version:       "v1.2.3",
+			wantShortname: "ref/feat-xyz/stream/nightly/v1.2.3",
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			gotShortname := FromParts(tc.ref, tc.stream, tc.version)
+			assert.Equal(tc.wantShortname, gotShortname)
+		})
+	}
+}