ci: refactor e2e test failure notifications (#1625)

Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>

.github/actions/e2e_test/teams-payload.json

@@ -1,24 +0,0 @@
-{
-    "@type": "MessageCard",
-    "@context": "http://schema.org/extensions",
-    "themeColor": "FF5733",
-    "summary": "E2E Job Failed",
-    "sections": [{
-        "activityTitle": "E2E Job Failed",
-        "activitySubtitle": "${TEAMS_JOB_NAME}",
-        "activityImage": "https://miro.medium.com/max/552/1*G7s61tFPaLI9JRxWYpRNLw.png",
-        "facts": [{
-            "name": "Status",
-            "value": "Error"
-        }],
-        "markdown": true
-    }],
-    "potentialAction": [{
-        "@type": "OpenUri",
-        "name": "Go To Failed Action",
-        "targets": [{
-            "os": "default",
-            "uri": "https://github.com/edgelesssys/constellation/actions/runs/${TEAMS_RUN_ID}"
-        }]
-    }]
-}

.github/actions/notify_failure/action.yml

@@ -0,0 +1,99 @@
+name: notify failure
+description: "Post a failure message to project board and teams"
+
+inputs:
+  projectWriteToken:
+    description: "Token to write to the project board"
+    required: true
+  teamsWebhookUri:
+    description: "URI to send a message to the teams channel"
+    required: true
+  test:
+    description: "Test name"
+    required: true
+  provider:
+    description: "CSP"
+    required: true
+  refStream:
+    description: "RefStream of the run"
+    required: false
+  kubernetesVersion:
+    description: "Kubernetes version"
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: Pick assignee
+      id: pick-assignee
+      uses: ./.github/actions/pick_assignee
+
+    - name: Create project card in case of failure
+      id: create-project-card
+      continue-on-error: true
+      shell: bash
+      env:
+        GHH_TOKEN: ${{ inputs.projectWriteToken }}
+      run: |
+        # TODO(katexochen): add job number when possible
+        jobURL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+        opensearchURL="https://search-e2e-logs-y46renozy42lcojbvrt3qq7csm.eu-central-1.es.amazonaws.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-7d,to:now))&_a=(columns:!(metadata.name,systemd.unit,kubernetes.pod_name,message),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'74517cf0-6442-11ed-acf1-47dda8fdfbbb',key:metadata.provider,negate:!f,params:(query:${{ inputs.provider }}),type:phrase),query:(match_phrase:(metadata.provider:${{ inputs.provider }}))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'74517cf0-6442-11ed-acf1-47dda8fdfbbb',key:metadata.github.run-id,negate:!f,params:(query:${{ github.run_id }}),type:phrase),query:(match_phrase:(metadata.github.run-id:${{ github.run_id }})))),index:'74517cf0-6442-11ed-acf1-47dda8fdfbbb',interval:auto,query:(language:kuery,query:''),sort:!())"
+
+        cat << EOF > header.md
+
+        ## Metadata
+
+        * [Job URL](${jobURL})
+        * [OpenSearch URL](${opensearchURL})
+
+        EOF
+
+        cat header.md .github/failure_project_template.md > body.md
+        cp .github/failure_project_metadata.json metadata.json
+        DATE=$(date '+%F %a %T %Z') yq -iP '.issueTitle = env(DATE)' metadata.json
+        yq -iP '.assignees += [ "${{ steps.pick-assignee.outputs.assignee }}" ]' metadata.json
+        yq -iP '.fields.cloudProvider = "${{ inputs.provider }}"' metadata.json
+        yq -iP '.fields.test = "${{ inputs.test }}"' metadata.json
+        yq -iP '.fields.workflow = "${{ github.workflow }}"' metadata.json
+        if [[ -n "${{ inputs.kubernetesVersion }}" ]]; then
+          yq -iP '.fields.kubernetesVersion = "${{ inputs.kubernetesVersion }}"' metadata.json
+        fi
+        if [[ -n "${{ inputs.refStream }}" ]]; then
+          yq -iP '.fields.refStream = "${{ inputs.refStream }}"' metadata.json
+        fi
+        cat metadata.json
+
+        issueURL=$(
+        bazelisk run //bazel/ci:ghh -- create-project-issue \
+          --body body.md \
+          --metadata metadata.json \
+          -v
+        )
+
+        echo "jobURL=${jobURL}" | tee -a "$GITHUB_OUTPUT"
+        echo "opensearchURL=${opensearchURL}" | tee -a "$GITHUB_OUTPUT"
+        echo "issueURL=${issueURL}" | tee -a "$GITHUB_OUTPUT"
+
+    - name: Notify teams channel
+      continue-on-error: true
+      shell: bash
+      run: |
+        cp .github/teams_payload_template.json teams_payload.json
+        yq -iP '.attachments[0].content.body[0].columns[1].items[1].text = "${{ github.workflow }}"' teams_payload.json
+        yq -iP '.attachments[0].content.body[0].columns[1].items[2].facts += [ { "title": "CloudProvider", "value": "${{ inputs.provider }}" } ]' teams_payload.json
+        yq -iP '.attachments[0].content.body[0].columns[1].items[2].facts += [ { "title": "Test", "value": "${{ inputs.test }}" } ]' teams_payload.json
+        if [[ -n "${{ inputs.kubernetesVersion }}" ]]; then
+          yq -iP '.attachments[0].content.body[0].columns[1].items[2].facts += [ { "title": "KubernetesVersion", "value": "${{ inputs.kubernetesVersion }}" } ]' teams_payload.json
+        fi
+        if [[ -n "${{ inputs.refStream }}" ]]; then
+          yq -iP '.attachments[0].content.body[0].columns[1].items[2].facts += [ { "title": "RefStream", "value": "${{ inputs.refStream }}" } ]' teams_payload.json
+        fi
+        yq -iP '.attachments[0].content.actions[0].url = "${{ steps.create-project-card.outputs.jobURL }}"' teams_payload.json
+        yq -iP '.attachments[0].content.actions[1].url = "${{ steps.create-project-card.outputs.issueURL }}"' teams_payload.json
+        yq -iP ".attachments[0].content.actions[2].url = \"${{ steps.create-project-card.outputs.opensearchURL }}\"" teams_payload.json
+        cat teams_payload.json
+
+        curl \
+          -H "Content-Type: application/json" \
+          -d @teams_payload.json \
+          "${{ inputs.teamsWebhookUri }}"

.github/actions/pick_assignee/action.yml

@@ -0,0 +1,24 @@
+name: Pick and assignee
+description: "Pick an assignee"
+
+outputs:
+  assignee:
+    description: "GitHub login of the assignee"
+    value: ${{ steps.pick-assignee.outputs.assignee }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Pick an assignee
+      id: pick-assignee
+      shell: bash
+      run: |
+        possibleAssignees=(
+          "katexochen"
+          "malt3"
+          "3u13r"
+          "daniel-weisse"
+          "derpsteb"
+        )
+        assignee=${possibleAssignees[$RANDOM % ${#possibleAssignees[@]}]}
+        echo "assignee=$assignee" | tee -a "$GITHUB_OUTPUT"

.github/failure_project_metadata.json

@@ -0,0 +1,14 @@
+{
+    "organization": "edgelesssys",
+    "projectNumber": 3,
+    "issueTitle": "",
+    "assignees": [],
+    "fields": {
+        "Status": "New failures",
+        "cloudProvider": "",
+        "test": "",
+        "workflow": "",
+        "kubernetesVersion": "",
+        "refStream": ""
+    }
+}

.github/failure_project_template.md

@@ -0,0 +1,7 @@
+## Tasks
+
+- [ ] Inspect GitHub run logs
+- [ ] Inspect Boot logs
+- [ ] Inspect OpenSearch logs
+- [ ] Correlate to existing bug tickets *or*
+- [ ] Add new bug ticket

.github/teams_payload_template.json

@@ -0,0 +1,85 @@
+{
+    "type": "message",
+    "attachments": [
+        {
+            "contentType": "application/vnd.microsoft.card.adaptive",
+            "contentUrl": null,
+            "content": {
+                "type": "AdaptiveCard",
+                "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+                "version": "1.2",
+                "msteams": {
+                    "width": "Full"
+                },
+                "body": [
+                    {
+                        "type": "ColumnSet",
+                        "columns": [
+                            {
+                                "type": "Column",
+                                "width": 1,
+                                "items": [
+                                    {
+                                        "type": "Image",
+                                        "url": "https://miro.medium.com/max/552/1*G7s61tFPaLI9JRxWYpRNLw.png",
+                                        "size": "large",
+                                        "horizontalAlignment": "center"
+                                    }
+                                ],
+                                "horizontalAlignment": "center",
+                                "verticalContentAlignment": "center"
+                            },
+                            {
+                                "type": "Column",
+                                "width": 2,
+                                "items": [
+                                    {
+                                        "type": "TextBlock",
+                                        "text": "Constellation E2E test failed",
+                                        "wrap": true,
+                                        "fontType": "Default",
+                                        "size": "large",
+                                        "weight": "bolder"
+                                    },
+                                    {
+                                        "type": "TextBlock",
+                                        "text": "Subtitle",
+                                        "wrap": true,
+                                        "size": "large",
+                                        "isSubtle": true,
+                                        "spacing": "Small"
+                                    },
+                                    {
+                                        "type": "FactSet",
+                                        "facts": [],
+                                        "spacing": "small"
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+                ],
+                "actions": [
+                    {
+                        "type": "Action.OpenUrl",
+                        "title": "GitHub workflow run",
+                        "url": "",
+                        "style": "positive"
+                    },
+                    {
+                        "type": "Action.OpenUrl",
+                        "title": "Project board issue",
+                        "url": "",
+                        "style": "positive"
+                    },
+                    {
+                        "type": "Action.OpenUrl",
+                        "title": "OpenSearch logs",
+                        "url": "",
+                        "style": "positive"
+                    }
+                ]
+            }
+        }
+    ]
+}

.github/workflows/e2e-test-daily.yml

@@ -45,8 +45,10 @@ jobs:
       fail-fast: false
       max-parallel: 5
       matrix:
+        kubernetesVersion: ["1.25"] # should be default
         provider: ["gcp", "azure", "aws"]
         refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"]
+        test: ["sonobuoy full"]
     runs-on: ubuntu-22.04
     permissions:
       id-token: write
@@ -79,7 +81,8 @@ jobs:
           gcpClusterCreateServiceAccount: "constellation-e2e-cluster@constellation-331613.iam.gserviceaccount.com"
           gcpIAMCreateServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
           gcpInClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
-          test: "sonobuoy full"
+          kubernetesVersion: ${{ matrix.kubernetesVersion }}
+          test: ${{ matrix.test }}
           buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
           azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
           azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
@@ -98,17 +101,14 @@ jobs:
           azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           gcpServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
 
-      - name: Notify teams channel
+      - name: Notify about failure
         if: failure() && github.ref == 'refs/heads/main'
         continue-on-error: true
-        shell: bash
+        uses: ./.github/actions/notify_failure
-        working-directory: .github/actions/e2e_test
+        with:
-        run: |
+          projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
-          sudo apt-get install gettext-base -y
+          teamsWebhookUri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}
-          export TEAMS_JOB_NAME=${{ matrix.provider }}
+          refStream: ${{ matrix.refStream }}
-          export TEAMS_RUN_ID=${{ github.run_id }}
+          test: ${{ matrix.test }}
-          envsubst < teams-payload.json > to-be-send.json
+          kubernetesVersion: ${{ matrix.kubernetesVersion }}
-          curl                                          \
+          provider: ${{ matrix.provider }}
-            -H "Content-Type: application/json"         \
-            -d @to-be-send.json                         \
-            "${{ secrets.MS_TEAMS_WEBHOOK_URI }}"

.github/workflows/e2e-test-release.yml

@@ -182,21 +182,6 @@ jobs:
           azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           gcpServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
 
-      - name: Notify teams channel
-        if: failure() && github.ref == 'refs/heads/main'
-        continue-on-error: true
-        shell: bash
-        working-directory: .github/actions/e2e_test
-        run: |
-          sudo apt-get install gettext-base -y
-          export TEAMS_JOB_NAME=${{ matrix.provider }}
-          export TEAMS_RUN_ID=${{ github.run_id }}
-          envsubst < teams-payload.json > to-be-send.json
-          curl                                          \
-            -H "Content-Type: application/json"         \
-            -d @to-be-send.json                         \
-            "${{ secrets.MS_TEAMS_WEBHOOK_URI }}"
-
       - name: Always destroy Azure resource group
         if: always() && matrix.provider == 'azure'
         shell: bash

.github/workflows/e2e-test-weekly.yml

@@ -160,28 +160,24 @@ jobs:
           azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           gcpServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
 
-      - name: Notify teams channel
+      - name: Notify about failure
         if: failure() && github.ref == 'refs/heads/main'
         continue-on-error: true
-        shell: bash
+        uses: ./.github/actions/notify_failure
-        working-directory: .github/actions/e2e_test
+        with:
-        run: |
+          projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
-          sudo apt-get install gettext-base -y
+          teamsWebhookUri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}
-          export TEAMS_JOB_NAME=${{ matrix.provider }}
+          refStream: ${{ matrix.refStream }}
-          export TEAMS_RUN_ID=${{ github.run_id }}
+          test: ${{ matrix.test }}
-          envsubst < teams-payload.json > to-be-send.json
+          kubernetesVersion: ${{ matrix.kubernetes-version }}
-          curl                                          \
+          provider: ${{ matrix.provider }}
-            -H "Content-Type: application/json"         \
-            -d @to-be-send.json                         \
-            "${{ secrets.MS_TEAMS_WEBHOOK_URI }}"
 
   e2e-upgrade:
     strategy:
       fail-fast: false
       max-parallel: 1
       matrix:
-          fromVersion:
+        fromVersion: ["v2.6.0"]
-            ["v2.6.0"]
         cloudProvider: ["gcp", "azure"]
     name: Run upgrade tests
     secrets: inherit

.github/workflows/e2e-upgrade.yml

@@ -73,7 +73,6 @@ on:
         type: string
         required: false
 
-
 env:
   ARM_CLIENT_ID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
   ARM_CLIENT_SECRET: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
@@ -204,21 +203,6 @@ jobs:
         with:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
 
-      - name: Notify teams channel
-        if: failure() && github.ref == 'refs/heads/main'
-        continue-on-error: true
-        shell: bash
-        working-directory: .github/actions/e2e_test
-        run: |
-          sudo apt-get install gettext-base -y
-          export TEAMS_JOB_NAME="upgrade-${{ inputs.cloudProvider }}"
-          export TEAMS_RUN_ID=${{ github.run_id }}
-          envsubst < teams-payload.json > to-be-send.json
-          curl                                          \
-            -H "Content-Type: application/json"         \
-            -d @to-be-send.json                         \
-            "${{ secrets.MS_TEAMS_WEBHOOK_URI }}"
-
       - name: Always destroy Azure resource group
         if: always() && inputs.cloudProvider == 'azure'
         shell: bash
@@ -229,3 +213,13 @@ jobs:
             --force-deletion-types Microsoft.Compute/virtualMachines \
             --no-wait \
             --yes
+
+      - name: Notify about failure
+        if: failure() && github.ref == 'refs/heads/main'
+        continue-on-error: true
+        uses: ./.github/actions/notify_failure
+        with:
+          projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
+          teamsWebhookUri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}
+          test: "upgrade"
+          provider: ${{ inputs.cloudProvider }}

bazel/ci/BUILD.bazel

@@ -415,6 +415,22 @@ sh_template(
     template = "cli_docgen.sh.in",
 )
 
+alias(
+    name = "com_github_katexochen_ghh",
+    actual = select({
+        "@io_bazel_rules_go//go/platform:darwin_amd64": "@com_github_katexochen_ghh_darwin_amd64//:ghh",
+        "@io_bazel_rules_go//go/platform:darwin_arm64": "@com_github_katexochen_ghh_darwin_arm64//:ghh",
+        "@io_bazel_rules_go//go/platform:linux_amd64": "@com_github_katexochen_ghh_linux_amd64//:ghh",
+        "@io_bazel_rules_go//go/platform:linux_arm64": "@com_github_katexochen_ghh_linux_arm64//:ghh",
+    }),
+)
+
+repo_command(
+    name = "ghh",
+    args = [],
+    command = ":com_github_katexochen_ghh",
+)
+
 multirun(
     name = "tidy",
     commands = [

bazel/sh/repo_command.sh.in

@@ -12,4 +12,4 @@ fi
 
 cd "${BUILD_WORKSPACE_DIRECTORY}" || exit 1
 
-"${cmd}" "${args[@]}"
+"${cmd}" "${args[@]}" "${@}"

bazel/toolchains/ci_deps.bzl

@@ -13,6 +13,7 @@ def ci_deps():
     _buf_deps()
     _talos_docgen_deps()
     _helm_deps()
+    _ghh_deps()
 
 def _shellcheck_deps():
     http_archive(
@@ -392,3 +393,45 @@ def _helm_deps():
             "https://get.helm.sh/helm-v3.11.2-darwin-arm64.tar.gz",
         ],
     )
+
+def _ghh_deps():
+    http_archive(
+        name = "com_github_katexochen_ghh_linux_amd64",
+        urls = [
+            "https://cdn.confidential.cloud/constellation/cas/sha256/753f2095fb0aea24e834b9ceb64d41eb19f8d8f5f04af745ee8dd0ef0e954135",
+            "https://github.com/katexochen/ghh/releases/download/v0.2.0/ghh_0.2.0_linux_amd64.tar.gz",
+        ],
+        type = "tar.gz",
+        build_file_content = """exports_files(["ghh"], visibility = ["//visibility:public"])""",
+        sha256 = "753f2095fb0aea24e834b9ceb64d41eb19f8d8f5f04af745ee8dd0ef0e954135",
+    )
+    http_archive(
+        name = "com_github_katexochen_ghh_linux_arm64",
+        urls = [
+            "https://cdn.confidential.cloud/constellation/cas/sha256/ccac401106e7937838420209e9b9bfba438952a251344f5fdf935dfeeaee0043",
+            "https://github.com/katexochen/ghh/releases/download/v0.2.0/ghh_0.2.0_linux_arm64.tar.gz",
+        ],
+        type = "tar.gz",
+        build_file_content = """exports_files(["ghh"], visibility = ["//visibility:public"])""",
+        sha256 = "ccac401106e7937838420209e9b9bfba438952a251344f5fdf935dfeeaee0043",
+    )
+    http_archive(
+        name = "com_github_katexochen_ghh_darwin_amd64",
+        urls = [
+            "https://cdn.confidential.cloud/constellation/cas/sha256/a6c89f1143230697f09d30ad86fc152d7daf2ddee4121dfc0aa89aac087b11c4",
+            "https://github.com/katexochen/ghh/releases/download/v0.2.0/ghh_0.2.0_darwin_amd64.tar.gz",
+        ],
+        type = "tar.gz",
+        build_file_content = """exports_files(["ghh"], visibility = ["//visibility:public"])""",
+        sha256 = "a6c89f1143230697f09d30ad86fc152d7daf2ddee4121dfc0aa89aac087b11c4",
+    )
+    http_archive(
+        name = "com_github_katexochen_ghh_darwin_arm64",
+        urls = [
+            "https://cdn.confidential.cloud/constellation/cas/sha256/4c11127967f8b4d1dc694f7a262243338b090e523c44017fe4a44c8864a6421c",
+            "https://github.com/katexochen/ghh/releases/download/v0.2.0/ghh_0.2.0_darwin_arm64.tar.gz",
+        ],
+        type = "tar.gz",
+        build_file_content = """exports_files(["ghh"], visibility = ["//visibility:public"])""",
+        sha256 = "4c11127967f8b4d1dc694f7a262243338b090e523c44017fe4a44c8864a6421c",
+    )