Co-authored-by: Malte Poll <mp@edgeless.systems>
Co-authored-by: katexochen <katexochen@users.noreply.github.com>
Co-authored-by: Daniel Weiße <dw@edgeless.systems>
Co-authored-by: Thomas Tendyck <tt@edgeless.systems>
Co-authored-by: Benedict Schlueter <bs@edgeless.systems>
Co-authored-by: leongross <leon.gross@rub.de>
Co-authored-by: Moritz Eckert <m1gh7ym0@gmail.com>
This commit is contained in:
Leonard Cohnen 2022-03-22 16:03:15 +01:00
commit 2d8fcd9bf4
362 changed files with 50980 additions and 0 deletions

55
.github/workflows/build-ami.yml vendored Normal file
View file

@ -0,0 +1,55 @@
name: Build the AMI Template
on:
workflow_dispatch:
workflow_call:
secrets:
AWS_ACCESS_KEY_ID:
required: true
AWS_SECRET_ACCESS_KEY:
required: true
AWS_DEFAULT_REGION:
required: true
BUCKET_NAME:
required: true
jobs:
build-enclave:
name: "Build the AMI"
runs-on: ubuntu-latest
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
working-directory: images/aws/ec2
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v2
- name: Install AWS CLI
id: prepare
run: sudo apt-get update && sudo apt-get -y install awscli
- name: Download eif
id: download_eif
run: aws s3 cp s3://${{ secrets.BUCKET_NAME }}/eif/ ${{ github.workspace }}/${{ env.working-directory }}/ --recursive --quiet
- name: Download gvproxy
id: download_gvproxy
run: aws s3 cp s3://${{ secrets.BUCKET_NAME }}/gvproxy/gvproxy ${{ github.workspace }}/${{ env.working-directory }}/ --quiet
- name: Install build dependencies
run: sudo apt-get -y install packer
- name: Init packer
run: packer init .
working-directory: ${{ env.working-directory }}
- name: Validate packer
run: packer validate -syntax-only .
working-directory: ${{ env.working-directory }}
- name: Build packer
run: packer build -color=false .
working-directory: ${{ env.working-directory }}

107
.github/workflows/build-coordinator.yml vendored Normal file
View file

@ -0,0 +1,107 @@
name: Build and Upload the Coordinator
on:
workflow_dispatch:
push:
branches:
- main
jobs:
build-coordinator:
name: "Build the Coordinator"
runs-on: ubuntu-latest
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
outputs:
coordinator-name: ${{ steps.copy.outputs.coordinator-name }}
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Install Dependencies
id: prepare
run: sudo apt-get update && sudo apt-get -y install awscli
- name: Build the Coordinator
uses: docker/build-push-action@v2
with:
context: .
file: Dockerfile.build
outputs: .
push: false
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
# This ugly bit is necessary if you don't want your cache to grow forever
# till it hits GitHub's limit of 5GB.
# Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- name: Move cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
- name: Copy Coordinator to S3 if not exists
id: copy
run: >
aws s3api head-object --bucket ${{ secrets.PUBLIC_BUCKET_NAME }} --key coordinator/$(ls | grep "coordinator-")
|| (
echo "::set-output name=coordinator-name::$(ls | grep "coordinator-")"
&& aws s3 cp ${{ github.workspace }}/ s3://${{ secrets.PUBLIC_BUCKET_NAME }}/coordinator/ --exclude "*" --include "coordinator-*" --include "constellation" --recursive --quiet)
shell: bash {0}
call-coreos:
needs: build-coordinator
if: startsWith(needs.build-coordinator.outputs.coordinator-name, 'coordinator-')
uses: ./.github/workflows/build-coreos.yml
with:
coordinator-name: ${{ needs.build-coordinator.outputs.coordinator-name }}
secrets:
CI_GITHUB_REPOSITORY: ${{ secrets.CI_GITHUB_REPOSITORY }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
BUCKET_NAME: ${{ secrets.BUCKET_NAME }}
PUBLIC_BUCKET_NAME: ${{ secrets.PUBLIC_BUCKET_NAME }}
SSH_PUB_KEY: ${{ secrets.SSH_PUB_KEY }}
SSH_PUB_KEY_PATH: ${{ secrets.SSH_PUB_KEY_PATH }}
AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
call-aws-enclave:
needs: build-coordinator
if: startsWith(needs.build-coordinator.outputs.coordinator-name, 'coordinator-')
uses: ./.github/workflows/build-enclave.yml
with:
coordinator-name: ${{ needs.build-coordinator.outputs.coordinator-name }}
secrets:
CI_GITHUB_REPOSITORY: ${{ secrets.CI_GITHUB_REPOSITORY }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
BUCKET_NAME: ${{ secrets.BUCKET_NAME }}
PUBLIC_BUCKET_NAME: ${{ secrets.PUBLIC_BUCKET_NAME }}
SSH_PUB_KEY: ${{ secrets.SSH_PUB_KEY }}
SSH_PUB_KEY_PATH: ${{ secrets.SSH_PUB_KEY_PATH }}
call-aws-ami:
needs: call-aws-enclave
uses: ./.github/workflows/build-ami.yml
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
BUCKET_NAME: ${{ secrets.BUCKET_NAME }}

View file

@ -0,0 +1,79 @@
name: Build and Upload CoreOS debug image
env:
REGISTRY: ghcr.io
on:
workflow_dispatch:
jobs:
build-enclave:
name: "Build CoreOS debug image using customized COSA"
runs-on: [self-hosted, linux, nested-virt]
permissions:
contents: read
packages: read
defaults:
run:
shell: bash
env:
working-directory: ${{ github.workspace }}/images/fcos
SHELL: /bin/bash
GOPATH: /home/github-actions-runner-user/go
GOCACHE: /home/github-actions-runner-user/.cache/go-build
GOMODCACHE: /home/github-actions-runner-user/.cache/go-mod
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v2
with:
submodules: recursive
token: ${{ secrets.CI_GITHUB_REPOSITORY }}
- name: Log in to the Container registry
id: docker-login
uses: docker/login-action@v1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: "Install azure CLI"
run: |
# use pip since azure cli repository is not working as expected
# https://github.com/Azure/azure-cli/issues/21532
# curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
sudo apt-get update
sudo apt-get install -y python3 python3-pip
sudo pip install azure-cli
wget -q https://aka.ms/downloadazcopy-v10-linux -O azcopy.tar.gz
tar --strip-components 1 -xf azcopy.tar.gz
rm azcopy.tar.gz
echo "$(pwd)" >> $GITHUB_PATH
- uses: azure/login@v1
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
- name: Setup Go environment
uses: actions/setup-go@v2.2.0
with:
go-version: "1.18"
- name: "Compile debugd"
run: GOCACHE=/home/github-actions-runner-user/.cache/go-build GOPATH=/home/github-actions-runner-user/go GOPRIVATE=github.com/edgelesssys GOMODCACHE=/home/github-actions-runner-user/.cache/go-mod go build -o constellation-debugd debugd.go
working-directory: ${{ github.workspace }}/debugd/debugd/cmd/debugd
- name: "Store GH token to be mounted by cosa"
run: echo "machine github.com login api password ${{ secrets.CI_GITHUB_REPOSITORY }}" > /tmp/.netrc
- name: "Set image timestamp"
run: |
TIMESTAMP=$(date +%s)
echo "TIMESTAMP=${TIMESTAMP}" >> $GITHUB_ENV
echo "IMAGE_TIMESTAMP=constellation-coreos-debugd-${TIMESTAMP}" >> $GITHUB_ENV
echo "IMAGE_VERSION=0.0.${TIMESTAMP}" >> $GITHUB_ENV
- name: "Build and Upload"
run: >
make -j$(nproc) CONTAINER_ENGINE=docker NETRC=/tmp/.netrc GCP_IMAGE_NAME="${{ env.IMAGE_TIMESTAMP }}" AZURE_IMAGE_NAME="${{ env.IMAGE_TIMESTAMP }}"
AZURE_IMAGE_DEFINITION="constellation-coreos-debugd" AZURE_IMAGE_VERSION="${{env.IMAGE_VERSION }}" DOWNLOAD_COORDINATOR=n COORDINATOR_BINARY="${{ github.workspace }}/debugd/debugd/cmd/debugd/constellation-debugd"
image-gcp image-azure upload-gcp upload-azure
working-directory: ${{ env.working-directory }}

99
.github/workflows/build-coreos.yml vendored Normal file
View file

@ -0,0 +1,99 @@
name: Build and Upload CoreOS
env:
REGISTRY: ghcr.io
on:
workflow_dispatch:
inputs:
coordinator-name:
description: Coordinator name
required: true
type: string
workflow_call:
inputs:
coordinator-name:
required: true
type: string
secrets:
CI_GITHUB_REPOSITORY:
required: true
AWS_ACCESS_KEY_ID:
required: true
AWS_SECRET_ACCESS_KEY:
required: true
AWS_DEFAULT_REGION:
required: true
BUCKET_NAME:
required: true
PUBLIC_BUCKET_NAME:
required: true
SSH_PUB_KEY:
required: true
SSH_PUB_KEY_PATH:
required: true
AZURE_CREDENTIALS:
required: true
jobs:
build-enclave:
name: "Build CoreOS using customized COSA"
runs-on: [self-hosted, linux, nested-virt]
permissions:
contents: read
packages: read
defaults:
run:
shell: bash
env:
working-directory: ${{ github.workspace }}/images/fcos
SHELL: /bin/bash
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v2
with:
submodules: recursive
token: ${{ secrets.CI_GITHUB_REPOSITORY }}
- name: Log in to the Container registry
id: docker-login
uses: docker/login-action@v1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: "Install azure CLI"
run: |
# use pip since azure cli repository is not working as expected
# https://github.com/Azure/azure-cli/issues/21532
# curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
sudo apt-get update
sudo apt-get install -y python3 python3-pip
sudo pip install azure-cli
wget -q https://aka.ms/downloadazcopy-v10-linux -O azcopy.tar.gz
tar --strip-components 1 -xf azcopy.tar.gz
rm azcopy.tar.gz
echo "$(pwd)" >> $GITHUB_PATH
- uses: azure/login@v1
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
- name: "Store GH token to be mounted by cosa"
run: echo "machine github.com login api password ${{ secrets.CI_GITHUB_REPOSITORY }}" > /tmp/.netrc
- name: "Set image timestamp"
run: |
TIMESTAMP=$(date +%s)
echo "TIMESTAMP=${TIMESTAMP}" >> $GITHUB_ENV
echo "IMAGE_TIMESTAMP=constellation-coreos-${TIMESTAMP}" >> $GITHUB_ENV
echo "IMAGE_VERSION=0.0.${TIMESTAMP}" >> $GITHUB_ENV
- name: "Build and Upload"
run: >
make -j$(nproc) CONTAINER_ENGINE=docker NETRC=/tmp/.netrc GCP_IMAGE_NAME="${{ env.IMAGE_TIMESTAMP }}" AZURE_IMAGE_NAME="${{ env.IMAGE_TIMESTAMP }}"
AZURE_IMAGE_DEFINITION="constellation-coreos" AZURE_IMAGE_VERSION="${{env.IMAGE_VERSION }}" COORDINATOR_URL="https://${{ secrets.PUBLIC_BUCKET_NAME }}.s3.us-east-2.amazonaws.com/coordinator/${{ inputs.coordinator-name }}"
image-gcp image-azure upload-gcp upload-azure
working-directory: ${{ env.working-directory }}

76
.github/workflows/build-enclave.yml vendored Normal file
View file

@ -0,0 +1,76 @@
name: Build and Upload the Enclave Image File
on:
workflow_dispatch:
inputs:
coordinator-name:
description: Coordinator name
required: true
type: string
workflow_call:
inputs:
coordinator-name:
required: true
type: string
secrets:
CI_GITHUB_REPOSITORY:
required: true
AWS_ACCESS_KEY_ID:
required: true
AWS_SECRET_ACCESS_KEY:
required: true
AWS_DEFAULT_REGION:
required: true
BUCKET_NAME:
required: true
PUBLIC_BUCKET_NAME:
required: true
SSH_PUB_KEY:
required: true
SSH_PUB_KEY_PATH:
required: true
jobs:
build-enclave:
name: "Build the Enclave"
runs-on: ubuntu-latest
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v2
with:
submodules: recursive
token: ${{ secrets.CI_GITHUB_REPOSITORY }}
- name: Install AWS CLI
id: prepare
run: sudo apt-get update && sudo apt-get -y install awscli
- name: Download bzImage, init and nsm.ko to AWS S3 Bucket
id: download-artifacts
run: aws s3 cp s3://${{ secrets.BUCKET_NAME }}/blobs/ ${{ github.workspace }}/images/aws/enclave/userland/dependencies/blobs/ --recursive
- name: Download Coordinator
id: download-coordinator
run: aws s3 cp s3://${{ secrets.PUBLIC_BUCKET_NAME }}/coordinator/${{ inputs.coordinator-name }} ${{ github.workspace }}/images/aws/enclave/userland/build/coordinator
- name: Write ssh public key to file
run: echo $SSH_PUB_KEY >> ${{ env.SSH_PUB_KEY_PATH }} && chmod 644 ${{ env.SSH_PUB_KEY_PATH }}
env:
SSH_PUB_KEY: ${{ secrets.SSH_PUB_KEY }}
SSH_PUB_KEY_PATH: ~/authorized_keys
- name: Build the eif file
run: make -j$(nproc) SSH_DIR=~/ -C ${{ github.workspace }}/images/aws/enclave/
- name: Upload eif file to AWS S3 Bucket
id: upload
run: aws s3 cp ${{ github.workspace }}/images/aws/enclave/userland/build/ s3://${{ secrets.BUCKET_NAME }}/eif/ --recursive --exclude "*" --include "*.eif" --quiet

36
.github/workflows/build-kernel.yml vendored Normal file
View file

@ -0,0 +1,36 @@
name: Build the Kernel
on:
push:
branches:
- main
paths:
- 'kernel/**'
workflow_dispatch:
jobs:
compile-and-upload-kernel:
name: "Compile and upload the Kernel"
runs-on: ubuntu-latest
steps:
- name: Install build dependencies
id: install
run: sudo apt-get update && sudo apt-get install -y git build-essential fakeroot libncurses5-dev libssl-dev ccache bison flex libelf-dev dwarves
- name: Checkout
id: checkout
uses: actions/checkout@v2
- name: Compile using make
id: compile
run: make -C ${{ github.workspace }}/images/aws/kernel/
- name: Install AWS CLI
id: prepare
run: sudo apt-get -y install awscli
- name: Upload bzImage, init and nsm.ko to AWS S3 Bucket
id: upload
run: aws s3 cp ${{ github.workspace }}/images/aws/kernel/build/blobs/ s3://${{ secrets.BUCKET_NAME }}/blobs/ --recursive --quiet
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

View file

@ -0,0 +1,47 @@
name: Patch gvisor-tap-vsock and Upload to S3
on:
workflow_dispatch:
inputs:
version:
description: "gvisor version"
required: true
default: 0.3.0
jobs:
build:
name: "Build"
runs-on: ubuntu-latest
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
working-directory: ec2
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v2
- name: Prepare Download
id: prepare
run: sudo apt-get update && sudo apt-get -y install wget tar make
- name: Download and unpack sources
id: unpack
run: wget -c https://github.com/containers/gvisor-tap-vsock/archive/refs/tags/v${{ github.event.inputs.version }}.tar.gz -O - | tar xz
working-directory: ${{ github.workspace }}
- name: Install go
uses: actions/setup-go@v2
with:
go-version: go1.17.6
- name: Patch source code
run: patch --ignore-whitespace ${{ github.workspace }}/gvisor-tap-vsock-${{ github.event.inputs.version }}/pkg/services/forwarder/tcp.go < ${{ github.workspace }}/images/aws/ec2/patches/remove_link_local.patch
working-directory: ${{ env.working-directory }}
- name: Build gvisor
id: build
run: make -C ${{ github.workspace }}/gvisor-tap-vsock-${{ github.event.inputs.version }}/
- name: Upload gvproxy
id: upload_gvproxy
run: aws s3 cp ${{ github.workspace }}/gvisor-tap-vsock-${{ github.event.inputs.version }}/bin/gvproxy s3://${{ secrets.BUCKET_NAME }}/gvproxy/gvproxy --quiet

View file

@ -0,0 +1,22 @@
name: Etcd Integration Test
on:
workflow_dispatch:
push:
branches:
- main
pull_request:
jobs:
integration-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Go environment
uses: actions/setup-go@v2.1.4
with:
go-version: "1.18"
- name: Test Constellation etcd integration
run: go test -v --race -cover -count=3 -tags integration
working-directory: coordinator/store

23
.github/workflows/test-integration.yml vendored Normal file
View file

@ -0,0 +1,23 @@
name: Integration Test
on:
workflow_dispatch:
push:
branches:
- main
pull_request:
jobs:
integration-test:
runs-on: ubuntu-latest
env:
GOPRIVATE: github.com/edgelesssys/*
steps:
- uses: actions/checkout@v2
- name: Setup Go environment
uses: actions/setup-go@v2.1.4
with:
go-version: "1.18"
- name: Run Integration Test
run: DEBUG=true go test -v -tags integration ./test/

23
.github/workflows/test-lint.yml vendored Normal file
View file

@ -0,0 +1,23 @@
name: Golangci-lint
on:
pull_request:
permissions:
contents: read
# Allow read access to pull request. Use with `only-new-issues` option.
pull-requests: read
jobs:
golangci:
name: lint
runs-on: ubuntu-latest
env:
GOPRIVATE: github.com/edgelesssys/*
steps:
- uses: actions/checkout@v2
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
with:
only-new-issues: true

18
.github/workflows/test-shellcheck.yml vendored Normal file
View file

@ -0,0 +1,18 @@
name: Shellcheck
on:
push:
branches:
- main
pull_request:
jobs:
shellcheck:
name: Shellcheck
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Run ShellCheck
uses: ludeeus/action-shellcheck@master
with:
severity: error
ignore_names: merge_config.sh

27
.github/workflows/test-unittest.yml vendored Normal file
View file

@ -0,0 +1,27 @@
name: Unit Tests
on:
workflow_dispatch:
push:
branches:
- main
pull_request:
jobs:
test:
runs-on: ubuntu-latest
env:
GOPRIVATE: github.com/edgelesssys/*
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.18
- name: Install Dependencies
run: sudo apt-get update && sudo apt-get install -y libcryptsetup-dev
- name: Test
run: go test -race -count=3 ./...