mirror of
https://github.com/edgelesssys/constellation.git
synced 2024-10-01 01:36:09 -04:00
cli: add basic support for constellation create
on OpenStack (#1283)
* image: support OpenStack image build / upload * cli: add OpenStack terraform template * config: add OpenStack as CSP * versionsapi: add OpenStack as CSP * cli: add OpenStack as provider for `config generate` and `create` * disk-mapper: add basic support for boot on OpenStack * debugd: add placeholder for OpenStack * image: fix config file sourcing for image upload
This commit is contained in:
parent
b013a7ab32
commit
b79f7d0c8c
24
.github/actions/os_build_variables/action.yml
vendored
24
.github/actions/os_build_variables/action.yml
vendored
@ -129,6 +129,18 @@ outputs:
|
||||
gcpImageFamily:
|
||||
description: "GCP image family"
|
||||
value: ${{ steps.gcp.outputs.imageFamily }}
|
||||
openStackJsonOutput:
|
||||
description: "OpenStack image json output path"
|
||||
value: ${{ steps.openstack.outputs.jsonOutput }}
|
||||
openStackBucket:
|
||||
description: "OpenStack S3 bucket"
|
||||
value: ${{ steps.openstack.outputs.bucket }}
|
||||
openStackBaseUrl:
|
||||
description: "OpenStack raw image base URL"
|
||||
value: ${{ steps.openstack.outputs.baseUrl }}
|
||||
openStackImagePath:
|
||||
description: "OpenStack image path"
|
||||
value: ${{ steps.openstack.outputs.imagePath }}
|
||||
qemuJsonOutput:
|
||||
description: "QEMU image json output path"
|
||||
value: ${{ steps.qemu.outputs.jsonOutput }}
|
||||
@ -270,6 +282,18 @@ runs:
|
||||
echo "imageFamily=constellation-${ref::45}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Configure OpenStack input variables
|
||||
id: openstack
|
||||
if: inputs.csp == 'openstack'
|
||||
shell: bash
|
||||
env:
|
||||
basePath: ${{ inputs.basePath }}
|
||||
run: |
|
||||
echo "bucket=cdn-constellation-backend" >> $GITHUB_OUTPUT
|
||||
echo "baseUrl=https://cdn.confidential.cloud" >> $GITHUB_OUTPUT
|
||||
echo "imagePath=${basePath}/mkosi.output.openstack/fedora~37/image.raw" >> $GITHUB_OUTPUT
|
||||
echo "jsonOutput=${basePath}/mkosi.output.openstack/fedora~37/image-upload.json" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Configure QEMU input variables
|
||||
id: qemu
|
||||
if: inputs.csp == 'qemu'
|
||||
|
38
.github/workflows/build-os-image.yml
vendored
38
.github/workflows/build-os-image.yml
vendored
@ -247,7 +247,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
csp: [aws, azure, gcp, qemu]
|
||||
csp: [aws, azure, gcp, openstack, qemu]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
|
||||
@ -359,7 +359,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
csp: [aws, azure, gcp, qemu]
|
||||
csp: [aws, azure, gcp, openstack, qemu]
|
||||
upload-variant: [""]
|
||||
include:
|
||||
- csp: azure
|
||||
@ -404,7 +404,7 @@ jobs:
|
||||
# on AWS, login is required to upload the image as AMI
|
||||
# on Azure, login is done to download the VMGS from S3
|
||||
# on QEMU, login is done to upload the image to S3
|
||||
if: matrix.csp == 'aws' || matrix.csp == 'azure' || matrix.csp == 'qemu'
|
||||
if: matrix.csp == 'aws' || matrix.csp == 'azure' || matrix.csp == 'openstack' || matrix.csp == 'qemu'
|
||||
uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # tag=v1.7.0
|
||||
with:
|
||||
role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline
|
||||
@ -516,6 +516,24 @@ jobs:
|
||||
echo -e "Uploaded Azure ${AZURE_SECURITY_TYPE} image: \n\n\`\`\`\n$(jq < "${AZURE_JSON_OUTPUT}")\n\`\`\`\n" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Upload OpenStack image
|
||||
if: matrix.csp == 'openstack'
|
||||
shell: bash
|
||||
working-directory: ${{ github.workspace }}/image
|
||||
env:
|
||||
OPENSTACK_JSON_OUTPUT: ${{ steps.vars.outputs.openStackJsonOutput }}
|
||||
OPENSTACK_BUCKET: ${{ steps.vars.outputs.openStackBucket }}
|
||||
OPENSTACK_BASE_URL: ${{ steps.vars.outputs.openStackBaseUrl }}
|
||||
OPENSTACK_IMAGE_PATH: ${{ steps.vars.outputs.openStackImagePath }}
|
||||
REF: ${{needs.build-settings.outputs.ref }}
|
||||
STREAM: ${{needs.build-settings.outputs.stream }}
|
||||
IMAGE_VERSION: ${{needs.build-settings.outputs.imageVersion }}
|
||||
run: |
|
||||
echo "::group::Upload OpenStack image"
|
||||
upload/upload_openstack.sh
|
||||
echo -e "Uploaded OpenStack image: \n\n\`\`\`\n$(jq < "${OPENSTACK_JSON_OUTPUT}")\n\`\`\`\n" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Upload QEMU image
|
||||
if: matrix.csp == 'qemu'
|
||||
shell: bash
|
||||
@ -550,7 +568,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
csp: [aws, azure, gcp, qemu]
|
||||
csp: [aws, azure, gcp, openstack, qemu]
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
|
||||
@ -672,6 +690,18 @@ jobs:
|
||||
.measurements.15.warnOnly = false' \
|
||||
-I 0 -o json -i "${{ github.workspace }}/pcrs-${{ matrix.csp }}.json"
|
||||
;;
|
||||
openstack)
|
||||
yq e '.csp = "OpenStack" |
|
||||
.image = "${{ needs.build-settings.outputs.imageNameShort }}" |
|
||||
.measurements.4.warnOnly = false |
|
||||
.measurements.8.warnOnly = false |
|
||||
.measurements.9.warnOnly = false |
|
||||
.measurements.11.warnOnly = false |
|
||||
.measurements.12.warnOnly = false |
|
||||
.measurements.13.warnOnly = false |
|
||||
.measurements.15.warnOnly = false' \
|
||||
-I 0 -o json -i "${{ github.workspace }}/pcrs-${{ matrix.csp }}.json"
|
||||
;;
|
||||
qemu)
|
||||
yq e '.csp = "QEMU" |
|
||||
.image = "${{ needs.build-settings.outputs.imageNameShort }}" |
|
||||
|
@ -8,6 +8,7 @@ package cloudcmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
@ -82,6 +83,13 @@ func (c *Creator) Create(ctx context.Context, provider cloudprovider.Provider, c
|
||||
}
|
||||
defer cl.RemoveInstaller()
|
||||
return c.createAzure(ctx, cl, config, insType, controlPlaneCount, workerCount, image)
|
||||
case cloudprovider.OpenStack:
|
||||
cl, err := c.newTerraformClient(ctx)
|
||||
if err != nil {
|
||||
return clusterid.File{}, err
|
||||
}
|
||||
defer cl.RemoveInstaller()
|
||||
return c.createOpenStack(ctx, cl, config, controlPlaneCount, workerCount, image)
|
||||
case cloudprovider.QEMU:
|
||||
if runtime.GOARCH != "amd64" || runtime.GOOS != "linux" {
|
||||
return clusterid.File{}, fmt.Errorf("creation of a QEMU based Constellation is not supported for %s/%s", runtime.GOOS, runtime.GOARCH)
|
||||
@ -241,6 +249,56 @@ func normalizeAzureURIs(vars terraform.AzureClusterVariables) terraform.AzureClu
|
||||
return vars
|
||||
}
|
||||
|
||||
func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, config *config.Config,
|
||||
controlPlaneCount, workerCount int, image string,
|
||||
) (idFile clusterid.File, retErr error) {
|
||||
// TODO: Remove this once OpenStack is supported.
|
||||
if os.Getenv("CONSTELLATION_OPENSTACK_DEV") != "1" {
|
||||
return clusterid.File{}, errors.New("OpenStack isn't supported yet")
|
||||
}
|
||||
if _, hasOSAuthURL := os.LookupEnv("OS_AUTH_URL"); !hasOSAuthURL && config.Provider.OpenStack.Cloud == "" {
|
||||
return clusterid.File{}, errors.New(
|
||||
"neither environment variable OS_AUTH_URL nor cloud name for \"clouds.yaml\" is set. OpenStack authentication requires a set of " +
|
||||
"OS_* environment variables that are typically sourced into the current shell with an openrc file " +
|
||||
"or a cloud name for \"clouds.yaml\". " +
|
||||
"See https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html for more information",
|
||||
)
|
||||
}
|
||||
|
||||
vars := terraform.OpenStackClusterVariables{
|
||||
CommonVariables: terraform.CommonVariables{
|
||||
Name: config.Name,
|
||||
CountControlPlanes: controlPlaneCount,
|
||||
CountWorkers: workerCount,
|
||||
StateDiskSizeGB: config.StateDiskSizeGB,
|
||||
},
|
||||
Cloud: config.Provider.OpenStack.Cloud,
|
||||
AvailabilityZone: config.Provider.OpenStack.AvailabilityZone,
|
||||
FloatingIPPoolID: config.Provider.OpenStack.FloatingIPPoolID,
|
||||
FlavorID: config.Provider.OpenStack.FlavorID,
|
||||
ImageURL: image,
|
||||
DirectDownload: *config.Provider.OpenStack.DirectDownload,
|
||||
Debug: config.IsDebugCluster(),
|
||||
}
|
||||
|
||||
if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.OpenStack.String())), &vars); err != nil {
|
||||
return clusterid.File{}, err
|
||||
}
|
||||
|
||||
defer rollbackOnError(context.Background(), c.out, &retErr, &rollbackerTerraform{client: cl})
|
||||
tfOutput, err := cl.CreateCluster(ctx)
|
||||
if err != nil {
|
||||
return clusterid.File{}, err
|
||||
}
|
||||
|
||||
return clusterid.File{
|
||||
CloudProvider: cloudprovider.OpenStack,
|
||||
IP: tfOutput.IP,
|
||||
InitSecret: []byte(tfOutput.Secret),
|
||||
UID: tfOutput.UID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, config *config.Config,
|
||||
controlPlaneCount, workerCount int, source string,
|
||||
) (idFile clusterid.File, retErr error) {
|
||||
|
@ -24,7 +24,7 @@ import (
|
||||
|
||||
func newConfigGenerateCmd() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "generate {aws|azure|gcp|qemu}",
|
||||
Use: "generate {aws|azure|gcp|openstack|qemu}",
|
||||
Short: "Generate a default configuration file",
|
||||
Long: "Generate a default configuration file for your selected cloud provider.",
|
||||
Args: cobra.MatchAll(
|
||||
|
@ -119,6 +119,9 @@ func (c *createCmd) create(cmd *cobra.Command, creator cloudCreator, fileHandler
|
||||
case cloudprovider.GCP:
|
||||
c.log.Debugf("Configuring instance type for GCP")
|
||||
instanceType = conf.Provider.GCP.InstanceType
|
||||
case cloudprovider.OpenStack:
|
||||
c.log.Debugf("Configuring instance type for OpenStack")
|
||||
instanceType = conf.Provider.OpenStack.FlavorID
|
||||
case cloudprovider.QEMU:
|
||||
c.log.Debugf("Configuring instance type for QEMU")
|
||||
cpus := conf.Provider.QEMU.VCPUs
|
||||
|
@ -98,6 +98,8 @@ func variant(provider cloudprovider.Provider, config *config.Config) (string, er
|
||||
|
||||
case cloudprovider.GCP:
|
||||
return "sev-es", nil
|
||||
case cloudprovider.OpenStack:
|
||||
return "sev", nil
|
||||
case cloudprovider.QEMU:
|
||||
return "default", nil
|
||||
default:
|
||||
@ -139,6 +141,8 @@ func getReferenceFromImageInfo(provider cloudprovider.Provider, variant string,
|
||||
providerList = imgInfo.Azure
|
||||
case cloudprovider.GCP:
|
||||
providerList = imgInfo.GCP
|
||||
case cloudprovider.OpenStack:
|
||||
providerList = imgInfo.OpenStack
|
||||
case cloudprovider.QEMU:
|
||||
providerList = imgInfo.QEMU
|
||||
default:
|
||||
|
@ -53,6 +53,12 @@ func TestGetReference(t *testing.T) {
|
||||
variant: "someVariant",
|
||||
wantReference: "someReference",
|
||||
},
|
||||
"reference exists openstack": {
|
||||
info: versionsapi.ImageInfo{OpenStack: map[string]string{"someVariant": "someReference"}},
|
||||
provider: cloudprovider.OpenStack,
|
||||
variant: "someVariant",
|
||||
wantReference: "someReference",
|
||||
},
|
||||
"reference exists qemu": {
|
||||
info: versionsapi.ImageInfo{QEMU: map[string]string{"someVariant": "someReference"}},
|
||||
provider: cloudprovider.QEMU,
|
||||
@ -141,6 +147,13 @@ func TestVariant(t *testing.T) {
|
||||
}},
|
||||
wantVariant: "sev-es",
|
||||
},
|
||||
"OpenStack": {
|
||||
csp: cloudprovider.OpenStack,
|
||||
config: &config.Config{Image: "someImage", Provider: config.ProviderConfig{
|
||||
OpenStack: &config.OpenStackConfig{},
|
||||
}},
|
||||
wantVariant: "sev",
|
||||
},
|
||||
"QEMU": {
|
||||
csp: cloudprovider.QEMU,
|
||||
config: &config.Config{Image: "someImage", Provider: config.ProviderConfig{
|
||||
|
@ -0,0 +1,52 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/random" {
|
||||
version = "3.4.3"
|
||||
constraints = "3.4.3"
|
||||
hashes = [
|
||||
"h1:hV66lcagXXRwwCW3Y542bI1JgPo8z/taYKT7K+a2Z5U=",
|
||||
"h1:hXUPrH8igYBhatzatkp80RCeeUJGu9lQFDyKemOlsTo=",
|
||||
"h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=",
|
||||
"h1:tL3katm68lX+4lAncjQA9AXL4GR/VM+RPwqYf4D2X8Q=",
|
||||
"h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=",
|
||||
"zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752",
|
||||
"zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b",
|
||||
"zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53",
|
||||
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
|
||||
"zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3",
|
||||
"zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5",
|
||||
"zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda",
|
||||
"zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6",
|
||||
"zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1",
|
||||
"zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d",
|
||||
"zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8",
|
||||
"zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93",
|
||||
]
|
||||
}
|
||||
|
||||
provider "registry.terraform.io/terraform-provider-openstack/openstack" {
|
||||
version = "1.48.0"
|
||||
constraints = "1.48.0"
|
||||
hashes = [
|
||||
"h1:0Oy4KDG/+l4tKeB+kYglsrFnghsAVJr60YTAbLkhfWc=",
|
||||
"h1:1OTZZtFI/HIdp052eDSmjZ29d/4YP0emMkH2VzQs/P0=",
|
||||
"h1:Q1+17/v0+xpNDUvEqVOX9UqYTwb1suAdH73ObnIVepc=",
|
||||
"h1:mTlCzugRpavDX3IG2zAs6ZainqpTUpNnQKbQuR523NA=",
|
||||
"h1:qjf/qyH9oKOMujQk59bNxV8yLRbUhmihxMRrKOeA8qI=",
|
||||
"zh:1fe237fa1153e05879fd26857416a1d029a3f108e32e83c4931dd874c777aa6a",
|
||||
"zh:2c4587b4c810d569aafd69e287ecc2ee910e6c16cfc784e49861e0a8066b8655",
|
||||
"zh:3f1a42fce3c925afeeaa96efae0bc9be95acfc80ba147a8123d03038d429df6b",
|
||||
"zh:430511b62dc2fdafa070e9bd88e5e1fc39b3d667151aa9bf8e21b2c2c5421281",
|
||||
"zh:4452279f6f23d3f2c5969deebf24ae2c38af8e02d52ee589b658c52b321835e5",
|
||||
"zh:5525d1ca817f28ec9f0f648ea38b94fd0741130eaed2260bbd734efd03aecfb8",
|
||||
"zh:675001e8cec8d0d4f006ce01b0608b7c5a378b4e56c6a27fbf5562f04371de70",
|
||||
"zh:6c0f4da6da81da562e16af6fbb36035c0797de2a0384d0ef7c9a8b4676f8eca9",
|
||||
"zh:79db708664ecbcf9d1a6d20e6a294716bff21a2641a8f58bfce60f3d11b944ef",
|
||||
"zh:7bfc5ee6765694779fbfc00954fe04795035e85dfefd916dc6601717116b7005",
|
||||
"zh:899a17c1547aa1bf732a55c903f3df25c8a0c107c16e0753677aecb8ed32130c",
|
||||
"zh:9e02fb5267dc415a763ef55a24f3890f7e63de8d61e05e220d90a5a4a4b891ed",
|
||||
"zh:a224e6e677e92cd31d0806a2d11c9bb17d032eaa0086e2aa8136ae0e9ce2fa83",
|
||||
"zh:b3905869f6fea27ffd144eb8221ea67aeca63e23c06af43a221e55634faef3e2",
|
||||
]
|
||||
}
|
245
cli/internal/terraform/terraform/openstack/main.tf
Normal file
245
cli/internal/terraform/terraform/openstack/main.tf
Normal file
@ -0,0 +1,245 @@
|
||||
terraform {
|
||||
required_providers {
|
||||
openstack = {
|
||||
source = "terraform-provider-openstack/openstack"
|
||||
version = "1.48.0"
|
||||
}
|
||||
|
||||
random = {
|
||||
source = "hashicorp/random"
|
||||
version = "3.4.3"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "openstack" {
|
||||
cloud = var.cloud
|
||||
}
|
||||
|
||||
locals {
|
||||
uid = random_id.uid.hex
|
||||
name = "${var.name}-${local.uid}"
|
||||
initSecretHash = random_password.initSecret.bcrypt_hash
|
||||
ports_node_range_start = "30000"
|
||||
ports_node_range_end = "32767"
|
||||
ports_kubernetes = "6443"
|
||||
ports_bootstrapper = "9000"
|
||||
ports_konnectivity = "8132"
|
||||
ports_verify = "30081"
|
||||
ports_recovery = "9999"
|
||||
ports_debugd = "4000"
|
||||
cidr_vpc_subnet_nodes = "192.168.178.0/24"
|
||||
tags = ["constellation-uid-${local.uid}"]
|
||||
}
|
||||
|
||||
resource "random_id" "uid" {
|
||||
byte_length = 4
|
||||
}
|
||||
|
||||
resource "random_password" "initSecret" {
|
||||
length = 32
|
||||
special = true
|
||||
override_special = "_%@"
|
||||
}
|
||||
|
||||
resource "openstack_images_image_v2" "constellation_os_image" {
|
||||
name = local.name
|
||||
image_source_url = var.image_url
|
||||
web_download = var.direct_download
|
||||
container_format = "bare"
|
||||
disk_format = "raw"
|
||||
visibility = "private"
|
||||
properties = {
|
||||
hw_firmware_type = "uefi"
|
||||
os_type = "linux"
|
||||
}
|
||||
}
|
||||
|
||||
data "openstack_networking_network_v2" "floating_ip_pool" {
|
||||
network_id = var.floating_ip_pool_id
|
||||
}
|
||||
|
||||
resource "openstack_networking_network_v2" "vpc_network" {
|
||||
name = local.name
|
||||
description = "Constellation VPC network"
|
||||
tags = local.tags
|
||||
}
|
||||
|
||||
resource "openstack_networking_subnet_v2" "vpc_subnetwork" {
|
||||
name = local.name
|
||||
description = "Constellation VPC subnetwork"
|
||||
network_id = openstack_networking_network_v2.vpc_network.id
|
||||
cidr = local.cidr_vpc_subnet_nodes
|
||||
dns_nameservers = [
|
||||
"1.1.1.1",
|
||||
"8.8.8.8",
|
||||
"9.9.9.9",
|
||||
]
|
||||
tags = local.tags
|
||||
}
|
||||
|
||||
resource "openstack_networking_router_v2" "vpc_router" {
|
||||
name = local.name
|
||||
external_network_id = data.openstack_networking_network_v2.floating_ip_pool.network_id
|
||||
}
|
||||
|
||||
resource "openstack_networking_router_interface_v2" "vpc_router_interface" {
|
||||
router_id = openstack_networking_router_v2.vpc_router.id
|
||||
subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id
|
||||
}
|
||||
|
||||
resource "openstack_compute_secgroup_v2" "vpc_secgroup" {
|
||||
name = local.name
|
||||
description = "Constellation VPC security group"
|
||||
|
||||
rule {
|
||||
from_port = -1
|
||||
to_port = -1
|
||||
ip_protocol = "icmp"
|
||||
self = true
|
||||
}
|
||||
|
||||
rule {
|
||||
from_port = local.ports_node_range_start
|
||||
to_port = local.ports_node_range_end
|
||||
ip_protocol = "tcp"
|
||||
cidr = "0.0.0.0/0"
|
||||
}
|
||||
|
||||
dynamic "rule" {
|
||||
for_each = flatten([
|
||||
local.ports_kubernetes,
|
||||
local.ports_bootstrapper,
|
||||
local.ports_konnectivity,
|
||||
local.ports_verify,
|
||||
local.ports_recovery,
|
||||
var.debug ? [local.ports_debugd] : [],
|
||||
])
|
||||
content {
|
||||
from_port = rule.value
|
||||
to_port = rule.value
|
||||
ip_protocol = "tcp"
|
||||
cidr = "0.0.0.0/0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module "instance_group_control_plane" {
|
||||
source = "./modules/instance_group"
|
||||
name = local.name
|
||||
role = "ControlPlane"
|
||||
instance_count = var.control_plane_count
|
||||
image_id = openstack_images_image_v2.constellation_os_image.image_id
|
||||
flavor_id = var.flavor_id
|
||||
security_groups = [
|
||||
openstack_compute_secgroup_v2.vpc_secgroup.id,
|
||||
]
|
||||
tags = local.tags
|
||||
disk_size = var.state_disk_size
|
||||
availability_zone = var.availability_zone
|
||||
network_id = openstack_networking_network_v2.vpc_network.id
|
||||
init_secret_hash = local.initSecretHash
|
||||
}
|
||||
|
||||
module "instance_group_worker" {
|
||||
source = "./modules/instance_group"
|
||||
name = local.name
|
||||
role = "Worker"
|
||||
instance_count = var.worker_count
|
||||
image_id = openstack_images_image_v2.constellation_os_image.image_id
|
||||
flavor_id = var.flavor_id
|
||||
tags = local.tags
|
||||
security_groups = [
|
||||
openstack_compute_secgroup_v2.vpc_secgroup.id,
|
||||
]
|
||||
disk_size = var.state_disk_size
|
||||
availability_zone = var.availability_zone
|
||||
network_id = openstack_networking_network_v2.vpc_network.id
|
||||
init_secret_hash = local.initSecretHash
|
||||
}
|
||||
|
||||
resource "openstack_networking_floatingip_v2" "public_ip" {
|
||||
pool = data.openstack_networking_network_v2.floating_ip_pool.name
|
||||
description = "Public ip for first control plane node"
|
||||
tags = local.tags
|
||||
}
|
||||
|
||||
|
||||
resource "openstack_compute_floatingip_associate_v2" "public_ip_associate" {
|
||||
floating_ip = openstack_networking_floatingip_v2.public_ip.address
|
||||
instance_id = module.instance_group_control_plane.instance_ids.0
|
||||
depends_on = [
|
||||
openstack_networking_router_v2.vpc_router,
|
||||
openstack_networking_router_interface_v2.vpc_router_interface,
|
||||
]
|
||||
}
|
||||
|
||||
# TODO: get LoadBalancer API enabled in the test environment
|
||||
|
||||
# resource "openstack_lb_loadbalancer_v2" "loadbalancer" {
|
||||
# name = local.name
|
||||
# description = "Constellation load balancer"
|
||||
# vip_subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id
|
||||
# }
|
||||
|
||||
|
||||
# resource "openstack_networking_floatingip_v2" "loadbalancer_ip" {
|
||||
# pool = data.openstack_networking_network_v2.floating_ip_pool.name
|
||||
# description = "Loadbalancer ip for ${local.name}"
|
||||
# tags = local.tags
|
||||
# }
|
||||
|
||||
# module "loadbalancer_kube" {
|
||||
# source = "./modules/loadbalancer"
|
||||
# name = "${local.name}-kube"
|
||||
# member_ips = module.instance_group_control_plane.ips.value
|
||||
# loadbalancer_id = openstack_lb_loadbalancer_v2.loadbalancer.id
|
||||
# subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id
|
||||
# port = local.ports_kubernetes
|
||||
# }
|
||||
|
||||
# module "loadbalancer_boot" {
|
||||
# source = "./modules/loadbalancer"
|
||||
# name = "${local.name}-boot"
|
||||
# member_ips = module.instance_group_control_plane.ips
|
||||
# loadbalancer_id = openstack_lb_loadbalancer_v2.loadbalancer.id
|
||||
# subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id
|
||||
# port = local.ports_bootstrapper
|
||||
# }
|
||||
|
||||
# module "loadbalancer_verify" {
|
||||
# source = "./modules/loadbalancer"
|
||||
# name = "${local.name}-verify"
|
||||
# member_ips = module.instance_group_control_plane.ips
|
||||
# loadbalancer_id = openstack_lb_loadbalancer_v2.loadbalancer.id
|
||||
# subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id
|
||||
# port = local.ports_verify
|
||||
# }
|
||||
|
||||
# module "loadbalancer_konnectivity" {
|
||||
# source = "./modules/loadbalancer"
|
||||
# name = "${local.name}-konnectivity"
|
||||
# member_ips = module.instance_group_control_plane.ips
|
||||
# loadbalancer_id = openstack_lb_loadbalancer_v2.loadbalancer.id
|
||||
# subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id
|
||||
# port = local.ports_konnectivity
|
||||
# }
|
||||
|
||||
# module "loadbalancer_recovery" {
|
||||
# source = "./modules/loadbalancer"
|
||||
# name = "${local.name}-recovery"
|
||||
# member_ips = module.instance_group_control_plane.ips
|
||||
# loadbalancer_id = openstack_lb_loadbalancer_v2.loadbalancer.id
|
||||
# subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id
|
||||
# port = local.ports_recovery
|
||||
# }
|
||||
|
||||
# module "loadbalancer_debugd" {
|
||||
# count = var.debug ? 1 : 0 // only deploy debugd in debug mode
|
||||
# source = "./modules/loadbalancer"
|
||||
# name = "${local.name}-debugd"
|
||||
# member_ips = module.instance_group_control_plane.ips
|
||||
# loadbalancer_id = openstack_lb_loadbalancer_v2.loadbalancer.id
|
||||
# subnet_id = openstack_networking_subnet_v2.vpc_subnetwork.id
|
||||
# port = local.ports_debugd
|
||||
# }
|
@ -0,0 +1,54 @@
|
||||
terraform {
|
||||
required_providers {
|
||||
openstack = {
|
||||
source = "terraform-provider-openstack/openstack"
|
||||
version = "1.48.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
locals {
|
||||
role_dashed = var.role == "ControlPlane" ? "control-plane" : "worker"
|
||||
name = "${var.name}-${local.role_dashed}"
|
||||
tags = distinct(sort(concat(var.tags, ["constellation-role-${local.role_dashed}"])))
|
||||
}
|
||||
|
||||
# TODO: get this API enabled in the test environment
|
||||
# resource "openstack_compute_servergroup_v2" "instance_group" {
|
||||
# name = local.name
|
||||
# policies = ["soft-anti-affinity"]
|
||||
# }
|
||||
|
||||
resource "openstack_compute_instance_v2" "instance_group_member" {
|
||||
name = "${local.name}-${count.index}"
|
||||
count = var.instance_count
|
||||
image_id = var.image_id
|
||||
flavor_id = var.flavor_id
|
||||
security_groups = var.security_groups
|
||||
tags = local.tags
|
||||
# TODO: get this API enabled in the test environment
|
||||
# scheduler_hints {
|
||||
# group = openstack_compute_servergroup_v2.instance_group.id
|
||||
# }
|
||||
network {
|
||||
uuid = var.network_id
|
||||
}
|
||||
block_device {
|
||||
uuid = var.image_id
|
||||
source_type = "image"
|
||||
destination_type = "local"
|
||||
boot_index = 0
|
||||
delete_on_termination = true
|
||||
}
|
||||
block_device {
|
||||
source_type = "blank"
|
||||
destination_type = "volume"
|
||||
volume_size = var.disk_size
|
||||
boot_index = 1
|
||||
delete_on_termination = true
|
||||
}
|
||||
metadata = {
|
||||
constellation-init-secret-hash = var.init_secret_hash
|
||||
}
|
||||
availability_zone_hints = var.availability_zone
|
||||
}
|
@ -0,0 +1,11 @@
|
||||
output "instance_group" {
|
||||
value = local.name
|
||||
}
|
||||
|
||||
output "ips" {
|
||||
value = openstack_compute_instance_v2.instance_group_member.*.access_ip_v4
|
||||
}
|
||||
|
||||
output "instance_ids" {
|
||||
value = openstack_compute_instance_v2.instance_group_member.*.id
|
||||
}
|
@ -0,0 +1,58 @@
|
||||
variable "name" {
|
||||
type = string
|
||||
description = "Base name of the instance group."
|
||||
}
|
||||
|
||||
variable "role" {
|
||||
type = string
|
||||
description = "The role of the instance group."
|
||||
validation {
|
||||
condition = contains(["ControlPlane", "Worker"], var.role)
|
||||
error_message = "The role has to be 'ControlPlane' or 'Worker'."
|
||||
}
|
||||
}
|
||||
|
||||
variable "instance_count" {
|
||||
type = number
|
||||
description = "Number of instances in the instance group."
|
||||
}
|
||||
|
||||
variable "image_id" {
|
||||
type = string
|
||||
description = "Image ID for the nodes."
|
||||
}
|
||||
|
||||
variable "flavor_id" {
|
||||
type = string
|
||||
description = "Flavor ID (machine type) to use for the nodes."
|
||||
}
|
||||
|
||||
variable "security_groups" {
|
||||
type = list(string)
|
||||
description = "Security groups to place the nodes in."
|
||||
}
|
||||
|
||||
variable "tags" {
|
||||
type = list(string)
|
||||
description = "Tags to attach to each node."
|
||||
}
|
||||
|
||||
variable "disk_size" {
|
||||
type = number
|
||||
description = "Disk size for the nodes, in GiB."
|
||||
}
|
||||
|
||||
variable "availability_zone" {
|
||||
type = string
|
||||
description = "The availability zone to deploy the nodes in."
|
||||
}
|
||||
|
||||
variable "network_id" {
|
||||
type = string
|
||||
description = "Network ID to attach each node to."
|
||||
}
|
||||
|
||||
variable "init_secret_hash" {
|
||||
type = string
|
||||
description = "Hash of the init secret."
|
||||
}
|
@ -0,0 +1,40 @@
|
||||
terraform {
|
||||
required_providers {
|
||||
openstack = {
|
||||
source = "terraform-provider-openstack/openstack"
|
||||
version = "1.48.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "openstack_lb_listener_v2" "listener" {
|
||||
name = var.name
|
||||
protocol = "TCP"
|
||||
protocol_port = var.port
|
||||
loadbalancer_id = var.loadbalancer_id
|
||||
}
|
||||
|
||||
resource "openstack_lb_pool_v2" "pool" {
|
||||
name = var.name
|
||||
protocol = "TCP"
|
||||
lb_method = "ROUND_ROBIN"
|
||||
listener_id = openstack_lb_listener_v2.listener.id
|
||||
}
|
||||
|
||||
resource "openstack_lb_member_v2" "member" {
|
||||
count = length(var.member_ips)
|
||||
name = format("${var.name}-member-%02d", count.index + 1)
|
||||
address = var.member_ips[count.index]
|
||||
protocol_port = var.port
|
||||
pool_id = openstack_lb_pool_v2.pool.id
|
||||
subnet_id = var.subnet_id
|
||||
}
|
||||
|
||||
resource "openstack_lb_monitor_v2" "k8s_api" {
|
||||
name = var.name
|
||||
pool_id = openstack_lb_pool_v2.pool.id
|
||||
type = "TCP"
|
||||
delay = 2
|
||||
timeout = 2
|
||||
max_retries = 2
|
||||
}
|
@ -0,0 +1,25 @@
|
||||
variable "name" {
|
||||
type = string
|
||||
description = "Base name of the load balancer rule."
|
||||
}
|
||||
|
||||
variable "member_ips" {
|
||||
type = list(string)
|
||||
description = "The IP addresses of the members of the load balancer pool."
|
||||
default = []
|
||||
}
|
||||
|
||||
variable "loadbalancer_id" {
|
||||
type = string
|
||||
description = "The ID of the load balancer."
|
||||
}
|
||||
|
||||
variable "subnet_id" {
|
||||
type = string
|
||||
description = "The ID of the members subnet."
|
||||
}
|
||||
|
||||
variable "port" {
|
||||
type = number
|
||||
description = "The port on which to listen for incoming traffic."
|
||||
}
|
12
cli/internal/terraform/terraform/openstack/outputs.tf
Normal file
12
cli/internal/terraform/terraform/openstack/outputs.tf
Normal file
@ -0,0 +1,12 @@
|
||||
output "ip" {
|
||||
value = openstack_networking_floatingip_v2.public_ip.address
|
||||
}
|
||||
|
||||
output "uid" {
|
||||
value = local.uid
|
||||
}
|
||||
|
||||
output "initSecret" {
|
||||
value = random_password.initSecret.result
|
||||
sensitive = true
|
||||
}
|
58
cli/internal/terraform/terraform/openstack/variables.tf
Normal file
58
cli/internal/terraform/terraform/openstack/variables.tf
Normal file
@ -0,0 +1,58 @@
|
||||
variable "cloud" {
|
||||
type = string
|
||||
default = null
|
||||
description = "The cloud to use within the OpenStack \"clouds.yaml\" file. Optional. If not set, environment variables are used."
|
||||
}
|
||||
|
||||
variable "name" {
|
||||
type = string
|
||||
default = "constell"
|
||||
description = "Base name of the cluster."
|
||||
}
|
||||
|
||||
variable "control_plane_count" {
|
||||
type = number
|
||||
description = "The number of control plane nodes to deploy."
|
||||
}
|
||||
|
||||
variable "worker_count" {
|
||||
type = number
|
||||
description = "The number of worker nodes to deploy."
|
||||
}
|
||||
|
||||
variable "state_disk_size" {
|
||||
type = number
|
||||
default = 30
|
||||
description = "The size of the state disk in GB."
|
||||
}
|
||||
|
||||
variable "availability_zone" {
|
||||
type = string
|
||||
description = "The availability zone to deploy the nodes in."
|
||||
}
|
||||
|
||||
variable "image_url" {
|
||||
type = string
|
||||
description = "The image to use for cluster nodes."
|
||||
}
|
||||
|
||||
variable "direct_download" {
|
||||
type = bool
|
||||
description = "If enabled, downloads OS image directly from source URL to OpenStack. Otherwise, downloads image to local machine and uploads to OpenStack."
|
||||
}
|
||||
|
||||
variable "flavor_id" {
|
||||
type = string
|
||||
description = "The flavor (machine type) to use for cluster nodes."
|
||||
}
|
||||
|
||||
variable "floating_ip_pool_id" {
|
||||
type = string
|
||||
description = "The pool (network name) to use for floating IPs."
|
||||
}
|
||||
|
||||
variable "debug" {
|
||||
type = bool
|
||||
default = false
|
||||
description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
|
||||
}
|
@ -216,6 +216,46 @@ func (v *AzureIAMVariables) String() string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// OpenStackClusterVariables is user configuration for creating a cluster with Terraform on OpenStack.
|
||||
type OpenStackClusterVariables struct {
|
||||
// CommonVariables contains common variables.
|
||||
CommonVariables
|
||||
|
||||
// Cloud is the (optional) name of the OpenStack cloud to use when reading the "clouds.yaml" configuration file. If empty, environment variables are used.
|
||||
Cloud string
|
||||
// AvailabilityZone is the OpenStack availability zone to use.
|
||||
AvailabilityZone string
|
||||
// Flavor is the ID of the OpenStack flavor (machine type) to use.
|
||||
FlavorID string
|
||||
// FloatingIPPoolID is the ID of the OpenStack floating IP pool to use for public IPs.
|
||||
FloatingIPPoolID string
|
||||
// ImageURL is the URL of the OpenStack image to use.
|
||||
ImageURL string
|
||||
// DirectDownload decides whether to download the image directly from the URL to OpenStack or to upload it from the local machine.
|
||||
DirectDownload bool
|
||||
// Debug is true if debug mode is enabled.
|
||||
Debug bool
|
||||
}
|
||||
|
||||
// String returns a string representation of the variables, formatted as Terraform variables.
|
||||
func (v *OpenStackClusterVariables) String() string {
|
||||
b := &strings.Builder{}
|
||||
b.WriteString(v.CommonVariables.String())
|
||||
if v.Cloud != "" {
|
||||
writeLinef(b, "cloud = %q", v.Cloud)
|
||||
}
|
||||
writeLinef(b, "availability_zone = %q", v.AvailabilityZone)
|
||||
writeLinef(b, "flavor_id = %q", v.FlavorID)
|
||||
writeLinef(b, "floating_ip_pool_id = %q", v.FloatingIPPoolID)
|
||||
writeLinef(b, "image_url = %q", v.ImageURL)
|
||||
writeLinef(b, "direct_download = %t", v.DirectDownload)
|
||||
writeLinef(b, "debug = %t", v.Debug)
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// TODO: Add support for OpenStack IAM variables.
|
||||
|
||||
// QEMUVariables is user configuration for creating a QEMU cluster with Terraform.
|
||||
type QEMUVariables struct {
|
||||
// CommonVariables contains common variables.
|
||||
|
@ -81,6 +81,9 @@ func main() {
|
||||
defer meta.Close()
|
||||
fetcher = cloudprovider.New(meta)
|
||||
|
||||
// TODO(malt3): implement OpenStack
|
||||
// case platform.OpenStack:
|
||||
|
||||
case platform.QEMU:
|
||||
fetcher = cloudprovider.New(qemucloud.New())
|
||||
|
||||
|
@ -44,10 +44,11 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
gcpStateDiskPath = "/dev/disk/by-id/google-state-disk"
|
||||
azureStateDiskPath = "/dev/disk/azure/scsi1/lun0"
|
||||
awsStateDiskPath = "/dev/sdb"
|
||||
qemuStateDiskPath = "/dev/vda"
|
||||
gcpStateDiskPath = "/dev/disk/by-id/google-state-disk"
|
||||
azureStateDiskPath = "/dev/disk/azure/scsi1/lun0"
|
||||
awsStateDiskPath = "/dev/sdb"
|
||||
qemuStateDiskPath = "/dev/vda"
|
||||
openstackStateDiskPath = "/dev/vdb"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@ -107,6 +108,12 @@ func main() {
|
||||
defer gcpMeta.Close()
|
||||
metadataAPI = gcpMeta
|
||||
|
||||
case cloudprovider.OpenStack:
|
||||
diskPath = openstackStateDiskPath
|
||||
// TODO(malt3): implement OpenStack metadata API and quote issuer
|
||||
// issuer = ...
|
||||
// metadataAPI = ...
|
||||
|
||||
case cloudprovider.QEMU:
|
||||
diskPath = qemuStateDiskPath
|
||||
issuer = qemu.NewIssuer()
|
||||
|
@ -15,7 +15,7 @@ KERNEL_DEBUG_CMDLNE := $(if $(filter true,$(DEBUG)),constellation.d
|
||||
export INSTALL_DEBUGD ?= $(DEBUG)
|
||||
export CONSOLE_MOTD = $(AUTOLOGIN)
|
||||
-include $(CURDIR)/config.mk
|
||||
csps := aws qemu gcp azure
|
||||
csps := aws azure gcp openstack qemu
|
||||
certs := $(PKI)/PK.cer $(PKI)/KEK.cer $(PKI)/db.cer
|
||||
|
||||
AZURE_FIXED_KERNEL_RPMS := kernel-6.1.14-200.fc37.x86_64.rpm kernel-core-6.1.14-200.fc37.x86_64.rpm kernel-modules-6.1.14-200.fc37.x86_64.rpm
|
||||
|
@ -239,6 +239,31 @@ upload/upload_azure.sh -g --disk-name "${AZURE_DISK_NAME}" "${AZURE_VMGS_PATH}"
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>OpenStack</summary>
|
||||
|
||||
Note:
|
||||
|
||||
> OpenStack is not one a global cloud provider, but rather a software that can be installed on-premises.
|
||||
> This means we do not upload the image to a cloud provider, but to our CDN.
|
||||
|
||||
- Install `aws` cli (see [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html))
|
||||
- Login to AWS (see [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-quickstart.html))
|
||||
|
||||
```sh
|
||||
# set these variables
|
||||
export REF= # e.g. feat-xyz (branch name encoded with dashes)
|
||||
export STREAM= # e.g. "nightly", "debug", "stable" (depends on the type of image and if it is a release)
|
||||
export IMAGE_VERSION= # e.g. v2.1.0" or output of pseudo-version tool
|
||||
export OPENSTACK_BUCKET=cdn-constellation-backend
|
||||
export OPENSTACK_BASE_URL="https://cdn.confidential.cloud"
|
||||
export OPENSTACK_IMAGE_PATH=${PWD}/mkosi.output.qemu/fedora~37/image.raw
|
||||
export OPENSTACK_JSON_OUTPUT=${PWD}/mkosi.output.qemu/fedora~37/image-upload.json
|
||||
upload/upload_openstack.sh
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>QEMU</summary>
|
||||
|
||||
|
7
image/mkosi.files/mkosi.openstack.conf
Normal file
7
image/mkosi.files/mkosi.openstack.conf
Normal file
@ -0,0 +1,7 @@
|
||||
[Output]
|
||||
KernelCommandLine=constel.csp=openstack mem_encrypt=on kvm_amd.sev=1 module_blacklist=qemu_fw_cfg console=tty0 console=ttyS0
|
||||
OutputDirectory=mkosi.output.openstack
|
||||
|
||||
[Content]
|
||||
Autologin=yes
|
||||
Environment=CONSOLE_MOTD=true
|
@ -6,7 +6,7 @@
|
||||
set -euo pipefail
|
||||
shopt -s inherit_errexit
|
||||
|
||||
if [[ -z ${CONFIG_FILE-} ]] && [[ -f ${CONFIG_FILE-} ]]; then
|
||||
if [[ -f ${CONFIG_FILE-} ]]; then
|
||||
# shellcheck source=/dev/null
|
||||
. "${CONFIG_FILE}"
|
||||
fi
|
||||
|
@ -6,7 +6,7 @@
|
||||
set -euo pipefail
|
||||
shopt -s inherit_errexit
|
||||
|
||||
if [[ -z ${CONFIG_FILE-} ]] && [[ -f ${CONFIG_FILE-} ]]; then
|
||||
if [[ -f ${CONFIG_FILE-} ]]; then
|
||||
# shellcheck source=/dev/null
|
||||
. "${CONFIG_FILE}"
|
||||
fi
|
||||
|
@ -6,7 +6,7 @@
|
||||
set -euo pipefail
|
||||
shopt -s inherit_errexit
|
||||
|
||||
if [[ -z ${CONFIG_FILE-} ]] && [[ -f ${CONFIG_FILE-} ]]; then
|
||||
if [[ -f ${CONFIG_FILE-} ]]; then
|
||||
# shellcheck source=/dev/null
|
||||
. "${CONFIG_FILE}"
|
||||
fi
|
||||
|
22
image/upload/upload_openstack.sh
Executable file
22
image/upload/upload_openstack.sh
Executable file
@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright (c) Edgeless Systems GmbH
|
||||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
set -euo pipefail
|
||||
shopt -s inherit_errexit
|
||||
|
||||
if [[ -f ${CONFIG_FILE-} ]]; then
|
||||
# shellcheck source=/dev/null
|
||||
. "${CONFIG_FILE}"
|
||||
fi
|
||||
|
||||
path="constellation/v1/ref/${REF}/stream/${STREAM}/${IMAGE_VERSION}/image/csp/openstack/image.raw"
|
||||
aws s3 cp "${OPENSTACK_IMAGE_PATH}" "s3://${OPENSTACK_BUCKET}/${path}" --no-progress
|
||||
|
||||
image_url="${OPENSTACK_BASE_URL}/${path}"
|
||||
|
||||
json=$(jq -ncS \
|
||||
--arg image_url "${image_url}" \
|
||||
'{"openstack": {"sev": $image_url}}')
|
||||
echo -n "${json}" > "${OPENSTACK_JSON_OUTPUT}"
|
@ -6,7 +6,7 @@
|
||||
set -euo pipefail
|
||||
shopt -s inherit_errexit
|
||||
|
||||
if [[ -z ${CONFIG_FILE-} ]] && [[ -f ${CONFIG_FILE-} ]]; then
|
||||
if [[ -f ${CONFIG_FILE-} ]]; then
|
||||
# shellcheck source=/dev/null
|
||||
. "${CONFIG_FILE}"
|
||||
fi
|
||||
|
@ -25,6 +25,8 @@ const (
|
||||
Azure
|
||||
// GCP is Google Compute Platform.
|
||||
GCP
|
||||
// OpenStack is an open standard cloud computing platform.
|
||||
OpenStack
|
||||
// QEMU for a local emulated installation.
|
||||
QEMU
|
||||
)
|
||||
@ -69,6 +71,8 @@ func FromString(s string) Provider {
|
||||
return Azure
|
||||
case "gcp":
|
||||
return GCP
|
||||
case "openstack":
|
||||
return OpenStack
|
||||
case "qemu":
|
||||
return QEMU
|
||||
default:
|
||||
|
@ -35,6 +35,10 @@ func TestMarshalJSON(t *testing.T) {
|
||||
input: GCP,
|
||||
want: []byte("\"GCP\""),
|
||||
},
|
||||
"openstack": {
|
||||
input: OpenStack,
|
||||
want: []byte("\"OpenStack\""),
|
||||
},
|
||||
"qemu": {
|
||||
input: QEMU,
|
||||
want: []byte("\"QEMU\""),
|
||||
@ -79,6 +83,10 @@ func TestUnmarshalJSON(t *testing.T) {
|
||||
input: []byte("\"gcp\""),
|
||||
want: GCP,
|
||||
},
|
||||
"openstack": {
|
||||
input: []byte("\"openstack\""),
|
||||
want: OpenStack,
|
||||
},
|
||||
"qemu": {
|
||||
input: []byte("\"qemu\""),
|
||||
want: QEMU,
|
||||
@ -123,6 +131,10 @@ func TestMarshalYAML(t *testing.T) {
|
||||
input: GCP,
|
||||
want: []byte("GCP\n"),
|
||||
},
|
||||
"openstack": {
|
||||
input: OpenStack,
|
||||
want: []byte("OpenStack\n"),
|
||||
},
|
||||
"qemu": {
|
||||
input: QEMU,
|
||||
want: []byte("QEMU\n"),
|
||||
@ -167,6 +179,10 @@ func TestUnmarshalYAML(t *testing.T) {
|
||||
input: []byte("gcp\n"),
|
||||
want: GCP,
|
||||
},
|
||||
"openstack": {
|
||||
input: []byte("openstack\n"),
|
||||
want: OpenStack,
|
||||
},
|
||||
"qemu": {
|
||||
input: []byte("qemu\n"),
|
||||
want: QEMU,
|
||||
@ -215,6 +231,10 @@ func TestFromString(t *testing.T) {
|
||||
input: "gcp",
|
||||
want: GCP,
|
||||
},
|
||||
"openstack": {
|
||||
input: "openstack",
|
||||
want: OpenStack,
|
||||
},
|
||||
"qemu": {
|
||||
input: "qemu",
|
||||
want: QEMU,
|
||||
|
@ -12,12 +12,13 @@ func _() {
|
||||
_ = x[AWS-1]
|
||||
_ = x[Azure-2]
|
||||
_ = x[GCP-3]
|
||||
_ = x[QEMU-4]
|
||||
_ = x[OpenStack-4]
|
||||
_ = x[QEMU-5]
|
||||
}
|
||||
|
||||
const _Provider_name = "UnknownAWSAzureGCPQEMU"
|
||||
const _Provider_name = "UnknownAWSAzureGCPOpenStackQEMU"
|
||||
|
||||
var _Provider_index = [...]uint8{0, 7, 10, 15, 18, 22}
|
||||
var _Provider_index = [...]uint8{0, 7, 10, 15, 18, 27, 31}
|
||||
|
||||
func (i Provider) String() string {
|
||||
if i >= Provider(len(_Provider_index)-1) {
|
||||
|
@ -114,6 +114,9 @@ type ProviderConfig struct {
|
||||
// Configuration for Google Cloud as provider.
|
||||
GCP *GCPConfig `yaml:"gcp,omitempty" validate:"omitempty,dive"`
|
||||
// description: |
|
||||
// Configuration for OpenStack as provider.
|
||||
OpenStack *OpenStackConfig `yaml:"openstack,omitempty" validate:"omitempty,dive"`
|
||||
// description: |
|
||||
// Configuration for QEMU as provider.
|
||||
QEMU *QEMUConfig `yaml:"qemu,omitempty" validate:"omitempty,dive"`
|
||||
}
|
||||
@ -220,6 +223,25 @@ type GCPConfig struct {
|
||||
Measurements Measurements `yaml:"measurements" validate:"required,no_placeholders"`
|
||||
}
|
||||
|
||||
// OpenStackConfig holds config information for OpenStack based Constellation deployments.
|
||||
type OpenStackConfig struct {
|
||||
// description: |
|
||||
// OpenStack cloud name to select from "clouds.yaml". Only required if config file for OpenStack is used. Fallback authentication uses environment variables. For details see: https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html.
|
||||
Cloud string `yaml:"cloud"`
|
||||
// description: |
|
||||
// Availability zone to place the VMs in. For details see: https://docs.openstack.org/nova/latest/admin/availability-zones.html
|
||||
AvailabilityZone string `yaml:"availabilityZone" validate:"required"`
|
||||
// description: |
|
||||
// Flavor ID (machine type) to use for the VMs. For details see: https://docs.openstack.org/nova/latest/admin/flavors.html
|
||||
FlavorID string `yaml:"flavorID" validate:"required"`
|
||||
// description: |
|
||||
// Floating IP pool to use for the VMs. For details see: https://docs.openstack.org/ocata/user-guide/cli-manage-ip-addresses.html
|
||||
FloatingIPPoolID string `yaml:"floatingIPPoolID" validate:"required"`
|
||||
// description: |
|
||||
// If enabled, downloads OS image directly from source URL to OpenStack. Otherwise, downloads image to local machine and uploads to OpenStack.
|
||||
DirectDownload *bool `yaml:"directDownload" validate:"required"`
|
||||
}
|
||||
|
||||
// QEMUConfig holds config information for QEMU based Constellation deployments.
|
||||
type QEMUConfig struct {
|
||||
// description: |
|
||||
@ -295,6 +317,9 @@ func Default() *Config {
|
||||
DeployCSIDriver: func() *bool { b := true; return &b }(),
|
||||
Measurements: measurements.DefaultsFor(cloudprovider.GCP),
|
||||
},
|
||||
OpenStack: &OpenStackConfig{
|
||||
DirectDownload: func() *bool { b := true; return &b }(),
|
||||
},
|
||||
QEMU: &QEMUConfig{
|
||||
ImageFormat: "raw",
|
||||
VCPUs: 2,
|
||||
@ -393,6 +418,8 @@ func (c *Config) RemoveProviderExcept(provider cloudprovider.Provider) {
|
||||
c.Provider.Azure = currentProviderConfigs.Azure
|
||||
case cloudprovider.GCP:
|
||||
c.Provider.GCP = currentProviderConfigs.GCP
|
||||
case cloudprovider.OpenStack:
|
||||
c.Provider.OpenStack = currentProviderConfigs.OpenStack
|
||||
case cloudprovider.QEMU:
|
||||
c.Provider.QEMU = currentProviderConfigs.QEMU
|
||||
default:
|
||||
@ -429,6 +456,9 @@ func (c *Config) GetProvider() cloudprovider.Provider {
|
||||
if c.Provider.GCP != nil {
|
||||
return cloudprovider.GCP
|
||||
}
|
||||
if c.Provider.OpenStack != nil {
|
||||
return cloudprovider.OpenStack
|
||||
}
|
||||
if c.Provider.QEMU != nil {
|
||||
return cloudprovider.QEMU
|
||||
}
|
||||
|
@ -11,13 +11,14 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ConfigDoc encoder.Doc
|
||||
UpgradeConfigDoc encoder.Doc
|
||||
ProviderConfigDoc encoder.Doc
|
||||
AWSConfigDoc encoder.Doc
|
||||
AzureConfigDoc encoder.Doc
|
||||
GCPConfigDoc encoder.Doc
|
||||
QEMUConfigDoc encoder.Doc
|
||||
ConfigDoc encoder.Doc
|
||||
UpgradeConfigDoc encoder.Doc
|
||||
ProviderConfigDoc encoder.Doc
|
||||
AWSConfigDoc encoder.Doc
|
||||
AzureConfigDoc encoder.Doc
|
||||
GCPConfigDoc encoder.Doc
|
||||
OpenStackConfigDoc encoder.Doc
|
||||
QEMUConfigDoc encoder.Doc
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -110,7 +111,7 @@ func init() {
|
||||
FieldName: "provider",
|
||||
},
|
||||
}
|
||||
ProviderConfigDoc.Fields = make([]encoder.Doc, 4)
|
||||
ProviderConfigDoc.Fields = make([]encoder.Doc, 5)
|
||||
ProviderConfigDoc.Fields[0].Name = "aws"
|
||||
ProviderConfigDoc.Fields[0].Type = "AWSConfig"
|
||||
ProviderConfigDoc.Fields[0].Note = ""
|
||||
@ -126,11 +127,16 @@ func init() {
|
||||
ProviderConfigDoc.Fields[2].Note = ""
|
||||
ProviderConfigDoc.Fields[2].Description = "Configuration for Google Cloud as provider."
|
||||
ProviderConfigDoc.Fields[2].Comments[encoder.LineComment] = "Configuration for Google Cloud as provider."
|
||||
ProviderConfigDoc.Fields[3].Name = "qemu"
|
||||
ProviderConfigDoc.Fields[3].Type = "QEMUConfig"
|
||||
ProviderConfigDoc.Fields[3].Name = "openstack"
|
||||
ProviderConfigDoc.Fields[3].Type = "OpenStackConfig"
|
||||
ProviderConfigDoc.Fields[3].Note = ""
|
||||
ProviderConfigDoc.Fields[3].Description = "Configuration for QEMU as provider."
|
||||
ProviderConfigDoc.Fields[3].Comments[encoder.LineComment] = "Configuration for QEMU as provider."
|
||||
ProviderConfigDoc.Fields[3].Description = "Configuration for OpenStack as provider."
|
||||
ProviderConfigDoc.Fields[3].Comments[encoder.LineComment] = "Configuration for OpenStack as provider."
|
||||
ProviderConfigDoc.Fields[4].Name = "qemu"
|
||||
ProviderConfigDoc.Fields[4].Type = "QEMUConfig"
|
||||
ProviderConfigDoc.Fields[4].Note = ""
|
||||
ProviderConfigDoc.Fields[4].Description = "Configuration for QEMU as provider."
|
||||
ProviderConfigDoc.Fields[4].Comments[encoder.LineComment] = "Configuration for QEMU as provider."
|
||||
|
||||
AWSConfigDoc.Type = "AWSConfig"
|
||||
AWSConfigDoc.Comments[encoder.LineComment] = "AWSConfig are AWS specific configuration values used by the CLI."
|
||||
@ -315,6 +321,42 @@ func init() {
|
||||
GCPConfigDoc.Fields[7].Description = "Expected confidential VM measurements."
|
||||
GCPConfigDoc.Fields[7].Comments[encoder.LineComment] = "Expected confidential VM measurements."
|
||||
|
||||
OpenStackConfigDoc.Type = "OpenStackConfig"
|
||||
OpenStackConfigDoc.Comments[encoder.LineComment] = "OpenStackConfig holds config information for OpenStack based Constellation deployments."
|
||||
OpenStackConfigDoc.Description = "OpenStackConfig holds config information for OpenStack based Constellation deployments."
|
||||
OpenStackConfigDoc.AppearsIn = []encoder.Appearance{
|
||||
{
|
||||
TypeName: "ProviderConfig",
|
||||
FieldName: "openstack",
|
||||
},
|
||||
}
|
||||
OpenStackConfigDoc.Fields = make([]encoder.Doc, 5)
|
||||
OpenStackConfigDoc.Fields[0].Name = "cloud"
|
||||
OpenStackConfigDoc.Fields[0].Type = "string"
|
||||
OpenStackConfigDoc.Fields[0].Note = ""
|
||||
OpenStackConfigDoc.Fields[0].Description = "OpenStack cloud name to select from \"clouds.yaml\". Only required if config file for OpenStack is used. Fallback authentication uses environment variables. For details see: https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html."
|
||||
OpenStackConfigDoc.Fields[0].Comments[encoder.LineComment] = "OpenStack cloud name to select from \"clouds.yaml\". Only required if config file for OpenStack is used. Fallback authentication uses environment variables. For details see: https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html."
|
||||
OpenStackConfigDoc.Fields[1].Name = "availabilityZone"
|
||||
OpenStackConfigDoc.Fields[1].Type = "string"
|
||||
OpenStackConfigDoc.Fields[1].Note = ""
|
||||
OpenStackConfigDoc.Fields[1].Description = "Availability zone to place the VMs in. For details see: https://docs.openstack.org/nova/latest/admin/availability-zones.html"
|
||||
OpenStackConfigDoc.Fields[1].Comments[encoder.LineComment] = "Availability zone to place the VMs in. For details see: https://docs.openstack.org/nova/latest/admin/availability-zones.html"
|
||||
OpenStackConfigDoc.Fields[2].Name = "flavorID"
|
||||
OpenStackConfigDoc.Fields[2].Type = "string"
|
||||
OpenStackConfigDoc.Fields[2].Note = ""
|
||||
OpenStackConfigDoc.Fields[2].Description = "Flavor ID (machine type) to use for the VMs. For details see: https://docs.openstack.org/nova/latest/admin/flavors.html"
|
||||
OpenStackConfigDoc.Fields[2].Comments[encoder.LineComment] = "Flavor ID (machine type) to use for the VMs. For details see: https://docs.openstack.org/nova/latest/admin/flavors.html"
|
||||
OpenStackConfigDoc.Fields[3].Name = "floatingIPPoolID"
|
||||
OpenStackConfigDoc.Fields[3].Type = "string"
|
||||
OpenStackConfigDoc.Fields[3].Note = ""
|
||||
OpenStackConfigDoc.Fields[3].Description = "Floating IP pool to use for the VMs. For details see: https://docs.openstack.org/ocata/user-guide/cli-manage-ip-addresses.html"
|
||||
OpenStackConfigDoc.Fields[3].Comments[encoder.LineComment] = "Floating IP pool to use for the VMs. For details see: https://docs.openstack.org/ocata/user-guide/cli-manage-ip-addresses.html"
|
||||
OpenStackConfigDoc.Fields[4].Name = "directDownload"
|
||||
OpenStackConfigDoc.Fields[4].Type = "bool"
|
||||
OpenStackConfigDoc.Fields[4].Note = ""
|
||||
OpenStackConfigDoc.Fields[4].Description = "If enabled, downloads OS image directly from source URL to OpenStack. Otherwise, downloads image to local machine and uploads to OpenStack."
|
||||
OpenStackConfigDoc.Fields[4].Comments[encoder.LineComment] = "If enabled, downloads OS image directly from source URL to OpenStack. Otherwise, downloads image to local machine and uploads to OpenStack."
|
||||
|
||||
QEMUConfigDoc.Type = "QEMUConfig"
|
||||
QEMUConfigDoc.Comments[encoder.LineComment] = "QEMUConfig holds config information for QEMU based Constellation deployments."
|
||||
QEMUConfigDoc.Description = "QEMUConfig holds config information for QEMU based Constellation deployments."
|
||||
@ -396,6 +438,10 @@ func (_ GCPConfig) Doc() *encoder.Doc {
|
||||
return &GCPConfigDoc
|
||||
}
|
||||
|
||||
func (_ OpenStackConfig) Doc() *encoder.Doc {
|
||||
return &OpenStackConfigDoc
|
||||
}
|
||||
|
||||
func (_ QEMUConfig) Doc() *encoder.Doc {
|
||||
return &QEMUConfigDoc
|
||||
}
|
||||
@ -412,6 +458,7 @@ func GetConfigurationDoc() *encoder.FileDoc {
|
||||
&AWSConfigDoc,
|
||||
&AzureConfigDoc,
|
||||
&GCPConfigDoc,
|
||||
&OpenStackConfigDoc,
|
||||
&QEMUConfigDoc,
|
||||
},
|
||||
}
|
||||
|
@ -184,7 +184,7 @@ func TestNewWithDefaultOptions(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
const defaultErrCount = 21 // expect this number of error messages by default because user-specific values are not set and multiple providers are defined by default
|
||||
const defaultErrCount = 24 // expect this number of error messages by default because user-specific values are not set and multiple providers are defined by default
|
||||
const azErrCount = 9
|
||||
const gcpErrCount = 6
|
||||
|
||||
|
@ -130,6 +130,9 @@ func validateProvider(sl validator.StructLevel) {
|
||||
if provider.GCP != nil {
|
||||
providerCount++
|
||||
}
|
||||
if provider.OpenStack != nil {
|
||||
providerCount++
|
||||
}
|
||||
if provider.QEMU != nil {
|
||||
providerCount++
|
||||
}
|
||||
@ -163,7 +166,7 @@ func translateGCPInstanceTypeError(ut ut.Translator, fe validator.FieldError) st
|
||||
|
||||
// Validation translation functions for Provider errors.
|
||||
func registerNoProviderError(ut ut.Translator) error {
|
||||
return ut.Add("no_provider", "{0}: No provider has been defined (requires either Azure, GCP or QEMU)", true)
|
||||
return ut.Add("no_provider", "{0}: No provider has been defined (requires either Azure, GCP, OpenStack or QEMU)", true)
|
||||
}
|
||||
|
||||
func translateNoProviderError(ut ut.Translator, fe validator.FieldError) string {
|
||||
|
@ -30,6 +30,8 @@ type ImageInfo struct {
|
||||
Azure map[string]string `json:"azure,omitempty"`
|
||||
// GCP is a map of image types to GCP image IDs.
|
||||
GCP map[string]string `json:"gcp,omitempty"`
|
||||
// OpenStack is a map of image types to OpenStack image IDs.
|
||||
OpenStack map[string]string `json:"openstack,omitempty"`
|
||||
// QEMU is a map of image types to QEMU image URLs.
|
||||
QEMU map[string]string `json:"qemu,omitempty"`
|
||||
}
|
||||
@ -78,6 +80,9 @@ func (i ImageInfo) ValidateRequest() error {
|
||||
if len(i.GCP) != 0 {
|
||||
retErr = errors.Join(retErr, errors.New("GCP map must be empty for request"))
|
||||
}
|
||||
if len(i.OpenStack) != 0 {
|
||||
retErr = errors.Join(retErr, errors.New("OpenStack map must be empty for request"))
|
||||
}
|
||||
if len(i.QEMU) != 0 {
|
||||
retErr = errors.Join(retErr, errors.New("QEMU map must be empty for request"))
|
||||
}
|
||||
@ -97,17 +102,14 @@ func (i ImageInfo) Validate() error {
|
||||
if !semver.IsValid(i.Version) {
|
||||
retErr = errors.Join(retErr, fmt.Errorf("version %q is not a valid semver", i.Version))
|
||||
}
|
||||
if len(i.AWS) == 0 {
|
||||
retErr = errors.Join(retErr, errors.New("AWS map must not be empty"))
|
||||
}
|
||||
if len(i.Azure) == 0 {
|
||||
retErr = errors.Join(retErr, errors.New("Azure map must not be empty"))
|
||||
}
|
||||
if len(i.GCP) == 0 {
|
||||
retErr = errors.Join(retErr, errors.New("GCP map must not be empty"))
|
||||
}
|
||||
if len(i.QEMU) == 0 {
|
||||
retErr = errors.Join(retErr, errors.New("QEMU map must not be empty"))
|
||||
var providers int
|
||||
providers += len(i.AWS)
|
||||
providers += len(i.Azure)
|
||||
providers += len(i.GCP)
|
||||
providers += len(i.OpenStack)
|
||||
providers += len(i.QEMU)
|
||||
if providers == 0 {
|
||||
retErr = errors.Join(retErr, errors.New("one or more providers must be specified"))
|
||||
}
|
||||
|
||||
return retErr
|
||||
|
@ -129,47 +129,11 @@ func TestImageInfoValidate(t *testing.T) {
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"invalid aws": {
|
||||
"no provider": {
|
||||
info: ImageInfo{
|
||||
Ref: "test-ref",
|
||||
Stream: "nightly",
|
||||
Version: "v1.0.0",
|
||||
GCP: map[string]string{"key": "value", "key2": "value2"},
|
||||
Azure: map[string]string{"key": "value", "key2": "value2"},
|
||||
QEMU: map[string]string{"key": "value", "key2": "value2"},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"invalid gcp": {
|
||||
info: ImageInfo{
|
||||
Ref: "test-ref",
|
||||
Stream: "nightly",
|
||||
Version: "v1.0.0",
|
||||
AWS: map[string]string{"key": "value", "key2": "value2"},
|
||||
Azure: map[string]string{"key": "value", "key2": "value2"},
|
||||
QEMU: map[string]string{"key": "value", "key2": "value2"},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"invalid azure": {
|
||||
info: ImageInfo{
|
||||
Ref: "test-ref",
|
||||
Stream: "nightly",
|
||||
Version: "v1.0.0",
|
||||
AWS: map[string]string{"key": "value", "key2": "value2"},
|
||||
GCP: map[string]string{"key": "value", "key2": "value2"},
|
||||
QEMU: map[string]string{"key": "value", "key2": "value2"},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"invalid qemu": {
|
||||
info: ImageInfo{
|
||||
Ref: "test-ref",
|
||||
Stream: "nightly",
|
||||
Version: "v1.0.0",
|
||||
AWS: map[string]string{"key": "value", "key2": "value2"},
|
||||
GCP: map[string]string{"key": "value", "key2": "value2"},
|
||||
Azure: map[string]string{"key": "value", "key2": "value2"},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
@ -269,6 +233,15 @@ func TestImageInfoValidateRequest(t *testing.T) {
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"invalid openstack": {
|
||||
info: ImageInfo{
|
||||
Ref: "test-ref",
|
||||
Stream: "nightly",
|
||||
Version: "v1.0.0",
|
||||
OpenStack: map[string]string{"key": "value", "key2": "value2"},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"multiple errors": {
|
||||
info: ImageInfo{
|
||||
Ref: "",
|
||||
|
@ -67,6 +67,7 @@ Where applicable, the API uses the following CSP names:
|
||||
- `aws` for Amazon Web Services
|
||||
- `azure` for Microsoft Azure
|
||||
- `gcp` for Google Cloud Platform
|
||||
- `openstack` for OpenStack
|
||||
- `qemu` for QEMU
|
||||
|
||||
The following HTTP endpoints are available:
|
||||
@ -106,6 +107,9 @@ The image lookup table is a JSON file that maps the image name consisting of `re
|
||||
"gcp": {
|
||||
"sev-es": "gcp-image-123"
|
||||
},
|
||||
"openstack": {
|
||||
"sev": "https://cdn.confidential.cloud/constellation/v1/ref/<REF>/stream/<STREAM>/<VERSION>/image/csp/openstack/image.raw"
|
||||
},
|
||||
"qemu": {
|
||||
"default": "https://cdn.confidential.cloud/constellation/v1/ref/<REF>/stream/<STREAM>/<VERSION>/image/csp/qemu/image.raw"
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user