Git-Mirrors/veilid
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2025-08-07 14:12:37 -04:00
Code Issues Projects Releases Packages Wiki Activity

Converting CICD from Bash to Python Scripts

Browse source 
Calling this Phase 1. I've switch the build machine philosophy from
using a dedicated Digital Ocean droplet per arch to using one large
build machine and the +package-linux Earthly target which results
in .deb and .rpm packages for both amd64 and arm64/aarch64.

The script to create and delete the build machine has been migrated
to Python. I feel like the error handling is better and the delete
function now does its thing by using the specific ID of the running
build machine vs the name. Using the name would, in rare circumstances,
fail when more than one machine of the same name existed causing
duplicates to be created, all very expensive and creating larger than
normal Digital Ocean costs.

Lastly, moving the .deb and .rpm packages from the build machine
to the build orchestrator for creating and signing the repositories
now uses the Gitlab CICD artifact system verses SCP. This switch
will allow us to include the packages in the release records and
maybe streamline the Python and Crates distribution jobs in a
later phase of this project.

Changes are made in the Dry Run section off the CICD config for
testing, which will start in a few minutes and probably result in
a bunch of failed pipelines and tweaking because there's just no
way I got all of this right on the first try.
This commit is contained in:
TC Johnson 2025-03-16 11:08:58 -05:00
parent 09f7210979
commit ea0c3b6469
12 changed files with 476 additions and 126 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
scripts/cicd-python/utils/__pycache__/__init__.cpython-313.pyc
View file

Binary file not shown.

BIN
scripts/cicd-python/utils/__pycache__/droplets.cpython-313.pyc
View file

Binary file not shown.

BIN
scripts/cicd-python/utils/__pycache__/test_credentials.cpython-313.pyc
View file

Binary file not shown.

104
scripts/cicd-python/utils/droplets.py → scripts/cicd-python/utils/build_machine_control.py
View file

@ -1,20 +1,29 @@
import aiohttp
import asyncio
import sys
import json
# Define droplet configurations for different droplet types.
DROPLET_CONFIGS = {
"amd64-deb": {
"name": "build-server-amd64-deb-tmp",
"image": 179066895,
"size": "c2-16vcpu-32gb"
},
}
CONFIG_FILE = "config.json"
async def create_droplet(token: str, droplet_type: str) -> None:
config = DROPLET_CONFIGS.get(droplet_type)
if not config:
print(f"Droplet type '{droplet_type}' not recognized.", file=sys.stderr)
# Load config from file
def load_config():
try:
with open(CONFIG_FILE, "r") as f:
return json.load(f)
except (FileNotFoundError, json.JSONDecodeError):
return {}
# Save config to file
def save_config(config):
with open(CONFIG_FILE, "w") as f:
json.dump(config, f, indent=4)
async def create_build_machine(token: str) -> None:
config = load_config()
droplet_config = config.get("droplet_config", {})
if not droplet_config:
print("Droplet configuration not found.", file=sys.stderr)
sys.exit(1)
headers = {
@ -23,10 +32,10 @@ async def create_droplet(token: str, droplet_type: str) -> None:
}
create_url = "https://api.digitalocean.com/v2/droplets"
payload = {
"name": config["name"],
"region": "nyc1", # Changed default region to "ncy1"
"size": config["size"],
"image": config["image"],
"name": droplet_config["name"],
"region": "nyc1",
"size": droplet_config["size"],
"image": droplet_config["image"],
"backups": False,
}
@ -42,17 +51,23 @@ async def create_droplet(token: str, droplet_type: str) -> None:
print("No droplet information returned.", file=sys.stderr)
sys.exit("No droplet information returned.")
droplet_id = droplet.get("id")
print(f"Droplet creation initiated. Droplet ID: {droplet_id}")
print(f"Droplet created. Droplet ID: {droplet_id}")
# Poll for droplet status until it becomes "active"
# Save droplet ID to config
config["droplet_id"] = droplet_id
save_config(config)
print("Droplet ID saved to config.")
# Poll every 10 second for droplet status until it becomes "active"
status = droplet.get("status", "new")
droplet_url = f"https://api.digitalocean.com/v2/droplets/{droplet_id}"
while status != "active":
await asyncio.sleep(2)
await asyncio.sleep(10)
async with session.get(droplet_url, headers=headers) as poll_resp:
if poll_resp.status != 200:
error_text = await poll_resp.text()
print(f"Error polling droplet status: {error_text}", file=sys.stderr)
print(f"Error polling droplet status: {error_text}",
file=sys.stderr)
sys.exit(error_text)
droplet_data = await poll_resp.json()
droplet = droplet_data.get("droplet")
@ -60,7 +75,8 @@ async def create_droplet(token: str, droplet_type: str) -> None:
status = droplet.get("status", status)
print(f"Droplet status: {status}")
else:
print("Droplet data missing in polling response", file=sys.stderr)
print("Droplet data missing in polling response",
file=sys.stderr)
sys.exit("Droplet data missing in polling response")
print("Droplet is up and running.")
@ -68,46 +84,36 @@ async def create_droplet(token: str, droplet_type: str) -> None:
async with session.get(droplet_url, headers=headers) as final_resp:
if final_resp.status != 200:
error_text = await final_resp.text()
print(f"Error retrieving droplet information: {error_text}", file=sys.stderr)
print(f"Error retrieving droplet information: {error_text}",
file=sys.stderr)
sys.exit(error_text)
final_data = await final_resp.json()
print("Droplet Information:")
print(final_data)
async def delete_droplet(token: str, droplet_type: str) -> None:
config = DROPLET_CONFIGS.get(droplet_type)
if not config:
print(f"Droplet type '{droplet_type}' not recognized.", file=sys.stderr)
sys.exit(1)
async def delete_build_machine(token: str) -> None:
config = load_config()
droplet_id = config.get("droplet_id")
if not droplet_id:
print("No droplet ID found in config.", file=sys.stderr)
return
headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
}
droplets_url = "https://api.digitalocean.com/v2/droplets"
delete_url = f"https://api.digitalocean.com/v2/droplets/{droplet_id}"
async with aiohttp.ClientSession() as session:
async with session.get(droplets_url, headers=headers) as resp:
if resp.status != 200:
async with session.delete(delete_url, headers=headers) as resp:
if resp.status != 204:
error_text = await resp.text()
print(f"Error retrieving droplets: {error_text}", file=sys.stderr)
print(f"Error deleting droplet: {error_text}", file=sys.stderr)
sys.exit(error_text)
data = await resp.json()
droplets = data.get("droplets", [])
target_droplet = None
for droplet in droplets:
if droplet.get("name") == config["name"]:
target_droplet = droplet
break
if not target_droplet:
print(f"No droplet found with name '{config['name']}'.")
return
print(f"Droplet {droplet_id} deleted successfully.")
droplet_id = target_droplet.get("id")
delete_url = f"https://api.digitalocean.com/v2/droplets/{droplet_id}"
async with session.delete(delete_url, headers=headers) as delete_resp:
if delete_resp.status != 204:
error_text = await delete_resp.text()
print(f"Error deleting droplet: {error_text}", file=sys.stderr)
sys.exit(error_text)
print(f"Droplet '{config['name']}' deleted successfully.")
# Remove droplet ID from config
config.pop("droplet_id", None)
save_config(config)
print("Droplet ID removed from config.")

8
scripts/cicd-python/utils/repos_builder.py Normal file
View file

@ -0,0 +1,8 @@
import subprocess
def build_deb_repo():
print("Creating and signing .deb package repository.")
def build_rpm_repo():
print("Creating and signing .rpm package repository.")