schildi-web/scripts/redeploy.py
2017-09-21 10:26:07 +01:00

247 lines
8.1 KiB
Python
Executable File

#!/usr/bin/env python
#
# auto-deploy script for https://riot.im/develop
#
# Listens for HTTP hits. When it gets one, downloads the artifact from jenkins
# and deploys it as the new version.
#
# Requires the following python packages:
#
# - requests
# - flask
#
from __future__ import print_function
import json, requests, tarfile, argparse, os, errno
import time
import traceback
from urlparse import urljoin
import glob
from flask import Flask, jsonify, request, abort
from deploy import Deployer, DeployException
app = Flask(__name__)
arg_jenkins_url = None
deployer = None
arg_extract_path = None
arg_symlink = None
def create_symlink(source, linkname):
try:
os.symlink(source, linkname)
except OSError, e:
if e.errno == errno.EEXIST:
# atomic modification
os.symlink(source, linkname + ".tmp")
os.rename(linkname + ".tmp", linkname)
else:
raise e
@app.route("/", methods=["POST"])
def on_receive_jenkins_poke():
# {
# "name": "VectorWebDevelop",
# "build": {
# "number": 8
# }
# }
incoming_json = request.get_json()
if not incoming_json:
abort(400, "No JSON provided!")
return
print("Incoming JSON: %s" % (incoming_json,))
job_name = incoming_json.get("name")
if not isinstance(job_name, basestring):
abort(400, "Bad job name: %s" % (job_name,))
return
build_num = incoming_json.get("build", {}).get("number", 0)
if not build_num or build_num <= 0 or not isinstance(build_num, int):
abort(400, "Missing or bad build number")
return
return fetch_jenkins_build(job_name, build_num)
def fetch_jenkins_build(job_name, build_num):
artifact_url = urljoin(
arg_jenkins_url, "job/%s/%s/api/json" % (job_name, build_num)
)
artifact_response = requests.get(artifact_url).json()
# {
# "actions": [],
# "artifacts": [
# {
# "displayPath": "vector-043f6991a4ed-react-20f77d1224ef-js-0a7efe3e8bd5.tar.gz",
# "fileName": "vector-043f6991a4ed-react-20f77d1224ef-js-0a7efe3e8bd5.tar.gz",
# "relativePath": "vector-043f6991a4ed-react-20f77d1224ef-js-0a7efe3e8bd5.tar.gz"
# }
# ],
# "building": false,
# "description": null,
# "displayName": "#11",
# "duration": 137976,
# "estimatedDuration": 132008,
# "executor": null,
# "fullDisplayName": "VectorWebDevelop #11",
# "id": "11",
# "keepLog": false,
# "number": 11,
# "queueId": 12254,
# "result": "SUCCESS",
# "timestamp": 1454432640079,
# "url": "http://matrix.org/jenkins/job/VectorWebDevelop/11/",
# "builtOn": "",
# "changeSet": {},
# "culprits": []
# }
if artifact_response.get("result") != "SUCCESS":
abort(404, "Not deploying. Build was not marked as SUCCESS.")
return
if len(artifact_response.get("artifacts", [])) != 1:
abort(404, "Not deploying. Build has an unexpected number of artifacts.")
return
tar_gz_path = artifact_response["artifacts"][0]["relativePath"]
if not tar_gz_path.endswith(".tar.gz"):
abort(404, "Not deploying. Artifact is not a .tar.gz file")
return
tar_gz_url = urljoin(
arg_jenkins_url, "job/%s/%s/artifact/%s" % (job_name, build_num, tar_gz_path)
)
# we extract into a directory based on the build number. This avoids the
# problem of multiple builds building the same git version and thus having
# the same tarball name. That would lead to two potential problems:
# (a) sometimes jenkins serves corrupted artifacts; we would replace
# a good deploy with a bad one
# (b) we'll be overwriting the live deployment, which means people might
# see half-written files.
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
try:
extracted_dir = deploy_tarball(tar_gz_url, build_dir)
except DeployException as e:
traceback.print_exc()
abort(400, e.message)
create_symlink(source=extracted_dir, linkname=arg_symlink)
return jsonify({})
def deploy_tarball(tar_gz_url, build_dir):
"""Download a tarball from jenkins and unpack it
Returns:
(str) the path to the unpacked deployment
"""
if os.path.exists(build_dir):
raise DeployException(
"Not deploying. We have previously deployed this build."
)
os.mkdir(build_dir)
# we rely on the fact that flask only serves one request at a time to
# ensure that we do not overwrite a tarball from a concurrent request.
return deployer.deploy(tar_gz_url, build_dir)
if __name__ == "__main__":
parser = argparse.ArgumentParser("Runs a Vector redeployment server.")
parser.add_argument(
"-j", "--jenkins", dest="jenkins", default="https://matrix.org/jenkins/", help=(
"The base URL of the Jenkins web server. This will be hit to get the\
built artifacts (the .gz file) for redeploying."
)
)
parser.add_argument(
"-p", "--port", dest="port", default=4000, type=int, help=(
"The port to listen on for requests from Jenkins."
)
)
parser.add_argument(
"-e", "--extract", dest="extract", default="./extracted", help=(
"The location to extract .tar.gz files to."
)
)
parser.add_argument(
"-b", "--bundles-dir", dest="bundles_dir", help=(
"A directory to move the contents of the 'bundles' directory to. A \
symlink to the bundles directory will also be written inside the \
extracted tarball. Example: './bundles'."
)
)
parser.add_argument(
"-c", "--clean", dest="clean", action="store_true", default=False, help=(
"Remove .tar.gz files after they have been downloaded and extracted."
)
)
parser.add_argument(
"-s", "--symlink", dest="symlink", default="./latest", help=(
"Write a symlink to this location pointing to the extracted tarball. \
New builds will keep overwriting this symlink. The symlink will point \
to the /vector directory INSIDE the tarball."
)
)
# --include ../../config.json ./localhost.json homepages/*
parser.add_argument(
"--include", nargs='*', default='./config*.json', help=(
"Symlink these files into the root of the deployed tarball. \
Useful for config files and home pages. Supports glob syntax. \
(Default: '%(default)s')"
)
)
parser.add_argument(
"--test", dest="tarball_uri", help=(
"Don't start an HTTP listener. Instead download a build from Jenkins \
immediately."
),
)
args = parser.parse_args()
if args.jenkins.endswith("/"): # important for urljoin
arg_jenkins_url = args.jenkins
else:
arg_jenkins_url = args.jenkins + "/"
arg_extract_path = args.extract
arg_symlink = args.symlink
if not os.path.isdir(arg_extract_path):
os.mkdir(arg_extract_path)
deployer = Deployer()
deployer.bundles_path = args.bundles_dir
deployer.should_clean = args.clean
for include in args.include:
deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })
# we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to
# the jenkins server (and the jenkins server not being compromised and/or
# github not serving it compromised source). If that's not good enough for
# you, don't use riot.im/develop.
deployer.verify_signature = False
if args.tarball_uri is not None:
build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
deploy_tarball(args.tarball_uri, build_dir)
else:
print(
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Include files: %s" %
(args.port,
arg_extract_path,
" (clean after)" if deployer.should_clean else "",
arg_symlink,
arg_jenkins_url,
deployer.symlink_paths,
)
)
app.run(host="0.0.0.0", port=args.port, debug=True)