Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update redeploy script to keep old bundles #2969

Merged
merged 1 commit into from
Jan 17, 2017
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 118 additions & 24 deletions scripts/redeploy.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,32 @@
#!/usr/bin/env python
#
# auto-deploy script for https://riot.im/develop
#
# Listens for HTTP hits. When it gets one, downloads the artifact from jenkins
# and deploys it as the new version.
#
# Requires the following python packages:
#
# - requests
# - flask
#
from __future__ import print_function
import json, requests, tarfile, argparse, os, errno
import time
from urlparse import urljoin
from flask import Flask, jsonify, request, abort

app = Flask(__name__)

arg_jenkins_url, arg_extract_path, arg_should_clean, arg_symlink, arg_config_location = (
None, None, None, None, None
)
arg_jenkins_url = None
arg_extract_path = None
arg_bundles_path = None
arg_should_clean = None
arg_symlink = None
arg_config_location = None

class DeployException(Exception):
pass

def download_file(url):
local_filename = url.split('/')[-1]
Expand Down Expand Up @@ -57,6 +76,9 @@ def on_receive_jenkins_poke():
abort(400, "Missing or bad build number")
return

return fetch_jenkins_build(job_name, build_num)

def fetch_jenkins_build(job_name, build_num):
artifact_url = urljoin(
arg_jenkins_url, "job/%s/%s/api/json" % (job_name, build_num)
)
Expand Down Expand Up @@ -106,27 +128,38 @@ def on_receive_jenkins_poke():
arg_jenkins_url, "job/%s/%s/artifact/%s" % (job_name, build_num, tar_gz_path)
)

print("Retrieving .tar.gz file: %s" % tar_gz_url)
# we extract into a directory based on the build number. This avoids the
# problem of multiple builds building the same git version and thus having
# the same tarball name. That would lead to two potential problems:
# (a) sometimes jenkins serves corrupted artifacts; we would replace
# a good deploy with a bad one
# (b) we'll be overwriting the live deployment, which means people might
# see half-written files.
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
try:
deploy_tarball(tar_gz_url, build_dir)
except DeployException as e:
abort(400, e.message)

return jsonify({})

def deploy_tarball(tar_gz_url, build_dir):
"""Download a tarball from jenkins and deploy it as the new version
"""
print("Deploying %s to %s" % (tar_gz_url, build_dir))

if os.path.exists(build_dir):
raise DeployException(
"Not deploying. We have previously deployed this build."
)
os.mkdir(build_dir)

# we rely on the fact that flask only serves one request at a time to
# ensure that we do not overwrite a tarball from a concurrent request.
filename = download_file(tar_gz_url)
print("Downloaded file: %s" % filename)

try:
# we extract into a directory based on the build number. This avoids the
# problem of multiple builds building the same git version and thus having
# the same tarball name. That would lead to two potential problems:
# (a) sometimes jenkins serves corrupted artifacts; we would replace
# a good deploy with a bad one
# (b) we'll be overwriting the live deployment, which means people might
# see half-written files.
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
if os.path.exists(build_dir):
abort(400, "Not deploying. We have previously deployed this build.")
return
os.mkdir(build_dir)

untar_to(filename, build_dir)
print("Extracted to: %s" % build_dir)
finally:
Expand All @@ -139,9 +172,47 @@ def on_receive_jenkins_poke():
if arg_config_location:
create_symlink(source=arg_config_location, linkname=os.path.join(extracted_dir, 'config.json'))

if arg_bundles_path:
extracted_bundles = os.path.join(extracted_dir, 'bundles')
move_bundles(source=extracted_bundles, dest=arg_bundles_path)

# replace the (hopefully now empty) extracted_bundles dir with a
# symlink to the common dir.
relpath = os.path.relpath(arg_bundles_path, extracted_dir)
os.rmdir(extracted_bundles)
print ("Symlink %s -> %s" % (extracted_bundles, relpath))
os.symlink(relpath, extracted_bundles)

create_symlink(source=extracted_dir, linkname=arg_symlink)

return jsonify({})
def move_bundles(source, dest):
"""Move the contents of the 'bundles' directory to a common dir

We check that we will not be overwriting anything before we proceed.

Args:
source (str): path to 'bundles' within the extracted tarball
dest (str): target common directory
"""

if not os.path.isdir(dest):
os.mkdir(dest)

# build a map from source to destination, checking for non-existence as we go.
renames = {}
for f in os.listdir(source):
dst = os.path.join(dest, f)
if os.path.exists(dst):
raise DeployException(
"Not deploying. The bundle includes '%s' which we have previously deployed."
% f
)
renames[os.path.join(source, f)] = dst

for (src, dst) in renames.iteritems():
print ("Move %s -> %s" % (src, dst))
os.rename(src, dst)


if __name__ == "__main__":
parser = argparse.ArgumentParser("Runs a Vector redeployment server.")
Expand All @@ -161,6 +232,13 @@ def on_receive_jenkins_poke():
"The location to extract .tar.gz files to."
)
)
parser.add_argument(
"-b", "--bundles-dir", dest="bundles_dir", help=(
"A directory to move the contents of the 'bundles' directory to. A \
symlink to the bundles directory will also be written inside the \
extracted tarball. Example: './bundles'."
)
)
parser.add_argument(
"-c", "--clean", dest="clean", action="store_true", default=False, help=(
"Remove .tar.gz files after they have been downloaded and extracted."
Expand All @@ -179,18 +257,34 @@ def on_receive_jenkins_poke():
To this location."
)
)
parser.add_argument(
"--test", dest="tarball_uri", help=(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this arg supposed to be called '--test'?

"Don't start an HTTP listener. Instead download a build from Jenkins \
immediately."
),
)

args = parser.parse_args()
if args.jenkins.endswith("/"): # important for urljoin
arg_jenkins_url = args.jenkins
else:
arg_jenkins_url = args.jenkins + "/"
arg_extract_path = args.extract
arg_bundles_path = args.bundles_dir
arg_should_clean = args.clean
arg_symlink = args.symlink
arg_config_location = args.config
print(
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config location: %s" %
(args.port, arg_extract_path,
" (clean after)" if arg_should_clean else "", arg_symlink, arg_jenkins_url, arg_config_location)
)
app.run(host="0.0.0.0", port=args.port, debug=True)

if not os.path.isdir(arg_extract_path):
os.mkdir(arg_extract_path)

if args.tarball_uri is not None:
build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
deploy_tarball(args.tarball_uri, build_dir)
else:
print(
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config location: %s" %
(args.port, arg_extract_path,
" (clean after)" if arg_should_clean else "", arg_symlink, arg_jenkins_url, arg_config_location)
)
app.run(host="0.0.0.0", port=args.port, debug=True)