Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deploy script #2974

Merged
merged 2 commits into from Jan 19, 2017
Merged
Changes from all commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -0,0 +1,183 @@
#!/usr/bin/env python
#
# download and unpack a riot-web tarball.
#
# Allows `bundles` to be extracted to a common directory, and a link to
# config.json to be added.

from __future__ import print_function

import argparse
import os
import os.path
import subprocess
import sys
import tarfile

try:
# python3
from urllib.request import urlretrieve
except ImportError:
# python2
from urllib import urlretrieve

class DeployException(Exception):
pass

def create_relative_symlink(linkname, target):
relpath = os.path.relpath(target, os.path.dirname(linkname))
print ("Symlink %s -> %s" % (linkname, relpath))
os.symlink(relpath, linkname)


def move_bundles(source, dest):
"""Move the contents of the 'bundles' directory to a common dir
We check that we will not be overwriting anything before we proceed.
Args:
source (str): path to 'bundles' within the extracted tarball
dest (str): target common directory
"""

if not os.path.isdir(dest):
os.mkdir(dest)

# build a map from source to destination, checking for non-existence as we go.
renames = {}
for f in os.listdir(source):
dst = os.path.join(dest, f)
if os.path.exists(dst):
raise DeployException(
"Not deploying. The bundle includes '%s' which we have previously deployed."
% f
)
renames[os.path.join(source, f)] = dst

for (src, dst) in renames.iteritems():
print ("Move %s -> %s" % (src, dst))
os.rename(src, dst)

class Deployer:
def __init__(self):
self.packages_path = "."
self.bundles_path = None
self.should_clean = False
self.config_location = None
self.verify_signature = True

def deploy(self, tarball, extract_path):
"""Download a tarball if necessary, and unpack it
Returns:
(str) the path to the unpacked deployment
"""
print("Deploying %s to %s" % (tarball, extract_path))

name_str = os.path.basename(tarball).replace(".tar.gz", "")
extracted_dir = os.path.join(extract_path, name_str)
if os.path.exists(extracted_dir):
raise DeployException('Cannot unpack %s: %s already exists' % (
tarball, extracted_dir))

downloaded = False
if tarball.startswith("http://") or tarball.startswith("https://"):
tarball = self.download_and_verify(tarball)
print("Downloaded file: %s" % tarball)
downloaded = True

try:
with tarfile.open(tarball) as tar:
tar.extractall(extract_path)
finally:
if self.should_clean and downloaded:
os.remove(tarball)

print ("Extracted into: %s" % extracted_dir)

if self.config_location:
create_relative_symlink(
target=self.config_location,
linkname=os.path.join(extracted_dir, 'config.json')
)

if self.bundles_path:
extracted_bundles = os.path.join(extracted_dir, 'bundles')
move_bundles(source=extracted_bundles, dest=self.bundles_path)

# replace the (hopefully now empty) extracted_bundles dir with a
# symlink to the common dir.
os.rmdir(extracted_bundles)
create_relative_symlink(
target=self.bundles_path,
linkname=extracted_bundles,
)
return extracted_dir

def download_and_verify(self, url):
tarball = self.download_file(url)

if self.verify_signature:
sigfile = self.download_file(url + ".asc")
subprocess.check_call(["gpg", "--verify", sigfile, tarball])

return tarball

def download_file(self, url):
if not os.path.isdir(self.packages_path):
os.mkdir(self.packages_path)
local_filename = os.path.join(self.packages_path,
url.split('/')[-1])
sys.stdout.write("Downloading %s -> %s..." % (url, local_filename))
sys.stdout.flush()
urlretrieve(url, local_filename)
print ("Done")
return local_filename

if __name__ == "__main__":
parser = argparse.ArgumentParser("Deploy a Riot build on a web server.")
parser.add_argument(
"-p", "--packages-dir", default="./packages", help=(
"The directory to download the tarball into. (Default: '%(default)s')"
)
)
parser.add_argument(
"-e", "--extract-path", default="./deploys", help=(
"The location to extract .tar.gz files to. (Default: '%(default)s')"
)
)
parser.add_argument(
"-b", "--bundles-dir", nargs='?', default="./bundles", help=(
"A directory to move the contents of the 'bundles' directory to. A \
symlink to the bundles directory will also be written inside the \
extracted tarball. Example: './bundles'. \
(Default: '%(default)s')"
)
)
parser.add_argument(
"-c", "--clean", action="store_true", default=False, help=(
"Remove .tar.gz files after they have been downloaded and extracted. \
(Default: %(default)s)"
)
)
parser.add_argument(
"--config", nargs='?', default='./config.json', help=(
"Write a symlink at config.json in the extracted tarball to this \
location. (Default: '%(default)s')"
)
)
parser.add_argument(
"tarball", help=(
"filename of tarball, or URL to download."
),
)

args = parser.parse_args()

deployer = Deployer()
deployer.packages_path = args.packages_dir
deployer.bundles_path = args.bundles_dir
deployer.should_clean = args.clean
deployer.config_location = args.config

deployer.deploy(args.tarball, args.extract_path)
@@ -14,32 +14,17 @@
import json, requests, tarfile, argparse, os, errno
import time
from urlparse import urljoin

from flask import Flask, jsonify, request, abort

from deploy import Deployer, DeployException

app = Flask(__name__)

arg_jenkins_url = None
deployer = None
arg_extract_path = None
arg_bundles_path = None
arg_should_clean = None
arg_symlink = None
arg_config_location = None

class DeployException(Exception):
pass

def download_file(url):
local_filename = url.split('/')[-1]
r = requests.get(url, stream=True)
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
return local_filename

def untar_to(tarball, dest):
with tarfile.open(tarball) as tar:
tar.extractall(dest)

def create_symlink(source, linkname):
try:
@@ -137,17 +122,20 @@ def fetch_jenkins_build(job_name, build_num):
# see half-written files.
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
try:
deploy_tarball(tar_gz_url, build_dir)
extracted_dir = deploy_tarball(tar_gz_url, build_dir)
except DeployException as e:
abort(400, e.message)

create_symlink(source=extracted_dir, linkname=arg_symlink)

return jsonify({})

def deploy_tarball(tar_gz_url, build_dir):
"""Download a tarball from jenkins and deploy it as the new version
"""
print("Deploying %s to %s" % (tar_gz_url, build_dir))
"""Download a tarball from jenkins and unpack it
Returns:
(str) the path to the unpacked deployment
"""
if os.path.exists(build_dir):
raise DeployException(
"Not deploying. We have previously deployed this build."
@@ -156,62 +144,8 @@ def deploy_tarball(tar_gz_url, build_dir):

# we rely on the fact that flask only serves one request at a time to
# ensure that we do not overwrite a tarball from a concurrent request.
filename = download_file(tar_gz_url)
print("Downloaded file: %s" % filename)

try:
untar_to(filename, build_dir)
print("Extracted to: %s" % build_dir)
finally:
if arg_should_clean:
os.remove(filename)

name_str = filename.replace(".tar.gz", "")
extracted_dir = os.path.join(build_dir, name_str)

if arg_config_location:
create_symlink(source=arg_config_location, linkname=os.path.join(extracted_dir, 'config.json'))

if arg_bundles_path:
extracted_bundles = os.path.join(extracted_dir, 'bundles')
move_bundles(source=extracted_bundles, dest=arg_bundles_path)

# replace the (hopefully now empty) extracted_bundles dir with a
# symlink to the common dir.
relpath = os.path.relpath(arg_bundles_path, extracted_dir)
os.rmdir(extracted_bundles)
print ("Symlink %s -> %s" % (extracted_bundles, relpath))
os.symlink(relpath, extracted_bundles)

create_symlink(source=extracted_dir, linkname=arg_symlink)

def move_bundles(source, dest):
"""Move the contents of the 'bundles' directory to a common dir
We check that we will not be overwriting anything before we proceed.
Args:
source (str): path to 'bundles' within the extracted tarball
dest (str): target common directory
"""

if not os.path.isdir(dest):
os.mkdir(dest)

# build a map from source to destination, checking for non-existence as we go.
renames = {}
for f in os.listdir(source):
dst = os.path.join(dest, f)
if os.path.exists(dst):
raise DeployException(
"Not deploying. The bundle includes '%s' which we have previously deployed."
% f
)
renames[os.path.join(source, f)] = dst

for (src, dst) in renames.iteritems():
print ("Move %s -> %s" % (src, dst))
os.rename(src, dst)
return deployer.deploy(tar_gz_url, build_dir)


if __name__ == "__main__":
@@ -270,21 +204,34 @@ def move_bundles(source, dest):
else:
arg_jenkins_url = args.jenkins + "/"
arg_extract_path = args.extract
arg_bundles_path = args.bundles_dir
arg_should_clean = args.clean
arg_symlink = args.symlink
arg_config_location = args.config

if not os.path.isdir(arg_extract_path):
os.mkdir(arg_extract_path)

deployer = Deployer()
deployer.bundles_path = args.bundles_dir
deployer.should_clean = args.clean
deployer.config_location = args.config

# we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to
# the jenkins server (and the jenkins server not being compromised and/or
# github not serving it compromised source). If that's not good enough for
# you, don't use riot.im/develop.
deployer.verify_signature = False

if args.tarball_uri is not None:
build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
deploy_tarball(args.tarball_uri, build_dir)
else:
print(
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config location: %s" %
(args.port, arg_extract_path,
" (clean after)" if arg_should_clean else "", arg_symlink, arg_jenkins_url, arg_config_location)
(args.port,
arg_extract_path,
" (clean after)" if deployer.should_clean else "",
arg_symlink,
arg_jenkins_url,
deployer.config_location,
)
)
app.run(host="0.0.0.0", port=args.port, debug=True)
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.