diff --git a/script/lib/util.py b/script/lib/util.py index 4db4e5fa6748a..933144d95c8ea 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -3,6 +3,7 @@ import atexit import contextlib import errno +import hashlib import platform import re import shutil @@ -15,9 +16,12 @@ import os import zipfile -from config import is_verbose_mode +from config import is_verbose_mode, s3_config from env_util import get_vs_env +BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', + 'boto')) + def get_host_arch(): """Returns the host architecture with a predictable string.""" @@ -129,6 +133,22 @@ def make_zip(zip_file_path, files, dirs): for f in filenames: zip_file.write(os.path.join(root, f)) zip_file.close() + upload_zip_sha256_checksum(zip_file_path) + + +def upload_zip_sha256_checksum(zip_file_path): + bucket, access_key, secret_key = s3_config() + checksum_path = '{}.sha256sum'.format(zip_file_path) + safe_unlink(checksum_path) + sha256 = hashlib.sha256() + with open(zip_file_path, 'rb') as f: + sha256.update(f.read()) + + zip_basename = os.path.basename(zip_file_path) + with open(checksum_path, 'w') as checksum: + checksum.write('{} *{}'.format(sha256.hexdigest(), zip_basename)) + s3put(bucket, access_key, secret_key, os.path.dirname(checksum_path), + 'atom-shell/tmp', [checksum_path]) def rm_rf(path): @@ -202,28 +222,39 @@ def parse_version(version): return vs + ['0'] * (4 - len(vs)) -def s3put(bucket, access_key, secret_key, prefix, key_prefix, files): - env = os.environ.copy() - BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', - 'boto')) - env['PYTHONPATH'] = os.path.pathsep.join([ - env.get('PYTHONPATH', ''), +def boto_path_dirs(): + return [ os.path.join(BOTO_DIR, 'build', 'lib'), - os.path.join(BOTO_DIR, 'build', 'lib.linux-x86_64-2.7')]) + os.path.join(BOTO_DIR, 'build', 'lib.linux-x86_64-2.7') + ] + + +def run_boto_script(access_key, secret_key, script_name, *args): + env = os.environ.copy() + env['AWS_ACCESS_KEY_ID'] = access_key + env['AWS_SECRET_ACCESS_KEY'] = secret_key + env['PYTHONPATH'] = os.path.pathsep.join( + [env.get('PYTHONPATH', '')] + boto_path_dirs()) + + boto = os.path.join(BOTO_DIR, 'bin', script_name) - boto = os.path.join(BOTO_DIR, 'bin', 's3put') args = [ sys.executable, - boto, + boto + ] + args + + execute(args, env) + + +def s3put(bucket, access_key, secret_key, prefix, key_prefix, files): + args = [ '--bucket', bucket, - '--access_key', access_key, - '--secret_key', secret_key, '--prefix', prefix, '--key_prefix', key_prefix, '--grant', 'public-read' ] + files - execute(args, env) + run_boto_script(access_key, secret_key, 's3put', *args) def import_vs_env(target_arch): diff --git a/script/merge-electron-checksums.py b/script/merge-electron-checksums.py new file mode 100644 index 0000000000000..7a5be5fe14af0 --- /dev/null +++ b/script/merge-electron-checksums.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Download individual checksum files for Electron zip files from S3, +# concatenate them, and upload to GitHub. + +from __future__ import print_function + +import argparse +import sys + +from lib.config import s3_config +from lib.util import boto_path_dirs + +sys.path.extend(boto_path_dirs()) + +from boto.s3.connection import S3Connection + +def main(): + args = parse_args() + bucket_name, access_key, secret_key = s3_config() + s3 = S3Connection(access_key, secret_key) + bucket = s3.get_bucket(bucket_name) + if bucket is None: + print('S3 bucket "{}" does not exist!'.format(bucket_name), file=sys.stderr) + return 1 + shasums = [s3_object.get_contents_as_string().strip() + for s3_object in bucket.list('atom-shell/tmp/', delimiter='/') + if s3_object.key.endswith('.sha256sum') and + args.version in s3_object.key] + print('\n'.join(shasums)) + return 0 + + +def parse_args(): + parser = argparse.ArgumentParser(description='Upload SHASUMS files to GitHub') + parser.add_argument('-v', '--version', help='Specify the version', + required=True) + return parser.parse_args() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/script/upload-checksums.py b/script/upload-node-checksums.py similarity index 100% rename from script/upload-checksums.py rename to script/upload-node-checksums.py diff --git a/script/upload.py b/script/upload.py index 1abd67aaf0d2e..e41946dffe4ce 100755 --- a/script/upload.py +++ b/script/upload.py @@ -2,6 +2,7 @@ import argparse import errno +from io import StringIO import os import subprocess import sys @@ -46,8 +47,7 @@ def main(): if not args.publish_release: if not dist_newer_than_head(): - create_dist = os.path.join(SOURCE_ROOT, 'script', 'create-dist.py') - execute([sys.executable, create_dist]) + run_python_script('create-dist.py') build_version = get_electron_build_version() if not ELECTRON_VERSION.startswith(build_version): @@ -69,14 +69,14 @@ def main(): tag_exists) if args.publish_release: - # Upload the SHASUMS.txt. - execute([sys.executable, - os.path.join(SOURCE_ROOT, 'script', 'upload-checksums.py'), - '-v', ELECTRON_VERSION]) + # Upload the Node SHASUMS*.txt. + run_python_script('upload-node-checksums.py', '-v', ELECTRON_VERSION) # Upload the index.json. - execute([sys.executable, - os.path.join(SOURCE_ROOT, 'script', 'upload-index-json.py')]) + run_python_script('upload-index-json.py') + + # Create and upload the Electron SHASUMS*.txt + release_electron_checksums(github, release) # Press the publish button. publish_release(github, release['id']) @@ -108,13 +108,10 @@ def main(): if PLATFORM == 'win32' and not tag_exists: # Upload PDBs to Windows symbol server. - execute([sys.executable, - os.path.join(SOURCE_ROOT, 'script', 'upload-windows-pdb.py')]) + run_python_script('upload-windows-pdb.py') # Upload node headers. - execute([sys.executable, - os.path.join(SOURCE_ROOT, 'script', 'upload-node-headers.py'), - '-v', args.version]) + run_python_script('upload-node-headers.py', '-v', args.version) def parse_args(): @@ -127,6 +124,11 @@ def parse_args(): return parser.parse_args() +def run_python_script(script, *args): + script_path = os.path.join(SOURCE_ROOT, 'script', script), + return execute([sys.executable, script_path] + args) + + def get_electron_build_version(): if get_target_arch() == 'arm' or os.environ.has_key('CI'): # In CI we just build as told. @@ -202,23 +204,35 @@ def create_release_draft(github, tag): return r +def release_electron_checksums(github, release): + checksums = run_python_script( + 'merge-electron-checksums.py', '-v', ELECTRON_VERSION) + upload_io_to_github(github, release, + 'SHASUMS256.txt', StringIO(checksums), 'text/plain') + + def upload_electron(github, release, file_path): # Delete the original file before uploading in CI. + filename = os.path.basename(file_path) if os.environ.has_key('CI'): try: for asset in release['assets']: - if asset['name'] == os.path.basename(file_path): + if asset['name'] == filename: github.repos(ELECTRON_REPO).releases.assets(asset['id']).delete() - break except Exception: pass # Upload the file. - params = {'name': os.path.basename(file_path)} - headers = {'Content-Type': 'application/zip'} + name = os.path.dirname(file_path) with open(file_path, 'rb') as f: - github.repos(ELECTRON_REPO).releases(release['id']).assets.post( - params=params, headers=headers, data=f, verify=False) + upload_io_to_github(github, release, name, f, 'application/zip') + + +def upload_io_to_github(github, release, name, io, content_type): + params = {'name': name} + headers = {'Content-Type': content_type} + github.repos(ELECTRON_REPO).releases(release['id']).assets.post( + params=params, headers=headers, data=io, verify=False) def publish_release(github, release_id):