build: upload to AZ as well as S3

package.json

@@ -4,6 +4,7 @@
   "repository": "https://github.com/electron/electron",
   "description": "Build cross platform desktop apps with JavaScript, HTML, and CSS",
   "devDependencies": {
+    "@azure/storage-blob": "^12.9.0",
     "@electron/docs-parser": "^0.12.4",
     "@electron/typescript-definitions": "^8.9.5",
     "@octokit/auth-app": "^2.10.0",
@@ -141,4 +142,4 @@
       "node script/gen-hunspell-filenames.js"
     ]
   }
-}
+}

script/lib/azput.js

@@ -0,0 +1,48 @@
+/* eslint-disable camelcase */
+const { BlobServiceClient } = require('@azure/storage-blob');
+const fs = require('fs');
+const path = require('path');
+
+const blobServiceClient = BlobServiceClient.fromConnectionString(process.env.ELECTRON_ARTIFACTS_BLOB_STORAGE);
+
+const args = require('minimist')(process.argv.slice(2));
+
+let { prefix = '/', key_prefix = '', _: files } = args;
+if (prefix && !prefix.endsWith(path.sep)) prefix = path.resolve(prefix) + path.sep;
+
+function filenameToKey (file) {
+  file = path.resolve(file);
+  if (file.startsWith(prefix)) file = file.substr(prefix.length - 1);
+  return key_prefix + (path.sep === '\\' ? file.replace(/\\/g, '/') : file);
+}
+
+let anErrorOccurred = false;
+function next (done) {
+  const file = files.shift();
+  if (!file) return done();
+  let key = filenameToKey(file);
+  // TODO: When we drop s3put, migrate the key to not include atom-shell in the callsites
+  key = key.replace('atom-shell/dist/', 'headers/dist/');
+  key = key.replace('atom-shell/symbols/', 'symbols/');
+  key = key.replace('atom-shell/tmp/', 'checksums-scratchpad/');
+  key = key.replace('electron-artifacts/', 'release-builds/');
+
+  const [containerName, ...keyPath] = key.split('/');
+  const blobKey = keyPath.join('/');
+  console.log(`Uploading '${file}' to container '${containerName}' with key '${blobKey}'...`);
+
+  const containerClient = blobServiceClient.getContainerClient(containerName);
+  const blockBlobClient = containerClient.getBlockBlobClient(blobKey);
+  blockBlobClient.uploadFile(file)
+    .then((uploadBlobResponse) => {
+      console.log(`Upload block blob ${blobKey} successfully: https://artifacts.electronjs.org/${key}`, uploadBlobResponse.requestId);
+    })
+    .catch((err) => {
+      console.error(err);
+      anErrorOccurred = true;
+    })
+    .then(() => next(done));
+}
+next(() => {
+  process.exit(anErrorOccurred ? 1 : 0);
+});

script/lib/util.py

@@ -15,7 +15,7 @@
   from urllib2 import urlopen
 import zipfile
 
-from lib.config import is_verbose_mode
+from lib.config import is_verbose_mode, s3_config
 
 ELECTRON_DIR = os.path.abspath(
   os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
@@ -155,7 +155,14 @@ def get_electron_version():
   with open(version_file) as f:
     return 'v' + f.read().strip()
 
-def s3put(bucket, access_key, secret_key, prefix, key_prefix, files):
+def store_artifact(prefix, key_prefix, files):
+  # Legacy S3 Bucket
+  s3put(prefix, key_prefix, files)
+  # New AZ Storage
+  azput(prefix, key_prefix, files)
+
+def s3put(prefix, key_prefix, files):
+  bucket, access_key, secret_key = s3_config()
   env = os.environ.copy()
   env['AWS_ACCESS_KEY_ID'] = access_key
   env['AWS_SECRET_ACCESS_KEY'] = secret_key
@@ -169,6 +176,16 @@ def s3put(bucket, access_key, secret_key, prefix, key_prefix, files):
   ] + files, env)
   print(output)
 
+def azput(prefix, key_prefix, files):
+  env = os.environ.copy()
+  output = execute([
+    'node',
+    os.path.join(os.path.dirname(__file__), 'azput.js'),
+    '--prefix', prefix,
+    '--key_prefix', key_prefix,
+  ] + files, env)
+  print(output)
+
 def get_out_dir():
   out_dir = 'Debug'
   override = os.environ.get('ELECTRON_OUT_DIR')

script/release/release.js

@@ -17,8 +17,8 @@ const pkgVersion = `v${pkg.version}`;
 const path = require('path');
 const temp = require('temp').track();
 const { URL } = require('url');
+const { BlobServiceClient } = require('@azure/storage-blob');
 const { Octokit } = require('@octokit/rest');
-const AWS = require('aws-sdk');
 
 require('colors');
 const pass = '✓'.green;
@@ -80,6 +80,8 @@ async function validateReleaseAssets (release, validatingRelease) {
     }
     const s3RemoteFiles = s3RemoteFilesForVersion(release.tag_name);
     await verifyShasumsForRemoteFiles(s3RemoteFiles, true);
+    const azRemoteFiles = azRemoteFilesForVersion(release.tag_name);
+    await verifyShasumsForRemoteFiles(s3RemoteFiles, true);
   }
 }
 
@@ -181,26 +183,36 @@ function assetsForVersion (version, validatingRelease) {
   return patterns;
 }
 
+const cloudStoreFilePaths = (version) => [
+  `iojs-${version}-headers.tar.gz`,
+  `iojs-${version}.tar.gz`,
+  `node-${version}.tar.gz`,
+  'node.lib',
+  'x64/node.lib',
+  'win-x64/iojs.lib',
+  'win-x86/iojs.lib',
+  'win-arm64/iojs.lib',
+  'win-x64/node.lib',
+  'win-x86/node.lib',
+  'win-arm64/node.lib',
+  'arm64/node.lib',
+  'SHASUMS.txt',
+  'SHASUMS256.txt'
+];
+
 function s3RemoteFilesForVersion (version) {
   const bucket = 'https://gh-contractor-zcbenz.s3.amazonaws.com/';
   const versionPrefix = `${bucket}atom-shell/dist/${version}/`;
-  const filePaths = [
-    `iojs-${version}-headers.tar.gz`,
-    `iojs-${version}.tar.gz`,
-    `node-${version}.tar.gz`,
-    'node.lib',
-    'x64/node.lib',
-    'win-x64/iojs.lib',
-    'win-x86/iojs.lib',
-    'win-arm64/iojs.lib',
-    'win-x64/node.lib',
-    'win-x86/node.lib',
-    'win-arm64/node.lib',
-    'arm64/node.lib',
-    'SHASUMS.txt',
-    'SHASUMS256.txt'
-  ];
-  return filePaths.map((filePath) => ({
+  return cloudStoreFilePaths(version).map((filePath) => ({
+    file: filePath,
+    url: `${versionPrefix}${filePath}`
+  }));
+}
+
+function azRemoteFilesForVersion (version) {
+  const azCDN = 'https://artifacts.electronjs.org/headers/';
+  const versionPrefix = `${azCDN}dist/${version}/`;
+  return cloudStoreFilePaths(version).map((filePath) => ({
     file: filePath,
     url: `${versionPrefix}${filePath}`
   }));
@@ -221,49 +233,39 @@ function runScript (scriptName, scriptArgs, cwd) {
 }
 
 function uploadNodeShasums () {
-  console.log('Uploading Node SHASUMS file to S3.');
+  console.log('Uploading Node SHASUMS file to artifacts.electronjs.org.');
   const scriptPath = path.join(ELECTRON_DIR, 'script', 'release', 'uploaders', 'upload-node-checksums.py');
   runScript(scriptPath, ['-v', pkgVersion]);
-  console.log(`${pass} Done uploading Node SHASUMS file to S3.`);
+  console.log(`${pass} Done uploading Node SHASUMS file to artifacts.electronjs.org.`);
 }
 
 function uploadIndexJson () {
-  console.log('Uploading index.json to S3.');
+  console.log('Uploading index.json to artifacts.electronjs.org.');
   const scriptPath = path.join(ELECTRON_DIR, 'script', 'release', 'uploaders', 'upload-index-json.py');
   runScript(scriptPath, [pkgVersion]);
-  console.log(`${pass} Done uploading index.json to S3.`);
+  console.log(`${pass} Done uploading index.json to artifacts.electronjs.org.`);
 }
 
 async function mergeShasums (pkgVersion) {
-  // Download individual checksum files for Electron zip files from S3,
+  // Download individual checksum files for Electron zip files from artifact storage,
   // concatenate them, and upload to GitHub.
 
-  const bucket = process.env.ELECTRON_S3_BUCKET;
-  const accessKeyId = process.env.ELECTRON_S3_ACCESS_KEY;
-  const secretAccessKey = process.env.ELECTRON_S3_SECRET_KEY;
-  if (!bucket || !accessKeyId || !secretAccessKey) {
-    throw new Error('Please set the $ELECTRON_S3_BUCKET, $ELECTRON_S3_ACCESS_KEY, and $ELECTRON_S3_SECRET_KEY environment variables');
+  const connectionString = process.env.ELECTRON_ARTIFACTS_BLOB_STORAGE;
+  if (!connectionString) {
+    throw new Error('Please set the $ELECTRON_ARTIFACTS_BLOB_STORAGE environment variable');
   }
 
-  const s3 = new AWS.S3({
-    apiVersion: '2006-03-01',
-    accessKeyId,
-    secretAccessKey,
-    region: 'us-west-2'
+  const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString);
+  const containerClient = blobServiceClient.getContainerClient('checksums-scratchpad');
+  const blobsIter = containerClient.listBlobsFlat({
+    prefix: `${pkgVersion}/`
   });
-  const objects = await s3.listObjectsV2({
-    Bucket: bucket,
-    Prefix: `atom-shell/tmp/${pkgVersion}/`,
-    Delimiter: '/'
-  }).promise();
   const shasums = [];
-  for (const obj of objects.Contents) {
-    if (obj.Key.endsWith('.sha256sum')) {
-      const data = await s3.getObject({
-        Bucket: bucket,
-        Key: obj.Key
-      }).promise();
-      shasums.push(data.Body.toString('ascii').trim());
+  for await (const blob of blobsIter) {
+    if (blob.name.endsWith('.sha256sum')) {
+      const blobClient = containerClient.getBlockBlobClient(blob.name);
+      const response = await blobClient.downloadToBuffer();
+      shasums.push(response.toString('ascii').trim());
     }
   }
   return shasums.join('\n');

script/release/uploaders/upload-index-json.py

@@ -9,8 +9,8 @@
 sys.path.append(
   os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../.."))
 
-from lib.config import s3_config
-from lib.util import s3put, scoped_cwd, safe_mkdir, get_out_dir, ELECTRON_DIR
+from lib.util import store_artifact, scoped_cwd, safe_mkdir, get_out_dir, \
+  ELECTRON_DIR
 
 OUT_DIR     = get_out_dir()
 
@@ -59,9 +59,7 @@ def main():
     with open(index_json, "w") as f:
       f.write(new_content)
 
-    bucket, access_key, secret_key = s3_config()
-    s3put(bucket, access_key, secret_key, OUT_DIR, 'atom-shell/dist',
-          [index_json])
+    store_artifact(OUT_DIR, 'atom-shell/dist', [index_json])
 
 
 if __name__ == '__main__':

script/release/uploaders/upload-node-checksums.py

@@ -10,8 +10,7 @@
 sys.path.append(
   os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../.."))
 
-from lib.config import s3_config
-from lib.util import download, rm_rf, s3put, safe_mkdir
+from lib.util import download, rm_rf, store_artifact, safe_mkdir
 
 DIST_URL = 'https://electronjs.org/headers/'
 
@@ -30,9 +29,8 @@ def main():
   ]
 
   if args.target_dir is None:
-    bucket, access_key, secret_key = s3_config()
-    s3put(bucket, access_key, secret_key, directory,
-          'atom-shell/dist/{0}'.format(args.version), checksums)
+    store_artifact(directory, 'atom-shell/dist/{0}'.format(args.version),
+                   checksums)
   else:
     copy_files(checksums, args.target_dir)
 

script/release/uploaders/upload-node-headers.py

@@ -9,8 +9,9 @@
 sys.path.append(
   os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../.."))
 
-from lib.config import PLATFORM, get_target_arch, s3_config
-from lib.util import safe_mkdir, scoped_cwd, s3put, get_out_dir, get_dist_dir
+from lib.config import PLATFORM, get_target_arch
+from lib.util import safe_mkdir, scoped_cwd, store_artifact, get_out_dir, \
+  get_dist_dir
 
 DIST_DIR    = get_dist_dir()
 OUT_DIR     = get_out_dir()
@@ -26,9 +27,8 @@
 def main():
   args = parse_args()
 
-  # Upload node's headers to S3.
-  bucket, access_key, secret_key = s3_config()
-  upload_node(bucket, access_key, secret_key, args.version)
+  # Upload node's headers to artifact storage.
+  upload_node(args.version)
 
 
 def parse_args():
@@ -38,17 +38,17 @@ def parse_args():
   return parser.parse_args()
 
 
-def upload_node(bucket, access_key, secret_key, version):
+def upload_node(version):
   with scoped_cwd(GEN_DIR):
     generated_tar = os.path.join(GEN_DIR, 'node_headers.tar.gz')
     for header_tar in HEADER_TAR_NAMES:
       versioned_header_tar = header_tar.format(version)
       shutil.copy2(generated_tar, os.path.join(GEN_DIR, versioned_header_tar))
 
-    s3put(bucket, access_key, secret_key, GEN_DIR,
-          'atom-shell/dist/{0}'.format(version), glob.glob('node-*.tar.gz'))
-    s3put(bucket, access_key, secret_key, GEN_DIR,
-          'atom-shell/dist/{0}'.format(version), glob.glob('iojs-*.tar.gz'))
+    store_artifact(GEN_DIR, 'atom-shell/dist/{0}'.format(version),
+                   glob.glob('node-*.tar.gz'))
+    store_artifact(GEN_DIR, 'atom-shell/dist/{0}'.format(version),
+                   glob.glob('iojs-*.tar.gz'))
 
   if PLATFORM == 'win32':
     if get_target_arch() == 'ia32':
@@ -73,16 +73,14 @@ def upload_node(bucket, access_key, secret_key, version):
     shutil.copy2(electron_lib, v4_node_lib)
 
     # Upload the node.lib.
-    s3put(bucket, access_key, secret_key, DIST_DIR,
-          'atom-shell/dist/{0}'.format(version), [node_lib])
+    store_artifact(DIST_DIR, 'atom-shell/dist/{0}'.format(version), [node_lib])
 
     # Upload the iojs.lib.
-    s3put(bucket, access_key, secret_key, DIST_DIR,
-          'atom-shell/dist/{0}'.format(version), [iojs_lib])
+    store_artifact(DIST_DIR, 'atom-shell/dist/{0}'.format(version), [iojs_lib])
 
     # Upload the v4 node.lib.
-    s3put(bucket, access_key, secret_key, DIST_DIR,
-          'atom-shell/dist/{0}'.format(version), [v4_node_lib])
+    store_artifact(DIST_DIR, 'atom-shell/dist/{0}'.format(version),
+                   [v4_node_lib])
 
 
 if __name__ == '__main__':

script/release/uploaders/upload-symbols.py

@@ -14,8 +14,8 @@ def is_fs_case_sensitive():
 sys.path.append(
   os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../.."))
 
-from lib.config import PLATFORM, s3_config
-from lib.util import get_electron_branding, execute, s3put, \
+from lib.config import PLATFORM
+from lib.util import get_electron_branding, execute, store_artifact, \
                      get_out_dir, ELECTRON_DIR
 
 RELEASE_DIR = get_out_dir()
@@ -76,16 +76,15 @@ def main():
   for f in files:
     assert os.path.exists(f)
 
-  bucket, access_key, secret_key = s3_config()
-  upload_symbols(bucket, access_key, secret_key, files)
+  upload_symbols(files)
 
 
 def run_symstore(pdb, dest, product):
   execute(['symstore', 'add', '/r', '/f', pdb, '/s', dest, '/t', product])
 
 
-def upload_symbols(bucket, access_key, secret_key, files):
-  s3put(bucket, access_key, secret_key, SYMBOLS_DIR, 'atom-shell/symbols',
+def upload_symbols(files):
+  store_artifact(SYMBOLS_DIR, 'atom-shell/symbols',
         files)