Fix 'npm run gulp updatePackageDependencies'

.vscode/launch.json

@@ -1,6 +1,6 @@
 {
     "version": "0.2.0",
-    "configurations": [
+    "configurations": [    
         {
             "name": "Launch Extension",
             "type": "extensionHost",
@@ -100,6 +100,21 @@
             "outFiles": [
                 "${workspaceRoot}/out/test/**/*.js"
             ]
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Launch gulp task",
+            "preLaunchTask": "build",
+            "program": "${workspaceFolder}/node_modules/gulp/bin/gulp.js",
+            "args": [
+                "updatePackageDependencies"
+            ],
+            "env": {
+                "NEW_DEPS_URLS": "https://download.visualstudio.microsoft.com/download/pr/9c36608c-cb04-451f-8df7-547eeb9b5d55/8626f25cb9fb40cd6fa2508009fccd1d/coreclr-debug-linux-x64.zip,https://download.visualstudio.microsoft.com/download/pr/9c36608c-cb04-451f-8df7-547eeb9b5d55/b5b307980f5da0bc161018acfad043c9/coreclr-debug-osx-x64.zip,https://download.visualstudio.microsoft.com/download/pr/9c36608c-cb04-451f-8df7-547eeb9b5d55/4b9c72a90d4558d8e72e653c1ea79936/coreclr-debug-win7-x64.zip",
+                "NEW_DEPS_VERSION": "1.18.0"
+            },
+            "cwd": "${workspaceFolder}"
         }
     ]
 }

gulpfile.ts

@@ -16,12 +16,14 @@ require('./tasks/offlinePackagingTasks');
 require('./tasks/backcompatTasks');
 require('./tasks/coverageTasks');
 
-gulp.task('generateOptionsSchema', () => {
+gulp.task('generateOptionsSchema', () : void => {
     optionsSchemaGenerator.GenerateOptionsSchema();
 });
 
-gulp.task('updatePackageDependencies', () => {
-    packageDependencyUpdater.updatePackageDependencies();
+// Disable warning about wanting an async function
+// tslint:disable-next-line
+gulp.task('updatePackageDependencies', () : Promise<void> => {
+    return packageDependencyUpdater.updatePackageDependencies();
 });
 
 gulp.task('tslint', () => {

src/packageManager/isValidDownload.ts

@@ -12,12 +12,10 @@ export interface DownloadValidator {
 }
 
 export function isValidDownload(buffer: Buffer, integrity: string, eventStream: EventStream): boolean {
-    let hash = crypto.createHash('sha256');
     if (integrity && integrity.length > 0) {
         eventStream.post(new DownloadValidation());
-        hash.update(buffer);
-        let value = hash.digest('hex');       
-        if (value.toUpperCase() == integrity.toUpperCase()) {         
+        let value = getBufferIntegrityHash(buffer);       
+        if (value == integrity.toUpperCase()) {         
             eventStream.post(new IntegrityCheckSuccess());
             return true;
         }
@@ -28,4 +26,11 @@ export function isValidDownload(buffer: Buffer, integrity: string, eventStream:
 
     // no integrity has been specified
     return true;
+}
+
+export function getBufferIntegrityHash(buffer: Buffer) : string {
+    let hash = crypto.createHash('sha256');
+    hash.update(buffer);
+    let value = hash.digest('hex').toUpperCase();
+    return value;
 }

src/tools/UpdatePackageDependencies.ts

@@ -6,33 +6,47 @@
 import * as fs from 'fs';
 import * as os from 'os';
 import { Package } from '../packageManager/Package';
-
+import { DownloadFile } from '../packageManager/FileDownloader';
+import { EventStream } from '../EventStream';
+import * as Event from "../omnisharp/loggingEvents";
+import NetworkSettings, { NetworkSettingsProvider } from '../NetworkSettings';
+import { getBufferIntegrityHash } from '../packageManager/isValidDownload';
 
 interface PackageJSONFile
 {
     runtimeDependencies : Package[];
 }
 
-export function updatePackageDependencies() {
+export async function updatePackageDependencies() : Promise<void> {
 
-    const urlsIndex = process.argv.indexOf("--urls");
-    const newPrimaryUrls = urlsIndex >= 0 ? process.argv[urlsIndex+1] : undefined;
-
-    const fallbackUrlsIndex = process.argv.indexOf("--fallbackUrls");
-    const newFallbackUrls = fallbackUrlsIndex >= 0 ? process.argv[fallbackUrlsIndex+1] : undefined;
+    const newPrimaryUrls = process.env["NEW_DEPS_URLS"];
+    const newVersion = process.env["NEW_DEPS_VERSION"];
 
-    if (newPrimaryUrls === undefined || newPrimaryUrls === "-?" || newPrimaryUrls === "-h") {
-        console.log("This command will update the URLs for package dependencies in package.json");
+    if (!newPrimaryUrls || !newVersion) {
+        console.log();
+        console.log("'npm run gulp updatePackageDependencies' will update package.json with new URLs of dependencies.");
         console.log();
-        console.log("Syntax: updatePackageDependencies --urls \"<url1>,<url2>,...\" [--fallbackUrls \"<fallback-url-1>,<fallback-url-2>...\"]");
+        console.log("To use:");
+        const setEnvVarPrefix = os.platform() === 'win32' ? "set " : "export ";
+        const setEnvVarQuote = os.platform() === 'win32' ? "" : "\'";
+        console.log(`  ${setEnvVarPrefix}NEW_DEPS_URLS=${setEnvVarQuote}https://example1/foo-osx.zip,https://example1/foo-win.zip,https://example1/foo-linux.zip${setEnvVarQuote}`);
+        console.log(`  ${setEnvVarPrefix}NEW_DEPS_VERSION=${setEnvVarQuote}1.2.3${setEnvVarQuote}`);
+        console.log("  npm run gulp updatePackageDependencies");
         console.log();
         return;
     }
 
-    if (newPrimaryUrls.length === 0) {
-        throw new Error("Invalid first argument to updatePackageDependencies. URL string argument expected.");
+    const newPrimaryUrlArray = newPrimaryUrls.split(',');
+    for (let urlToUpdate of newPrimaryUrlArray) {
+        if (!urlToUpdate.startsWith("https://")) {
+            throw new Error("Unexpected 'NEW_DEPS_URLS' value. All URLs should start with 'https://'.");
+        }
     }
 
+    if (! /^[0-9]+\.[0-9]+\.[0-9]+$/.test(newVersion)) {
+        throw new Error("Unexpected 'NEW_DEPS_VERSION' value. Expected format similar to: 1.2.3.");
+    }
+
     let packageJSON: PackageJSONFile = JSON.parse(fs.readFileSync('package.json').toString());
 
     // map from lowercase filename to Package
@@ -54,23 +68,79 @@ export function updatePackageDependencies() {
         if (dependency === undefined) {
             throw new Error(`Unable to update item for url '${url}'. No 'runtimeDependency' found with filename '${fileName}'.`);
         }
-
-        console.log(`Updating ${url}`);
         return dependency;
     };
 
-    newPrimaryUrls.split(',').forEach(urlToUpdate =>{
-        console.log(`Trying to update ${urlToUpdate}`);
-        let dependency = findDependencyToUpdate(urlToUpdate);
-        dependency.url = urlToUpdate;
+    const dottedVersionRegExp = /[0-9]+\.[0-9]+\.[0-9]+/g;
+    const dashedVersionRegExp = /[0-9]+-[0-9]+-[0-9]+/g;
+
+    const getMatchCount = (regexp: RegExp, searchString: string) : number => {
+        regexp.lastIndex = 0;
+        let retVal = 0;
+        while (regexp.test(searchString)) {
+            retVal++;
+        }
+        regexp.lastIndex = 0;
+        return retVal;
+    };
+
+    // First quickly make sure we could match up the URL to an existing item.
+    for (let urlToUpdate of newPrimaryUrlArray) {
+        const dependency = findDependencyToUpdate(urlToUpdate);
+        if (dependency.fallbackUrl) {
+            const dottedMatches : number = getMatchCount(dottedVersionRegExp, dependency.fallbackUrl);
+            const dashedMatches : number = getMatchCount(dashedVersionRegExp, dependency.fallbackUrl);
+            const matchCount : number = dottedMatches + dashedMatches;
+
+            if (matchCount == 0) {
+                throw new Error(`Version number not found in fallback URL '${dependency.fallbackUrl}'.`);
+            }
+            if (matchCount > 1) {
+                throw new Error(`Ambiguous version pattern found in fallback URL '${dependency.fallbackUrl}'. Multiple version strings found.`);
+            }
+        }
+    }
+
+    // Next take another pass to try and update to the URL  
+    const eventStream = new EventStream();
+    eventStream.subscribe((event: Event.BaseEvent) => {
+        switch (event.constructor.name) {
+            case Event.DownloadFailure.name:
+                console.log("Failed to download: " + (<Event.DownloadFailure>event).message);
+                break;
+        }
     });
+    const networkSettingsProvider : NetworkSettingsProvider = () => new NetworkSettings(/*proxy:*/ null, /*stringSSL:*/ true);
 
-    if (newFallbackUrls !== undefined) {
-        newFallbackUrls.split(',').forEach(urlToUpdate =>{
-            console.log(`Trying to update ${urlToUpdate}`);
-            let dependency = findDependencyToUpdate(urlToUpdate);
-            dependency.fallbackUrl = urlToUpdate;
-        }); 
+    const downloadAndGetHash = async (url:string) : Promise<string> => {
+        console.log(`Downlodaing from '${url}'`);
+        const buffer : Buffer = await DownloadFile(url, eventStream, networkSettingsProvider, url, null);
+        return getBufferIntegrityHash(buffer);
+    };
+
+    for (let urlToUpdate of newPrimaryUrlArray) {
+        let dependency = findDependencyToUpdate(urlToUpdate);
+        dependency.url = urlToUpdate;
+        dependency.integrity = await downloadAndGetHash(dependency.url);
+
+        if (dependency.fallbackUrl) {
+
+            // NOTE: We already verified in the first loop that one of these patterns will work, grab the one that does
+            let regex: RegExp = dottedVersionRegExp;
+            let newValue: string = newVersion;
+            if (!dottedVersionRegExp.test(dependency.fallbackUrl)) {
+                regex = dashedVersionRegExp;
+                newValue = newVersion.replace(/\./g, "-");
+            }
+            dottedVersionRegExp.lastIndex = 0;
+
+            dependency.fallbackUrl = dependency.fallbackUrl.replace(regex, newValue);
+            const fallbackUrlIntegrity = await downloadAndGetHash(dependency.fallbackUrl);
+
+            if (dependency.integrity !== fallbackUrlIntegrity) {
+                throw new Error(`File downloaded from primary URL '${dependency.url}' doesn't match '${dependency.fallbackUrl}'.`);
+            }
+        }
     }
 
     let content = JSON.stringify(packageJSON, null, 2);