Generate yarn cache with THEIA_VERSION. Generate resolutions to use strict Theia dependencies

dockerfiles/theia/Dockerfile

@@ -49,23 +49,23 @@ RUN \
     rm -rf /tmp/* /var/cache/apk/* && \
     # Change version of Theia to specified in THEIA_VERSION
     ${HOME}/versions.sh && rm ${HOME}/versions.sh && \
-    # Add Theia extensions
-    #==TODO== replace with master branch after Theia release
-    git clone --depth=1 -b latest-deps https://github.com/eclipse/che-theia-hosted-plugin-manager-extension /tmp/hosted-plugin-extension && \
-    node ${HOME}/add-extensions.js \
-        @eclipse-che/che-theia-hosted-plugin-manager-extension:file:///tmp/hosted-plugin-extension \
-        #che-theia-ssh-extension:https://github.com/eclipse/che-theia-ssh-plugin.git \
-    && rm ${HOME}/add-extensions.js && \
-    # Build Theia with all the extensions
-    cd ${HOME} && \
-    cat package.json && \
+    # Apply resolution section to the Theia package.json to use strict versions for Theia dependencies
+    node ${HOME}/resolutions-provider.js ${HOME}/package.json && \
+    # Generate node_modules, should be reused with default extensions
+    yarn && \
+    # Add default Theia extensions
+    node ${HOME}/add-extensions.js && \
+    # Launch `yarn` again to apply default extensions to the Theia node_modules
     yarn && \
+    yarn theia clean && \
+    # Build Theia with all the extensions
     yarn theia build && \
     # Install Theia plugin generator
     npm install -g yo @theia/generator-plugin && \
     # Change permissions to allow editing of files for openshift user
     find ${HOME} -exec sh -c "chgrp 0 {}; chmod g+rwX {}" \; && \
     # Grant permissions for modifying supervisor log file
-    touch /var/log/supervisord.log && chmod g+rwX /var/log/supervisord.log && chgrp 0 /var/log/supervisord.log
+    touch /var/log/supervisord.log && chmod g+rwX /var/log/supervisord.log && chgrp 0 /var/log/supervisord.log && \
+    yarn cache clean
 
 ENTRYPOINT ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisord.conf"]

dockerfiles/theia/README.md

@@ -3,9 +3,16 @@
 ## Build image manually
 Example:
 using build script:
+
+```
 ./build.sh --build-args:GITHUB_TOKEN=$GITHUB_TOKEN,THEIA_VERSION=0.3.10 --tag:0.3.10-nightly
+```
+
 with native docker:
+
+```
 docker build -t eclipse/che-theia:0.3.10-nightly --build-arg GITHUB_TOKEN={your token} --build-arg THEIA_VERSION=0.3.10 .
+```
 
 ## Theia version
 

dockerfiles/theia/src/add-extensions.js

@@ -11,83 +11,102 @@
  */
 
 const fs = require("fs");
-const cp = require("child_process");
+const spawnSync = require("child_process").spawnSync;
 
-const DEFAULT_THEIA_ROOT = '/home/theia';
-const EXTENSIONS_DIR = '/home/theia/extensions';
+const DEFAULT_THEIA_ROOT = process.env.HOME;
+const EXTENSIONS_DIR = `${DEFAULT_THEIA_ROOT}/extensions`;
+const EXTENSION_FILE = 'extensions.json';
+const EXTENSIONS_FILE_PATH = `${DEFAULT_THEIA_ROOT}/${EXTENSION_FILE}`;
 
-const givenExtensions = process.argv;
-// remove nodejs binary path
-givenExtensions.shift();
-// remove script name
-givenExtensions.shift();
+const EXTENSION_TYPE_GIT = "git";
+const EXTENSION_TYPE_DIR = "dir";
 
-if (givenExtensions.length > 0) {
-    cp.execSync(`mkdir -p ${EXTENSIONS_DIR}`);
-    addExtensions(parseExtensions(givenExtensions));
+if (!fs.existsSync(EXTENSIONS_FILE_PATH)) {
+    console.error(`${EXTENSIONS_FILE_PATH} was not found.`);
+    process.exit(1);
 }
 
-/**
- * Converts given extensions args format to map:
- * name:git://github.com/user/extension.git to: name -> url
- * name:file:///path/to/extension to: name -> path
- */
-function parseExtensions(givenExtensions) {
-    const extensions = {};
-    for (let extension of givenExtensions) {
-        const colonPos = extension.indexOf(':');
-        if (colonPos === -1) {
-            console.error('Invalid extension format: ', extension);
-            process.exit(1);
-        }
-        const extensionName = extension.substring(0, colonPos).trim();
-        const extensionUri = extension.substring(colonPos + 1).trim();
-        if (extensionUri.indexOf('://') === -1) {
-            console.error('Invalid extension uri: ', extensionUri);
-            process.exit(2);
-        }
-        extensions[extensionName] = extensionUri;
-    }
-    return extensions;
+const givenExtensions = require(EXTENSIONS_FILE_PATH)["extensions"];
+if (givenExtensions.length > 0) {
+    spawnSync(`mkdir -p ${EXTENSIONS_DIR}`);
+    addExtensions(givenExtensions);
 }
 
 function addExtensions(extensions) {
-    for (let extensionName in extensions) {
+    const extensionsToAdd = {};
+    for (let extension of extensions) {
+        const extensionName = extension["name"];
         const extensionRootPath = EXTENSIONS_DIR + '/' + extensionName.replace(/\//g, '_');
-        const extensionUri = extensions[extensionName];
-        if (extensionUri.startsWith('file://')) {
-            const extensionPath = extensionUri.substring(7); // remove protocol
-            if (!fs.existsSync(extensionPath)) {
-                console.error('Invalid extension path: ', extensionPath);
-                process.exit(2);
-            }
-            cp.execSync(`mv ${extensionPath} ${extensionRootPath}`);
-        } else {
-            cloneRepository(extensionRootPath, extensionUri);
+
+        const extensionType = extension["type"];
+        switch(extensionType) {
+            case EXTENSION_TYPE_DIR:
+                addExtensionFromDir(extensionRootPath, extension);
+                break;
+            case EXTENSION_TYPE_GIT:
+                addExtensionFromGit(extensionRootPath, extension);
+                break;
+            default:
+                throw new Error(`Invalid extension type ${extensionType}.`);
         }
         buildExtension(extensionRootPath);
-        extensions[extensionName] = getBinaryPath(extensionRootPath, extensionName);
+        extensionsToAdd[extensionName] = getBinaryPath(extensionRootPath, extensionName);
+    }
+    console.log("Extension to add: ", extensionsToAdd);
+    addExtensionsIntoDefaultPackageJson(extensionsToAdd);
+}
+
+function addExtensionFromDir(extensionRootPath, extension) {
+    const extensionSource = extension["source"];
+    if (!fs.existsSync(extensionSource)) {
+        console.error('Invalid extension path: ', extensionSource);
+        process.exit(2);
     }
-    addExtensionsIntoDefaultPackageJson(extensions);
+    spawnSync(`mv ${extensionSource} ${extensionRootPath}`);
+}
+
+function addExtensionFromGit(extensionRootPath, extension) {
+    const checkoutTarget = extension["checkoutTo"];
+    const extensionSource = extension["source"];
+    cloneRepository(extensionRootPath, extensionSource);
+
+    checkoutRepo(extensionRootPath, checkoutTarget);
 }
 
 function cloneRepository(path, url) {
     try {
-        console.log('Cloning repository: ', url);
-        cp.execSync(`git clone --depth=1 --quiet ${url} ${path}`);
+        console.log(`>>> Cloning repository: ${url}`);
+        spawnSync(`git`, ['clone', `${url}`, `${path}`], {stdio:[0,1,2]});
     } catch (error) {
-        console.error('Failed to clone repository: ', url);
+        console.error(`Failed to clone repository: ${url}`, error);
         process.exit(3);
     }
 }
 
+function checkoutRepo(path, checkoutTarget) {
+    try {
+         if (!checkoutTarget) {
+            console.error(`Field 'checkoutTo' is required for git extensions. Path ${path}`);
+            process.exit(4);
+        }
+        console.log(`>>> Checkout repository to: ${checkoutTarget}`);
+
+        spawnSync('git', ['checkout', checkoutTarget], {cwd: `${path}`, stdio:[0,1,2]});
+    } catch (error) {
+        console.error(`Failed to checkout repository to branch: ${checkoutTarget}`, error);
+        process.exit(5);
+    }
+}
+
 function buildExtension(path) {
     try {
         console.log('Building extension: ', path);
-        cp.execSync(`cd ${path} && yarn`);
+        const nodeModulesPath = `${DEFAULT_THEIA_ROOT}/node_modules`;
+        // build extension, but use Theia node_modules to reuse dependencies and prevent growing docker image.
+        spawnSync(`yarn`, ['--modules-folder', nodeModulesPath, '--global-folder', nodeModulesPath, '--cache-folder', nodeModulesPath], {cwd: `${path}`, stdio:[0,1,2]});
     } catch (error) {
         console.error('Failed to build extension located in: ', path);
-        process.exit(4);
+        process.exit(6);
     }
 }
 
@@ -109,7 +128,7 @@ function getBinaryPath(extensionRoot, extensionName) {
         }
     }
     console.error('Failed to find folder with binaries for extension: ', extensionRoot);
-    process.exit(5);
+    process.exit(7);
 }
 
 function addExtensionsIntoDefaultPackageJson(extensions) {
@@ -119,5 +138,6 @@ function addExtensionsIntoDefaultPackageJson(extensions) {
         dependencies[extension] = extensions[extension];
     }
     theiaPackageJson['dependencies'] = dependencies;
-    fs.writeFileSync(`${DEFAULT_THEIA_ROOT}/package.json`, JSON.stringify(theiaPackageJson), 'utf8');
+    const json = JSON.stringify(theiaPackageJson, undefined, 4);
+    fs.writeFileSync(`${DEFAULT_THEIA_ROOT}/package.json`, json, 'utf8');
 }

dockerfiles/theia/src/extensions.json

@@ -0,0 +1,16 @@
+{
+    "extensions": [
+        {
+            "name": "theia-machines-extension",
+            "source": "https://github.com/eclipse/che-theia-machines-plugin.git",
+            "checkoutTo": "master",
+            "type": "git"
+        },
+        {
+            "name": "@eclipse-che/che-theia-hosted-plugin-manager-extension",
+            "source": "https://github.com/eclipse/che-theia-hosted-plugin-manager-extension.git",
+            "checkoutTo": "latest-deps",
+            "type": "git"
+        }
+    ]
+}

dockerfiles/theia/src/resolutions-provider.js

@@ -0,0 +1,92 @@
+"use strict";
+/*
+ * Copyright (c) 2018 Red Hat, Inc.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *   Red Hat, Inc. - initial API and implementation
+ */
+const fs = require("fs");
+const spawnSync = require("child_process").spawnSync;
+
+const PACKAGE_JSON_PATH = process.argv[2];
+console.log(`Generate resolutions for ${PACKAGE_JSON_PATH}`);
+
+const PACKAGE_JSON = require(PACKAGE_JSON_PATH);
+const THEIA_VERSION = process.env.THEIA_VERSION;
+const HOME = process.env.HOME;
+
+const NPM_API_URL = 'https://api.npms.io/v2';
+const keyWords = 'keywords:theia-extension';
+const resultSize = 200;
+
+const SEARCH_JSON_PATH = `${HOME}/search.json`;
+
+try {
+    spawnSync('curl',[`${NPM_API_URL}/search?q=${keyWords}&size=${resultSize}`, '-o', SEARCH_JSON_PATH]);
+} catch(error) {
+    console.error("Failed to get Theia depedencies. Cause: ", error);
+    process.exit(2);
+}
+
+const packageScopeRegexp = '^@theia/.*$';
+let theiaResolutionsList = [];
+try {
+    const filteredDepList = spawnSync('jq', ['-c', `.results | map(.package) | map(.name|select(test("${packageScopeRegexp}")))`, SEARCH_JSON_PATH]);
+    theiaResolutionsList = JSON.parse(filteredDepList.stdout);
+} catch(error) {
+    console.error("Failed to filter Theia resolutions. Cause: ", error);
+    process.exit(2);
+}
+
+const depResolutions = listToResolutions(theiaResolutionsList);
+console.log(depResolutions);
+
+PACKAGE_JSON["resolutions"] = depResolutions;
+console.log(`Write generated resolutions to the package.json ${PACKAGE_JSON_PATH}`);
+writeJsonToFile(PACKAGE_JSON_PATH, PACKAGE_JSON);
+
+function resolutionExist(resolutionName) {
+    try {
+        console.log(`Check depedency ${resolutionName}`);
+        const info = spawnSync('npm', ['view', `${resolutionName}@${THEIA_VERSION}`, 'version']);
+        if (info.stdout.toString()) {
+            return true;
+        }
+    } catch(error) {
+        console.log(`Unable to check resolution with name ${resolutionName}`);
+        process.exit(3);
+    }
+    return false;
+}
+
+/**
+ * Convert resolutions list to resolutions object with required THEIA_VERSION.
+ *
+ * @param resolutionsList - resolutions list.
+ */
+function listToResolutions(resolutionsList) {
+    const RESOLUTIONS = {};
+
+    for (const resolution of resolutionsList) {
+        if (resolutionExist(resolution)) {
+            RESOLUTIONS[resolution] = THEIA_VERSION;
+        }
+    }
+
+    return RESOLUTIONS;
+}
+
+/**
+ * Write json to the file by path.
+ *
+ * @param filePath - file system location of the file.
+ * @param json - json object to write.
+ */
+function writeJsonToFile(filePath, json) {
+    const content = JSON.stringify(json, undefined, 4);
+    fs.writeFileSync(filePath, content, 'utf8');
+}