diff --git a/android/cli/commands/_build.js b/android/cli/commands/_build.js index ecbd1cb3ed2..d9fab9a42dd 100644 --- a/android/cli/commands/_build.js +++ b/android/cli/commands/_build.js @@ -30,13 +30,11 @@ const ADB = require('node-titanium-sdk/lib/adb'), fields = require('fields'), fs = require('fs-extra'), i18n = require('node-titanium-sdk/lib/i18n'), - jsanalyze = require('node-titanium-sdk/lib/jsanalyze'), path = require('path'), temp = require('temp'), ti = require('node-titanium-sdk'), tiappxml = require('node-titanium-sdk/lib/tiappxml'), util = require('util'), - Cloak = require('ti.cloak').default, afs = appc.fs, i18nLib = appc.i18n(__dirname), @@ -45,6 +43,8 @@ const ADB = require('node-titanium-sdk/lib/adb'), version = appc.version, V8_STRING_VERSION_REGEXP = /(\d+)\.(\d+)\.\d+\.\d+/; +const platformsRegExp = new RegExp('^(' + ti.allPlatformNames.join('|') + ')$'); // eslint-disable-line security/detect-non-literal-regexp + function AndroidBuilder() { Builder.apply(this, arguments); @@ -2243,44 +2243,6 @@ AndroidBuilder.prototype.generateAppProject = async function generateAppProject( await fs.emptyDir(rootLibsDirPath); await fs.ensureDir(rootLibsDirPath); - // Copy "./platform/android" directory tree from all modules and main project to "app" project's "./src/main". - // Android build tools auto-grabs folders named "assets", "res", "aidl", etc. from this folder. - // Note 1: Our "build.gradle" is configured to look for JAR/AAR files here too. (Needed by hyperloop.) - // Note 2: Main Titanium project's folder must be copied last, allowing it to replace asset or res files. - const platformDirPaths = []; - for (const module of this.modules) { - if (!module.native) { - platformDirPaths.push(path.join(module.modulePath, 'platform', 'android')); - } - } - platformDirPaths.push(path.join(this.projectDir, 'platform', 'android')); - for (const nextPath of platformDirPaths) { - if (await fs.exists(nextPath)) { - afs.copyDirSyncRecursive(nextPath, this.buildAppMainDir, { - logger: this.logger.debug, - preserve: true - }); - } - } - - const googleServicesFile = path.join(this.projectDir, 'platform', 'android', 'google-services.json'); - if (await fs.exists(googleServicesFile)) { - afs.copyFileSync(googleServicesFile, path.join(this.buildAppDir, 'google-services.json'), { - logger: this.logger.debug - }); - } - - // Copy Titanium project's "./platform/android" directory tree to "app" project's "./src/main". - // Android build tools auto-grabs folders named "assets", "res", "aidl", etc. in this folder. - // Note: Our "build.gradle" is configured to look for JAR/AAR files here too. (Needed by hyperloop.) - const tiPlatformAndroidDirPath = path.join(this.projectDir, 'platform', 'android'); - if (await fs.exists(tiPlatformAndroidDirPath)) { - afs.copyDirSyncRecursive(tiPlatformAndroidDirPath, this.buildAppMainDir, { - logger: this.logger.debug, - preserve: false - }); - } - // Create a "deploy.json" file if debugging/profiling is enabled. const deployJsonFile = path.join(this.buildAppMainAssetsDir, 'deploy.json'); const deployData = { @@ -2298,9 +2260,7 @@ AndroidBuilder.prototype.generateAppProject = async function generateAppProject( // Copy files from Titanium project's "Resources" directory to the build directory. await fs.ensureDir(this.buildAppMainResDrawableDir); - await new Promise((resolve) => { - this.copyResources(resolve); - }); + await this.copyResources(); // We can do the following in parallel. await Promise.all([ @@ -2415,604 +2375,478 @@ AndroidBuilder.prototype.generateAppProject = async function generateAppProject( await fs.writeFile(path.join(this.buildAppDir, 'build.gradle'), buildGradleContent); }; -AndroidBuilder.prototype.copyResources = function copyResources(next) { - const ignoreDirs = this.ignoreDirs, - ignoreFiles = this.ignoreFiles, - extRegExp = /\.(\w+)$/, - drawableRegExp = /^images\/(high|medium|low|res-[^/]+)(\/(.*))/, - drawableDpiRegExp = /^(high|medium|low)$/, - drawableExtRegExp = /((\.9)?\.(png|jpg))$/, - splashScreenRegExp = /^default\.(9\.png|png|jpg)$/, - relSplashScreenRegExp = /^default\.(9\.png|png|jpg)$/, - drawableResources = {}, - jsFiles = {}, - jsFilesToEncrypt = this.jsFilesToEncrypt = [], - jsBootstrapFiles = [], - htmlJsFiles = this.htmlJsFiles = {}, - symlinkFiles = process.platform !== 'win32' && this.config.get('android.symlinkResources', true), - _t = this, - cloak = this.encryptJS ? new Cloak() : null; - - this.logger.info('Copying resource files'); - - function copyDir(opts, callback) { - if (opts && opts.src && fs.existsSync(opts.src) && opts.dest) { - opts.origSrc = opts.src; - opts.origDest = opts.dest; - recursivelyCopy.call(this, opts.src, opts.dest, opts.ignoreRootDirs, opts, callback); - } else { - callback(); - } - } - - function copyFile(from, to, next) { - var d = path.dirname(to); - fs.ensureDirSync(d); - - if (fs.existsSync(to)) { - _t.logger.warn(__('Overwriting file %s', to.cyan)); - } - - if (symlinkFiles) { - // Remove prior symlink. - fs.existsSync(to) && fs.unlinkSync(to); - // Remove prior file. (if previously did not symlink) - fs.existsSync(to) && fs.removeSync(to); - this.logger.debug(__('Symlinking %s => %s', from.cyan, to.cyan)); - if (next) { - fs.symlink(from, to, next); - } else { - fs.symlinkSync(from, to); - } - } else { - this.logger.debug(__('Copying %s => %s', from.cyan, to.cyan)); - if (next) { - fs.readFile(from, function (err, data) { - if (err) { - throw err; - } - fs.writeFile(to, data, next); - }); - } else { - fs.writeFileSync(to, fs.readFileSync(from)); - } - } - } - - function recursivelyCopy(src, dest, ignoreRootDirs, opts, done) { - var files; - if (fs.statSync(src).isDirectory()) { - files = fs.readdirSync(src); - } else { - // we have a file, so fake a directory listing - files = [ path.basename(src) ]; - src = path.dirname(src); - } - - async.whilst( - function (cb) { - process.nextTick(() => cb(null, files.length)); - }, - - function (next) { - const filename = files.shift(), - from = path.join(src, filename); - - let destDir = dest, - to = path.join(destDir, filename); - - // check that the file actually exists and isn't a broken symlink - if (!fs.existsSync(from)) { - return next(); - } - - const isDir = fs.statSync(from).isDirectory(); - - // check if we are ignoring this file - if ((isDir && ignoreRootDirs && ignoreRootDirs.indexOf(filename) !== -1) || (isDir ? ignoreDirs : ignoreFiles).test(filename)) { - _t.logger.debug(__('Ignoring %s', from.cyan)); - return next(); - } - - // if this is a directory, recurse - if (isDir) { - recursivelyCopy.call(_t, from, path.join(destDir, filename), null, opts, next); - return; - } - - // we have a file, now we need to see what sort of file - - // check if it's a drawable resource - const relPath = from.replace(opts.origSrc, '').replace(/\\/g, '/').replace(/^\//, ''); - let m = relPath.match(drawableRegExp); - let isDrawable = false; - - if (m && m.length >= 4 && m[3]) { - const destFilename = m[3]; - const destLowerCaseFilename = destFilename.toLowerCase(); - const extMatch = destLowerCaseFilename.match(drawableExtRegExp); - const origExt = extMatch && extMatch[1] || ''; - - destDir = path.join( - _t.buildAppMainResDir, - drawableDpiRegExp.test(m[1]) ? 'drawable-' + m[1][0] + 'dpi' : 'drawable-' + m[1].substring(4) - ); - - if (splashScreenRegExp.test(filename)) { - // we have a splash screen image - to = path.join(destDir, 'background' + origExt); - } else { - // We have a drawable image file. (Rename it if it contains invalid characters.) - let warningMessages = []; - if (destFilename.includes('/') || destFilename.includes('\\')) { - warningMessages.push(__('- Files cannot be put into subdirectories.')); - } - let destFilteredFilename = destLowerCaseFilename.replace(drawableExtRegExp, ''); - destFilteredFilename = destFilteredFilename.replace(/[^a-z0-9_]/g, '_') + origExt; - if (destFilteredFilename !== destFilename) { - warningMessages.push(__('- Names must contain only lowercase a-z, 0-9, or underscore.')); - } - if (/^\d/.test(destFilteredFilename)) { - warningMessages.push(__('- Names cannot start with a number.')); - destFilteredFilename = '_' + destFilteredFilename; - } - if (warningMessages.length > 0) { - _t.logger.warn(__(`Invalid "res" file: ${path.relative(_t.projectDir, from)}`)); - for (const nextMessage of warningMessages) { - _t.logger.warn(nextMessage); - } - _t.logger.warn(__(`- Titanium will rename to: ${destFilteredFilename}`)); - } - to = path.join(destDir, destFilteredFilename); - } - isDrawable = true; - } else if (m = relPath.match(relSplashScreenRegExp)) { - // we have a splash screen - // if it's a 9 patch, then the image goes in drawable-nodpi, not drawable - if (m[1] === '9.png') { - destDir = path.join(_t.buildAppMainResDir, 'drawable-nodpi'); - to = path.join(destDir, filename.replace('default.', 'background.')); - } else { - destDir = _t.buildAppMainResDrawableDir; - to = path.join(_t.buildAppMainResDrawableDir, filename.replace('default.', 'background.')); - } - isDrawable = true; - } - - if (isDrawable) { - const _from = from.replace(_t.projectDir, '').substring(1), - _to = to.replace(_t.buildAppMainResDir, '').replace(drawableExtRegExp, '').substring(1); - if (drawableResources[_to]) { - _t.logger.error(__('Found conflicting resources:')); - _t.logger.error(' ' + drawableResources[_to]); - _t.logger.error(' ' + from.replace(_t.projectDir, '').substring(1)); - _t.logger.error(__('You cannot have resources that resolve to the same resource entry name') + '\n'); - process.exit(1); - } - drawableResources[_to] = _from; - } - - // if the destination directory does not exists, create it - fs.ensureDirSync(destDir); - - const ext = filename.match(extRegExp); - - if (ext && ext[1] !== 'js') { - // we exclude js files because we'll check if they need to be removed after all files have been copied - _t.unmarkBuildDirFile(to); - } - - switch (ext && ext[1]) { - case 'css': - // if we encounter a css file, check if we should minify it - if (_t.minifyCSS) { - _t.logger.debug(__('Copying and minifying %s => %s', from.cyan, to.cyan)); - fs.readFile(from, function (err, data) { - if (err) { - throw err; - } - fs.writeFile(to, new CleanCSS({ processImport: false }).minify(data.toString()).styles, next); - }); - } else { - copyFile.call(_t, from, to, next); - } - break; - - case 'html': - // find all js files referenced in this html file - let htmlRelPath = from.replace(opts.origSrc, '').replace(/\\/g, '/').replace(/^\//, '').split('/'); - htmlRelPath.pop(); // remove the filename - htmlRelPath = htmlRelPath.join('/'); - jsanalyze.analyzeHtmlFile(from, htmlRelPath).forEach(function (file) { - htmlJsFiles[file] = 1; - }); - - _t.cli.createHook('build.android.copyResource', _t, function (from, to, cb) { - copyFile.call(_t, from, to, cb); - })(from, to, next); - break; - - case 'js': - // track each js file so we can copy/minify later - - // we use the destination file name minus the path to the assets dir as the id - // which will eliminate dupes - const id = to.replace(opts.origDest, opts.prefix ? opts.prefix : '').replace(/\\/g, '/').replace(/^\//, ''); - - if (!jsFiles[id] || !opts || !opts.onJsConflict || opts.onJsConflict(from, to, id)) { - jsFiles[id] = from; - } - - next(); - break; - - default: - // normal file, just copy it to the app project's "assets" directory - _t.cli.createHook('build.android.copyResource', _t, function (from, to, cb) { - copyFile.call(_t, from, to, cb); - })(from, to, next); - } - }, - - done - ); - } - - function warnDupeDrawableFolders(resourceDir) { - const dir = path.join(resourceDir, 'images'); - [ 'high', 'medium', 'low' ].forEach(function (dpi) { - let oldDir = path.join(dir, dpi), - newDir = path.join(dir, 'res-' + dpi[0] + 'dpi'); - if (fs.existsSync(oldDir) && fs.existsSync(newDir)) { - oldDir = oldDir.replace(this.projectDir, '').replace(/^\//, ''); - newDir = newDir.replace(this.projectDir, '').replace(/^\//, ''); - this.logger.warn(__('You have both an %s folder and an %s folder', oldDir.cyan, newDir.cyan)); - this.logger.warn(__('Files from both of these folders will end up in %s', ('res/drawable-' + dpi[0] + 'dpi').cyan)); - this.logger.warn(__('If two files are named the same, there is no guarantee which one will be copied last and therefore be the one the application uses')); - this.logger.warn(__('You should use just one of these folders to avoid conflicts')); - } - }, this); - } - - const tasks = [ - // First copy all of the Titanium SDK's core JS files shared by all platforms. - function (cb) { - const src = path.join(this.titaniumSdkPath, 'common', 'Resources', 'android'); - warnDupeDrawableFolders.call(this, src); - _t.logger.debug(__('Copying %s', src.cyan)); - copyDir.call(this, { - src: src, - dest: this.buildAppMainAssetsResourcesDir, - ignoreRootDirs: ti.allPlatformNames - }, cb); - }, - - // Next, copy all files in the project's Resources directory, - // but ignore any directory that is the name of a known platform. - function (cb) { - const src = path.join(this.projectDir, 'Resources'); - warnDupeDrawableFolders.call(this, src); - _t.logger.debug(__('Copying %s', src.cyan)); - copyDir.call(this, { - src: src, - dest: this.buildAppMainAssetsResourcesDir, - ignoreRootDirs: ti.allPlatformNames - }, cb); - }, - - // Last, copy all files from the Android specific Resources directory. - function (cb) { - const src = path.join(this.projectDir, 'Resources', 'android'); - warnDupeDrawableFolders.call(this, src); - _t.logger.debug(__('Copying %s', src.cyan)); - copyDir.call(this, { - src: src, - dest: this.buildAppMainAssetsResourcesDir - }, cb); - } - ]; +/** + * Walks the project resources/assets, module resources/assets and gathers up a categorized listing + * of files to process. (css, html, js, images, etc) + * @returns {Promise} + */ +AndroidBuilder.prototype.gatherResources = async function gatherResources() { + console.time('gathering resources'); + const gather = require('../../../cli/lib/gather'); + const walker = new gather.Walker({ + ignoreDirs: this.ignoreDirs, + ignoreFiles: this.ignoreFiles, + }); - // Fire an event requesting additional "Resources" paths from plugins. - tasks.push((done) => { - const hook = this.cli.createHook('build.android.requestResourcesDirPaths', this, (paths, done) => { + this.logger.info(__('Analyzing Resources directory')); + const firstWave = await Promise.all([ + walker.walk(path.join(this.titaniumSdkPath, 'common', 'Resources', 'android'), this.buildAppMainAssetsResourcesDir), + // NOTE: we copy over platform/android as-is without any transform/walk. Should iOS do the same? + // walker.walk(path.join(this.projectDir, 'platform', 'android'), this.buildAppMainDir), + walker.walk(path.join(this.projectDir, 'Resources'), this.buildAppMainAssetsResourcesDir, platformsRegExp), + walker.walk(path.join(this.projectDir, 'Resources', 'android'), this.buildAppMainAssetsResourcesDir), + ]); + let combined = gather.mergeMaps(firstWave); + + // node_modules + this.logger.info(__('Analyzing NPM package files')); + const moduleCopier = require('../../../cli/lib/module-copier'); + const dirSet = await moduleCopier.gather(this.projectDir); + const nodeModuleDirs = Array.from(dirSet); + const secondWave = await Promise.all(nodeModuleDirs.map(async dir => { + // here dir is the absolute path to the directory + // That means we need to construct the relative path to append to this.projectDir and this.xcodeAppDir + const relativePath = dir.substring(this.projectDir.length + 1); + return walker.walk(dir, path.join(this.buildAppMainAssetsResourcesDir, relativePath), null, null, relativePath); + })); + // merge the node_modules results on top of the project results... (shouldn't be any conflicts!) + secondWave.unshift(combined); + combined = gather.mergeMaps(secondWave); + + // Fire an event requesting additional "Resources" paths from plugins. (used by hyperloop) + this.logger.info(__('Analyzing plugin-contributed files')); + const hook = this.cli.createHook('build.android.requestResourcesDirPaths', this, async (paths, done) => { + try { const newTasks = []; if (Array.isArray(paths)) { for (const nextPath of paths) { if (typeof nextPath !== 'string') { continue; } - if (!fs.existsSync(nextPath) || !fs.statSync(nextPath).isDirectory()) { + if (!await fs.exists(nextPath) || !(await fs.stat(nextPath)).isDirectory()) { continue; } - newTasks.push((done) => { - _t.logger.debug(__('Copying %s', nextPath.cyan)); - copyDir.call(this, { - src: nextPath, - dest: this.buildAppMainAssetsResourcesDir - }, done); - }); + newTasks.push( + walker.walk(nextPath, this.buildAppMainAssetsResourcesDir) + ); } } - appc.async.series(this, newTasks, done); - }); - hook([], done); + const results = await Promise.all(newTasks); + done(null, results); + } catch (err) { + return done(err); + } }); - - // Copy resource files from all modules. - for (const module of this.modules) { - // Create a task which copies commonjs non-asset files. + const hookResults = await util.promisify(hook)([]); + // merge the hook results on top of the project/node_modules results... (shouldn't be any conflicts!) + hookResults.unshift(combined); + combined = gather.mergeMaps(hookResults); + + this.logger.info(__('Analyzing module files')); + // detect ambiguous modules + // this.modules.forEach(module => { + // const filename = `${module.id}.js`; + // if (combined.has(filename)) { + // this.logger.error(__('There is a project resource "%s" that conflicts with a native Android module', filename)); + // this.logger.error(__('Please rename the file, then rebuild') + '\n'); + // process.exit(1); + // } + // }); + // do modules in parallel - and for each we need to merge the results together! + const allModulesResults = await Promise.all(this.modules.map(async module => { + const tasks = []; + let assetDest = this.buildAppMainAssetsResourcesDir; if (!module.native) { - tasks.push(function (cb) { - _t.logger.debug(__('Copying %s', module.modulePath.cyan)); - copyDir.call(this, { - src: module.modulePath, - // Copy under subfolder named after module.id - dest: path.join(this.buildAppMainAssetsResourcesDir, path.basename(module.id)), - // Don't copy files under apidoc, docs, documentation, example or assets (assets is handled below) - ignoreRootDirs: [ 'apidoc', 'documentation', 'docs', 'example', 'assets' ], - // Make note that files are copied relative to the module.id folder at dest - // so that we don't see clashes between module1/index.js and module2/index.js - prefix: module.id, - onJsConflict: function (src, dest, id) { - this.logger.error(__('There is a project resource "%s" that conflicts with a CommonJS module', id)); - this.logger.error(__('Please rename the file, then rebuild') + '\n'); - process.exit(1); - }.bind(this) - }, cb); - }); + // Copy CommonJS non-asset files + const dest = path.join(this.buildAppMainAssetsResourcesDir, path.basename(module.id)); + // Pass in the relative path prefix we should give because we aren't copying direct to the root here. + // Otherwise index.js in one module "overwrites" index.js in another (because they're at same relative path inside module) + tasks.push(walker.walk(module.modulePath, dest, /^(apidoc|docs|documentation|example|assets)$/, null, module.id)); // TODO Consult some .moduleignore file in the module or something? .npmignore? + // CommonJS assets go to special location + assetDest = path.join(assetDest, 'modules', module.id.toLowerCase()); } - // Create a task which copies "assets" file tree from all modules. // Note: Android native module asset handling is inconsistent with commonjs modules and iOS native modules where // we're not copying assets to "modules/moduleId" directory. Continue doing this for backward compatibility. - const sourceAssetsDirPath = path.join(module.modulePath, 'assets'); - if (fs.existsSync(sourceAssetsDirPath) && fs.statSync(sourceAssetsDirPath).isDirectory()) { - let destinationDirPath = this.buildAppMainAssetsResourcesDir; - if (!module.native) { - destinationDirPath = path.join(destinationDirPath, 'modules', module.id.toLowerCase()); + tasks.push(walker.walk(path.join(module.modulePath, 'assets'), assetDest)); + + // NOTE: Android just copies without any special processing for platform/android. Should iOS? + // walker.walk(path.join(module.modulePath, 'platform', 'android'), this.buildAppMainDir), + + // Resources + tasks.push(walker.walk(path.join(module.modulePath, 'Resources'), this.buildAppMainAssetsResourcesDir, platformsRegExp)); + tasks.push(walker.walk(path.join(module.modulePath, 'Resources', 'android'), this.buildAppMainAssetsResourcesDir)); + const moduleResults = await Promise.all(tasks); + return gather.mergeMaps(moduleResults); + })); + // merge the allModulesResults over top our current combined! + allModulesResults.unshift(combined); + combined = gather.mergeMaps(allModulesResults); + + // Ok, so we have a Map for the full set of unique relative paths + // now categorize (i.e. lump into buckets of js/css/html/assets/generic resources) + const categorizer = new gather.Categorizer({ + tiappIcon: this.tiapp.icon, + }); + const categorized = await categorizer.run(combined); + + console.timeEnd('gathering resources'); + console.log(categorized); + return categorized; +}; + +/** + * Optionally mifies the input css files and copies them to the app + * @param {Map} cssFiles map from filename to file info + * @returns {Promise} + */ +AndroidBuilder.prototype.copyCSSFiles = async function copyCSSFiles(cssFiles) { + this.logger.debug(__('Copying CSS files')); + // iOS impl + // FIXME: do async in parallel? + cssFiles.forEach((info, file) => { + if (this.minifyCSS) { + this.logger.debug(__('Copying and minifying %s => %s', info.src.cyan, info.dest.cyan)); + const dir = path.dirname(info.dest); + fs.ensureDirSync(dir); + // TODO: use Promise API of clean-css... + fs.writeFileSync(info.dest, new CleanCSS({ processImport: false }).minify(fs.readFileSync(info.src).toString()).styles); + } else if (!this.copyFileSync(info.src, info.dest, { forceCopy: false })) { + this.logger.trace(__('No change, skipping %s', info.dest.cyan)); + } + this.unmarkBuildDirFile(info.dest); + }); +}; + +/** + * Used to de4termine the destination path for special assets (_app_props_.json, bootstrap.json) based on encyption or not. + * @returns {string} destination directory to place file + */ +AndroidBuilder.prototype.buildAssetsPath = function buildAssetsPath() { + return this.encryptJS ? this.buildAssetsDir : this.buildAppMainAssetsResourcesDir; +}; + +/** + * Write out file used by Ti.Properties to access properties at runtime + * This may modify this.jsFilesToEncrypt + * @returns {Promise} + */ +AndroidBuilder.prototype.writeAppProps = async function writeAppProps() { + const appPropsFile = path.join(this.buildAssetsPath(), '_app_props_.json'); + const props = {}; + Object.keys(this.tiapp.properties).forEach(prop => { + props[prop] = this.tiapp.properties[prop].value; + }); + await fs.writeFile(appPropsFile, JSON.stringify(props)); + this.encryptJS && this.jsFilesToEncrypt.push('_app_props_.json'); + this.unmarkBuildDirFile(appPropsFile); +}; + +/** + * Write the env variables file - used by node shim for process.env + * This may modify this.jsFilesToEncrypt + * @returns {Promise} + */ +AndroidBuilder.prototype.writeEnvironmentVariables = async function writeEnvironmentVariables() { + const envVarsFile = path.join(this.buildAssetsPath(), '_env_.json'); + await fs.writeFile( + envVarsFile, + // for non-development builds, DO NOT WRITE OUT ENV VARIABLES TO APP + this.writeEnvVars ? JSON.stringify(process.env) : {} + ); + this.encryptJS && this.jsFilesToEncrypt.push('_env_.json'); + this.unmarkBuildDirFile(envVarsFile); +}; + +/** + * This may modify this.jsFilesToEncrypt + * @param {Map} jsFilesMap map from filename to file info + * @returns {Promise} + */ +AndroidBuilder.prototype.processJSFiles = async function processJSFiles(jsFilesMap) { + // do the processing + this.logger.info(__('Processing JavaScript files')); + const sdkCommonFolder = path.join(this.titaniumSdkPath, 'common', 'Resources', 'android'); + // For now, need to adapt our Map results to String[] and Object for ProcessJsTask + const jsFiles = {}; + const inputFiles = []; + for (let [ key, value ] of jsFilesMap) { + jsFiles[key] = value; + inputFiles.push(value.src); + } + const jsBootstrapFiles = []; // modified by the task and then used after the fact to write our bootstrap.json file + const task = new ProcessJsTask({ + inputFiles, + incrementalDirectory: path.join(this.buildTiIncrementalDir, 'process-js'), + logger: this.logger, + builder: this, + jsFiles, + jsBootstrapFiles, + sdkCommonFolder, + defaultAnalyzeOptions: { + minify: this.minifyJS, + transpile: this.transpile, + sourceMap: this.sourceMaps, + resourcesDir: this.buildAssetsDir, + logger: this.logger, + targets: { + chrome: this.chromeVersion } - tasks.push(function (cb) { - _t.logger.debug(__('Copying %s', sourceAssetsDirPath.cyan)); - copyDir.call(this, { - src: sourceAssetsDirPath, - dest: destinationDirPath - }, cb); - }); } + }); + await task.run(); + if (this.useWebpack) { + // Merge Ti symbols from Webpack with the ones from legacy js processing + Object.keys(task.data.tiSymbols).forEach(file => { + const existingSymbols = this.tiSymbols[file] || []; + const additionalSymbols = task.data.tiSymbols[file]; + this.tiSymbols[file] = Array.from(new Set(existingSymbols.concat(additionalSymbols))); + }); + } else { + this.tiSymbols = task.data.tiSymbols; // record API usage for analytics + } - // Create a task which copies "Resources" file tree from all modules to APK "assets/Resources". - const sourceResourcesDirPath = path.join(module.modulePath, 'Resources'); - if (fs.existsSync(sourceResourcesDirPath) && fs.statSync(sourceResourcesDirPath).isDirectory()) { - tasks.push(function (cb) { - _t.logger.debug(__('Copying %s', sourceResourcesDirPath.cyan)); - copyDir.call(this, { - src: sourceResourcesDirPath, - dest: this.buildAppMainAssetsResourcesDir - }, cb); - }); - } + // then write the bootstrap json + return this.writeBootstrapJson(jsBootstrapFiles); +}; + +/** + * @param {string[]} jsBootstrapFiles list of bootstrap js files to add to listing we generate + * @returns {Promise} + */ +AndroidBuilder.prototype.writeBootstrapJson = async function writeBootstrapJson(jsBootstrapFiles) { + this.logger.info(__('Writing bootstrap json')); + // Write the "bootstrap.json" file, even if the bootstrap array is empty. + // Note: An empty array indicates the app has no bootstrap files. + const bootstrapJsonRelativePath = path.join('ti.internal', 'bootstrap.json'); + const bootstrapJsonAbsolutePath = path.join(this.buildAssetsPath(), bootstrapJsonRelativePath); + await fs.ensureDir(path.dirname(bootstrapJsonAbsolutePath)); + await fs.writeFile(bootstrapJsonAbsolutePath, JSON.stringify({ scripts: jsBootstrapFiles })); + this.encryptJS && this.jsFilesToEncrypt.push(bootstrapJsonRelativePath); + this.unmarkBuildDirFile(bootstrapJsonAbsolutePath); +}; - // Create a task which copies "Resources/android" file tree from all modules to APK "assets/Resources". - const sourceResourcesAndroidDirPath = path.join(module.modulePath, 'Resources', 'android'); - if (fs.existsSync(sourceResourcesAndroidDirPath) && fs.statSync(sourceResourcesAndroidDirPath).isDirectory()) { - tasks.push(function (cb) { - _t.logger.debug(__('Copying %s', sourceResourcesAndroidDirPath.cyan)); - copyDir.call(this, { - src: sourceResourcesAndroidDirPath, - dest: this.buildAppMainAssetsResourcesDir - }, cb); +/** + * Copy "./platform/android" directory tree from all modules and main project to "app" project's "./src/main". + * Android build tools auto-grabs folders named "assets", "res", "aidl", etc. from this folder. + * Note 1: Our "build.gradle" is configured to look for JAR/AAR files here too. (Needed by hyperloop.) + * Note 2: Main Titanium project's folder must be copied last, allowing it to replace asset or res files. + * @returns {Promise} + */ +AndroidBuilder.prototype.copyPlatformDirs = async function copyPlatformDirs() { + const platformDirPaths = []; + for (const module of this.modules) { + if (!module.native) { + platformDirPaths.push(path.join(module.modulePath, 'platform', 'android')); + } + } + const googleServicesFile = path.join(this.projectDir, 'platform', 'android', 'google-services.json'); + if (await fs.exists(googleServicesFile)) { + afs.copyFileSync(googleServicesFile, path.join(this.buildAppDir, 'google-services.json'), { + logger: this.logger.debug, + preserve: true + }); + } + platformDirPaths.push(path.join(this.projectDir, 'platform', 'android')); + for (const nextPath of platformDirPaths) { + if (await fs.exists(nextPath)) { + afs.copyDirSyncRecursive(nextPath, this.buildAppMainDir, { + logger: this.logger.debug, + preserve: true }); } } +}; - tasks.push(done => { - // copy js files into assets directory and minify if needed - this.logger.info(__('Processing JavaScript files')); +AndroidBuilder.prototype.copyResources = async function copyResources() { + // First walk all the input dirs and gather/categorize the files into buckets + const gatheredResults = await this.gatherResources(); + this.jsFilesToEncrypt = []; // set listing of files to encrypt to empty array (may be modified by tasks below) - const inputFiles = []; - const outputFileMap = {}; - const copyUnmodified = []; - Object.keys(jsFiles).forEach(relPath => { - const from = jsFiles[relPath]; - if (htmlJsFiles[relPath]) { - // this js file is referenced from an html file, so don't minify or encrypt - copyUnmodified.push(relPath); - } else { - inputFiles.push(from); - } - outputFileMap[relPath] = path.join(this.buildAssetsDir, relPath); - }); + // ok we now have them organized into broad categories + // we can schedule tasks to happen in parallel: + await Promise.all([ + this.copyCSSFiles(gatheredResults.cssFiles), + this.processJSFiles(gatheredResults.jsFiles), + this.writeAppProps(), // writes _app_props_.json for Ti.Properties + this.writeEnvironmentVariables(), // writes _env_.json for process.env + this.copyPlatformDirs(), // copies platform/android dirs from project/modules + this.copyUnmodifiedResources(gatheredResults.resourcesToCopy), // copies any other files that don't require special handling (like JS/CSS do) + ]); - const task = new ProcessJsTask({ - inputFiles, - incrementalDirectory: path.join(this.buildTiIncrementalDir, 'process-js'), - logger: this.logger, - builder: this, - jsFiles: Object.keys(jsFiles).reduce((jsFilesInfo, relPath) => { - jsFilesInfo[relPath] = { - src: jsFiles[relPath], - dest: outputFileMap[relPath] - }; - return jsFilesInfo; - }, {}), - jsBootstrapFiles, - sdkCommonFolder: path.join(this.titaniumSdkPath, 'common', 'Resources'), - defaultAnalyzeOptions: { - minify: this.minifyJS, - transpile: this.transpile, - sourceMap: this.sourceMaps, - resourcesDir: this.buildAssetsDir, - logger: this.logger, - targets: { - chrome: this.chromeVersion - } - } - }); - task.run() - .then(() => { - // Copy all unencrypted files processed by ProcessJsTask to "app" project's APK "assets" directory. - // Note: Encrypted files are handled by "titanium_prep" instead. - if (this.encryptJS) { - return null; - } - return new Promise((resolve) => { - appc.async.parallel(this, Object.keys(outputFileMap).map(relPath => { - return next => { - const from = outputFileMap[relPath]; - const to = path.join(this.buildAppMainAssetsResourcesDir, relPath); - this.unmarkBuildDirFile(to); - if (fs.existsSync(from)) { - copyFile.call(this, from, to, next); - } else { - next(); // eslint-disable-line promise/no-callback-in-promise - } - }; - }), resolve); - }); - }) - .then(() => { - if (this.useWebpack) { - // Merge Ti symbols from Webpack with the ones from legacy js processing - Object.keys(task.data.tiSymbols).forEach(file => { - const existingSymbols = this.tiSymbols[file] || []; - const additionalSymbols = task.data.tiSymbols[file]; - this.tiSymbols[file] = Array.from(new Set(existingSymbols.concat(additionalSymbols))); - }); - } else { - this.tiSymbols = task.data.tiSymbols; - } + // Then do the rest of the shit... + const templateDir = path.join(this.platformPath, 'templates', 'app', 'default', 'template', 'Resources', 'android'); + return Promise.all([ + this.encryptJSFiles(), + this.ensureAppIcon(templateDir), + this.ensureSplashScreen(templateDir), + ]); +}; - // Copy all unprocessed files to "app" project's APK "assets" directory. - appc.async.parallel(this, copyUnmodified.map(relPath => { - return next => { - const from = jsFiles[relPath]; - const to = path.join(this.buildAppMainAssetsResourcesDir, relPath); - copyFile.call(this, from, to, next); - this.unmarkBuildDirFile(to); - }; - }), done); - - return null; - }) - .catch(e => { - this.logger.error(e); - process.exit(1); - }); - }); +/** + * Copies all the rest of the files that need no extra processing. + * @param {Map} resourcesToCopy filepaths to file info + * @returns {Promise} + */ +AndroidBuilder.prototype.copyUnmodifiedResources = async function copyUnmodifiedResources(resourcesToCopy) { + this.logger.debug(__('Copying resources')); + const symlinkFiles = process.platform !== 'win32' && this.config.get('android.symlinkResources', true); + + // FIXME: use p-limit to limit number of files in process? + // TODO: Can we move all of this into a task like the process JS task? + // Then it can handle the incremental build stuff and we don't have to worry about it + const tasks = []; + resourcesToCopy.forEach((info, file) => tasks.push(this.smartCopy(info, file, symlinkFiles))); + return Promise.all(tasks); +}; - appc.async.series(this, tasks, async () => { - const templateDir = path.join(this.platformPath, 'templates', 'app', 'default', 'template', 'Resources', 'android'); - const srcIcon = path.join(templateDir, 'appicon.png'); - const destIcon = path.join(this.buildAppMainAssetsResourcesDir, this.tiapp.icon); +/** + * @param {object} info file info + * @param {string} file filepath (relative?) + * @param {boolean} symlinkFiles shoudl we attempt to symlink files isntead of copying? + * @returns {Promise} + */ +AndroidBuilder.prototype.smartCopy = async function smartCopy(info, file, symlinkFiles) { + // Dumb Android version + const to = info.dest; + const from = info.src; + this.unmarkBuildDirFile(to); + + const exists = await fs.exists(to); + if (exists) { + this.logger.warn(__('Overwriting file %s', to.cyan)); + // TODO: Avoid copying if file hasn't changed! + } else { + const d = path.dirname(to); + await fs.ensureDir(d); + } - // if an app icon hasn't been copied, copy the default one - if (!(await fs.exists(destIcon))) { - copyFile.call(this, srcIcon, destIcon); + if (symlinkFiles) { + // Remove prior symlink. + if (exists) { + await fs.unlink(to); + // Remove prior file. (if previously did not symlink) + // NOTE: Should we ever have to do this? + // fs.existsSync(to) && fs.removeSync(to); } - this.unmarkBuildDirFile(destIcon); + this.logger.debug(__('Symlinking %s => %s', from.cyan, to.cyan)); + return fs.symlink(from, to); + } - const destIcon2 = path.join(this.buildAppMainResDrawableDir, this.tiapp.icon); - if (!(await fs.exists(destIcon2))) { - // Note, we are explicitly copying destIcon here as we want to ensure that we're - // copying the user specified icon, srcIcon is the default Titanium icon - copyFile.call(this, destIcon, destIcon2); - } - this.unmarkBuildDirFile(destIcon2); + this.logger.debug(__('Copying %s => %s', from.cyan, to.cyan)); + return fs.copyFile(from, to); +}; - // make sure we have a splash screen - const backgroundRegExp = /^background(\.9)?\.(png|jpg)$/, - destBg = path.join(this.buildAppMainResDrawableDir, 'background.png'), - nodpiDir = path.join(this.buildAppMainResDir, 'drawable-nodpi'); - if (!(await fs.readdir(this.buildAppMainResDrawableDir)).some(name => { +/** + * Ensures the generated app has a splash screen image + * @param {string} templateDir the filepath to the Titanium SDK's app template for Android apps + */ +AndroidBuilder.prototype.ensureSplashScreen = async function ensureSplashScreen(templateDir) { + // make sure we have a splash screen + const backgroundRegExp = /^background(\.9)?\.(png|jpg)$/; + const destBg = path.join(this.buildAppMainResDrawableDir, 'background.png'); + const nodpiDir = path.join(this.buildAppMainResDir, 'drawable-nodpi'); + if (!(await fs.readdir(this.buildAppMainResDrawableDir)).some(name => { + if (backgroundRegExp.test(name)) { + this.unmarkBuildDirFile(path.join(this.buildAppMainResDrawableDir, name)); + return true; + } + return false; + }, this)) { + // no background image in drawable, but what about drawable-nodpi? + if (!(await fs.exists(nodpiDir)) || !(await fs.readdir(nodpiDir)).some(name => { if (backgroundRegExp.test(name)) { - this.unmarkBuildDirFile(path.join(this.buildAppMainResDrawableDir, name)); + this.unmarkBuildDirFile(path.join(nodpiDir, name)); return true; } return false; }, this)) { - // no background image in drawable, but what about drawable-nodpi? - if (!(await fs.exists(nodpiDir)) || !(await fs.readdir(nodpiDir)).some(name => { - if (backgroundRegExp.test(name)) { - this.unmarkBuildDirFile(path.join(nodpiDir, name)); - return true; - } - return false; - }, this)) { - this.unmarkBuildDirFile(destBg); - copyFile.call(this, path.join(templateDir, 'default.png'), destBg); - } + this.unmarkBuildDirFile(destBg); + this.copyFileSync(path.join(templateDir, 'default.png'), destBg); } + } +}; - // write the properties file - const buildAssetsPath = this.encryptJS ? this.buildAssetsDir : this.buildAppMainAssetsResourcesDir, - appPropsFile = path.join(buildAssetsPath, '_app_props_.json'), - props = {}; - Object.keys(this.tiapp.properties).forEach(function (prop) { - props[prop] = this.tiapp.properties[prop].value; - }, this); - await fs.writeFile(appPropsFile, JSON.stringify(props)); - this.encryptJS && jsFilesToEncrypt.push('_app_props_.json'); - this.unmarkBuildDirFile(appPropsFile); +/** + * Ensures the generated app has an app icon + * @param {string} templateDir the filepath to the Titanium SDK's app template for Android apps + */ +AndroidBuilder.prototype.ensureAppIcon = async function ensureAppIcon(templateDir) { + const srcIcon = path.join(templateDir, 'appicon.png'); + const destIcon = path.join(this.buildAppMainAssetsResourcesDir, this.tiapp.icon); - // Write the "bootstrap.json" file, even if the bootstrap array is empty. - // Note: An empty array indicates the app has no bootstrap files. - const bootstrapJsonRelativePath = path.join('ti.internal', 'bootstrap.json'), - bootstrapJsonAbsolutePath = path.join(buildAssetsPath, bootstrapJsonRelativePath); - await fs.ensureDir(path.dirname(bootstrapJsonAbsolutePath)); - await fs.writeFile(bootstrapJsonAbsolutePath, JSON.stringify({ scripts: jsBootstrapFiles })); - this.encryptJS && jsFilesToEncrypt.push(bootstrapJsonRelativePath); - this.unmarkBuildDirFile(bootstrapJsonAbsolutePath); + // if an app icon hasn't been copied, copy the default one from our app template + if (!(await fs.exists(destIcon))) { + this.copyFileSync(srcIcon, destIcon); // TODO: Use async call! + } + this.unmarkBuildDirFile(destIcon); - if (!jsFilesToEncrypt.length) { - // nothing to encrypt, continue - return next(); - } - if (!cloak) { - return next(new Error('Could not load encryption library!')); - } + const destIcon2 = path.join(this.buildAppMainResDrawableDir, this.tiapp.icon); + if (!(await fs.exists(destIcon2))) { + // Note, we are explicitly copying destIcon here as we want to ensure that we're + // copying the user specified icon, srcIcon is the default Titanium icon + this.copyFileSync(destIcon, destIcon2); // TODO: Use async call! + } + this.unmarkBuildDirFile(destIcon2); +}; + +/** + * @returns {Promise} + */ +AndroidBuilder.prototype.encryptJSFiles = async function encryptJSFiles() { + if (!this.jsFilesToEncrypt.length) { + // nothing to encrypt, continue + return; + } - this.logger.info('Encrypting javascript assets...'); + const Cloak = require('ti.cloak').default; + const cloak = this.encryptJS ? new Cloak() : null; + if (!cloak) { + throw new Error('Could not load encryption library!'); + } - // NOTE: maintain 'build.android.titaniumprep' hook for remote encryption policy. - this.cli.createHook('build.android.titaniumprep', this, async next => { - try { - await Promise.all( - jsFilesToEncrypt.map(async file => { - const from = path.join(this.buildAssetsDir, file); - const to = path.join(this.buildAppMainAssetsResourcesDir, file + '.bin'); - - this.logger.debug(__('Encrypting: %s', from.cyan)); - await fs.ensureDir(path.dirname(to)); - this.unmarkBuildDirFile(to); - return await cloak.encryptFile(from, to); - }) - ); - - this.logger.info('Writing encryption key...'); - await cloak.setKey('android', this.abis, path.join(this.buildAppMainDir, 'jniLibs')); - - // Generate 'AssetCryptImpl.java' from template. - const assetCryptDest = path.join(this.buildGenAppIdDir, 'AssetCryptImpl.java'); - this.unmarkBuildDirFile(assetCryptDest); - await fs.ensureDir(this.buildGenAppIdDir); - await fs.writeFile( - assetCryptDest, - ejs.render( - await fs.readFile(path.join(this.templatesDir, 'AssetCryptImpl.java'), 'utf8'), - { - appid: this.appid, - assets: jsFilesToEncrypt, - salt: cloak.salt - } - ) - ); + this.logger.info('Encrypting javascript assets...'); - next(); - } catch (e) { - next(new Error('Could not encrypt assets!\n' + e)); - } - })(next, [ this.tiapp.guid, '' ], {}, next); + // NOTE: maintain 'build.android.titaniumprep' hook for remote encryption policy. + const hook = this.cli.createHook('build.android.titaniumprep', this, async function (exe, args, opts, next) { + try { + await Promise.all( + this.jsFilesToEncrypt.map(async file => { + const from = path.join(this.buildAssetsDir, file); + const to = path.join(this.buildAppMainAssetsResourcesDir, file + '.bin'); + + this.logger.debug(__('Encrypting: %s', from.cyan)); + await fs.ensureDir(path.dirname(to)); + this.unmarkBuildDirFile(to); + return await cloak.encryptFile(from, to); + }) + ); + + this.logger.info('Writing encryption key...'); + await cloak.setKey('android', this.abis, path.join(this.buildAppMainDir, 'jniLibs')); + + // Generate 'AssetCryptImpl.java' from template. + const assetCryptDest = path.join(this.buildGenAppIdDir, 'AssetCryptImpl.java'); + this.unmarkBuildDirFile(assetCryptDest); + await fs.ensureDir(this.buildGenAppIdDir); + await fs.writeFile( + assetCryptDest, + ejs.render( + await fs.readFile(path.join(this.templatesDir, 'AssetCryptImpl.java'), 'utf8'), + { + appid: this.appid, + assets: this.jsFilesToEncrypt, + salt: cloak.salt + } + ) + ); + + next(); + } catch (e) { + next(new Error('Could not encrypt assets!\n' + e)); + } }); + return util.promisify(hook)(null, [ this.tiapp.guid, '' ], {}); }; AndroidBuilder.prototype.generateRequireIndex = async function generateRequireIndex() {