Permalink
Browse files

Add --maxWorkers flag and allow transformers to run in-band.

Summary:
This diff cleans up some cruft and adds some features:

* It removes the usage of an env variable to control workers.
* It removes the lazy and handwavy calculation on how many workers to use for jest-haste-map. Jest itself uses the maximum amount of workers available and it has never been reported as an issue – especially since it is a one-time startup cost of about 3 seconds on a cold cache only.
* It adds a `--max-workers` flag to replace the env variable. This one is able to control both the number of workers for `jest-haste-map` as well as the transformers.
* It makes the transformers run in the parent process if 1 or fewer workers are are specified. This should help with debugging.

Once you approve this diff, I will publish a new version of metro to npm and update the version used in RN and remove the use of the env variable altogether: https://our.intern.facebook.com/intern/biggrep/?corpus=xplat&filename=&case=false&view=default&extre=&s=REACT_NATIVE_MAX_WORKERS&engine=apr_strmatch&context=false&filter[uninteresting]=false&filter[intern]=false&filter[test]=false&grep_regex=

Note: the process of adding a CLI option is really broken. Commander also has a weird API. We should consider building a better public API for Metro and then consider how to build a new CLI on top of it and simplify our internal integration. I really don't like how Metro is integrated across pieces of the RN cli in ways that is hard to manage. But that is a larger task for another time :)

Reviewed By: jeanlauliac

Differential Revision: D5217726

fbshipit-source-id: 74efddbb87755a9e744c816fbc62efa21f6a79bf
  • Loading branch information...
cpojer authored and facebook-github-bot committed Jun 13, 2017
1 parent f704441 commit 29d9c35e122861d86c70b1e27a6e34eda4d82369
@@ -3,7 +3,6 @@ FROM containership/android-base:latest
# set default environment variables
ENV GRADLE_OPTS="-Dorg.gradle.jvmargs=\"-Xmx512m -XX:+HeapDumpOnOutOfMemoryError\""
ENV JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8"
ENV REACT_NATIVE_MAX_WORKERS=1
# add ReactAndroid directory
ADD .buckconfig /app/.buckconfig
@@ -17,7 +17,6 @@ AVD_UUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 8 | head -n 1)
ANDROID_NPM_DEPS="appium@1.5.1 mocha@2.4.5 wd@0.3.11 colors@1.0.3 pretty-data2@0.40.1"
CLI_PACKAGE=$ROOT/react-native-cli/react-native-cli-*.tgz
PACKAGE=$ROOT/react-native-*.tgz
REACT_NATIVE_MAX_WORKERS=1
# solve issue with max user watches limit
echo 65536 | tee -a /proc/sys/fs/inotify/max_user_watches
@@ -231,13 +230,13 @@ function e2e_suite() {
# js tests
if [ $RUN_JS -ne 0 ]; then
# Check the packager produces a bundle (doesn't throw an error)
REACT_NATIVE_MAX_WORKERS=1 react-native bundle --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js
react-native bundle --max-workers 1 --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js
if [ $? -ne 0 ]; then
echo "Could not build android bundle"
return 1
fi
REACT_NATIVE_MAX_WORKERS=1 react-native bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js
react-native bundle --max-workers 1 --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js
if [ $? -ne 0 ]; then
echo "Could not build iOS bundle"
return 1
View
@@ -69,7 +69,7 @@ test:
# integration tests
# build JS bundle for instrumentation tests
- REACT_NATIVE_MAX_WORKERS=1 node local-cli/cli.js bundle --platform android --dev true --entry-file ReactAndroid/src/androidTest/js/TestBundle.js --bundle-output ReactAndroid/src/androidTest/assets/AndroidTestBundle.js
- node local-cli/cli.js bundle --max-workers 1 --platform android --dev true --entry-file ReactAndroid/src/androidTest/js/TestBundle.js --bundle-output ReactAndroid/src/androidTest/assets/AndroidTestBundle.js
# build test APK
- buck install ReactAndroid/src/androidTest/buck-runner:instrumentation-tests --config build.threads=1
# run installed apk with tests
@@ -13,7 +13,7 @@
const log = require('../util/log').out('bundle');
const Server = require('metro-bundler/build/Server');
const Terminal = require('metro-bundler/build/lib/TerminalClass');
const Terminal = require('metro-bundler/build/lib/Terminal');
const TerminalReporter = require('metro-bundler/build/lib/TerminalReporter');
const TransformCaching = require('metro-bundler/build/lib/TransformCaching');
@@ -38,6 +38,7 @@ function buildBundle(
args: OutputOptions & {
assetsDest: mixed,
entryFile: string,
maxWorkers: number,
resetCache: boolean,
transformer: string,
},
@@ -89,6 +90,7 @@ function buildBundle(
getTransformOptions: config.getTransformOptions,
globalTransformCache: null,
hasteImpl: config.hasteImpl,
maxWorkers: args.maxWorkers,
platforms: defaultPlatforms.concat(platforms),
postMinifyProcess: config.postMinifyProcess,
postProcessModules: config.postProcessModules,
@@ -31,6 +31,12 @@ module.exports = [
command: '--bundle-encoding [string]',
description: 'Encoding the bundle should be written in (https://nodejs.org/api/buffer.html#buffer_buffer).',
default: 'utf8',
}, {
command: '--max-workers [number]',
description: 'Specifies the maximum number of workers the worker-pool ' +
'will spawn for transforming files. This defaults to the number of the ' +
'cores available on your machine.',
parse: (workers: string) => Number(workers),
}, {
command: '--sourcemap-output [string]',
description: 'File name where to store the sourcemap file for resulting bundle, ex. /tmp/groups.map',
@@ -96,6 +96,12 @@ module.exports = {
}, {
command: '--transformer [path]',
description: 'Specify a custom transformer to be used'
}, {
command: '--max-workers [number]',
description: 'Specifies the maximum number of workers the worker-pool ' +
'will spawn for transforming files. This defaults to the number of the ' +
'cores available on your machine.',
parse: (workers: string) => Number(workers),
}, {
command: '--dev [boolean]',
description: 'If false, skip all dev-only code path',
@@ -15,7 +15,7 @@
require('../../setupBabel')();
const InspectorProxy = require('./util/inspectorProxy.js');
const ReactPackager = require('metro-bundler');
const Terminal = require('metro-bundler/build/lib/TerminalClass');
const Terminal = require('metro-bundler/build/lib/Terminal');
const attachHMRServer = require('./util/attachHMRServer');
const connect = require('connect');
@@ -46,6 +46,7 @@ import type {Reporter} from 'metro-bundler/build/lib/reporting';
export type Args = {|
+assetExts: $ReadOnlyArray<string>,
+host: string,
+maxWorkers: number,
+nonPersistent: boolean,
+platforms: $ReadOnlyArray<string>,
+port: number,
@@ -160,10 +161,11 @@ function getPackagerServer(args, config) {
extraNodeModules: config.extraNodeModules,
getTransformOptions: config.getTransformOptions,
hasteImpl: config.hasteImpl,
maxWorkers: args.maxWorkers,
platforms: defaultPlatforms.concat(args.platforms),
polyfillModuleNames: config.getPolyfillModuleNames(),
postProcessModules: config.postProcessModules,
postMinifyProcess: config.postMinifyProcess,
postProcessModules: config.postProcessModules,
projectRoots: args.projectRoots,
providesModuleNodeModules: providesModuleNodeModules,
reporter: new LogReporter(terminal),
@@ -103,6 +103,12 @@ module.exports = {
}
return null;
},
}, {
command: '--max-workers [number]',
description: 'Specifies the maximum number of workers the worker-pool ' +
'will spawn for transforming files. This defaults to the number of the ' +
'cores available on your machine.',
parse: (workers: string) => Number(workers),
}, {
command: '--skipflow',
description: 'Disable flow checks'
View
@@ -10,9 +10,8 @@
*/
'use strict';
const findSymlinksPaths = require('./findSymlinksPaths');
const blacklist = require('metro-bundler/build/blacklist');
const findSymlinksPaths = require('./findSymlinksPaths');
const fs = require('fs');
const invariant = require('fbjs/lib/invariant');
const path = require('path');
View
@@ -182,7 +182,7 @@
"left-pad": "^1.1.3",
"lodash": "^4.16.6",
"merge-stream": "^1.0.1",
"metro-bundler": "^0.7.4",
"metro-bundler": "^0.8.1",
"mime": "^1.3.4",
"mime-types": "2.1.11",
"minimist": "^1.2.0",
@@ -125,12 +125,9 @@ try {
}
echo(`Starting packager server, ${SERVER_PID}`);
const packagerEnv = Object.create(process.env);
packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
// shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
const packagerProcess = spawn('npm', ['start'], {
// stdio: 'inherit',
env: packagerEnv
const packagerProcess = spawn('npm', ['start', '--', '--max-workers 1'], {
env: process.env
});
SERVER_PID = packagerProcess.pid;
// wait a bit to allow packager to startup
@@ -193,12 +190,12 @@ try {
if (argv.js) {
// Check the packager produces a bundle (doesn't throw an error)
if (exec('REACT_NATIVE_MAX_WORKERS=1 react-native bundle --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js').code) {
if (exec('react-native bundle --max-workers 1 --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js').code) {
echo('Could not build Android bundle');
exitCode = 1;
throw Error(exitCode);
}
if (exec('REACT_NATIVE_MAX_WORKERS=1 react-native bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js').code) {
if (exec('react-native --max-workers 1 bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js').code) {
echo('Could not build iOS bundle');
exitCode = 1;
throw Error(exitCode);

0 comments on commit 29d9c35

Please sign in to comment.