Skip to content

Commit

Permalink
build: pass directory instead of list of files to js2c.py
Browse files Browse the repository at this point in the history
On Windows there is a limit to the length of commands, so there
will be an error once the lengths of the JS file names combined
exceed that limit. This patch modifies js2c.py so that
it now takes a --directory argument to glob for .js and
.mjs files in addition to the list of files passed directly.
We still pass the additional files we include from deps/
directly through the command line, as we only includes some of
them so we cannot simply glob, but those are limited so listing
them out should be fine.

Refs: https://docs.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation

PR-URL: #39069
Refs: #38971
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
  • Loading branch information
joyeecheung authored and targos committed Jul 11, 2021
1 parent c158888 commit 412b101
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 247 deletions.
265 changes: 19 additions & 246 deletions node.gyp
Original file line number Diff line number Diff line change
Expand Up @@ -27,251 +27,15 @@
'node_lib_target_name%': 'libnode',
'node_intermediate_lib_type%': 'static_library',
'node_builtin_modules_path%': '',
# We list the deps/ files out instead of globbing them in js2c.py since we
# only include a subset of all the files under these directories.
# The lengths of their file names combined should not exceed the
# Windows command length limit or there would be an error.
# See https://docs.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation
'library_files': [
'lib/internal/bootstrap/environment.js',
'lib/internal/bootstrap/loaders.js',
'lib/internal/bootstrap/node.js',
'lib/internal/bootstrap/pre_execution.js',
'lib/internal/bootstrap/switches/does_own_process_state.js',
'lib/internal/bootstrap/switches/does_not_own_process_state.js',
'lib/internal/bootstrap/switches/is_main_thread.js',
'lib/internal/bootstrap/switches/is_not_main_thread.js',
'lib/internal/per_context/primordials.js',
'lib/internal/per_context/domexception.js',
'lib/internal/per_context/messageport.js',
'lib/async_hooks.js',
'lib/assert.js',
'lib/assert/strict.js',
'lib/buffer.js',
'lib/child_process.js',
'lib/console.js',
'lib/constants.js',
'lib/crypto.js',
'lib/cluster.js',
'lib/diagnostics_channel.js',
'lib/dgram.js',
'lib/dns.js',
'lib/dns/promises.js',
'lib/domain.js',
'lib/events.js',
'lib/fs.js',
'lib/fs/promises.js',
'lib/http.js',
'lib/http2.js',
'lib/_http_agent.js',
'lib/_http_client.js',
'lib/_http_common.js',
'lib/_http_incoming.js',
'lib/_http_outgoing.js',
'lib/_http_server.js',
'lib/https.js',
'lib/inspector.js',
'lib/module.js',
'lib/net.js',
'lib/os.js',
'lib/path.js',
'lib/path/posix.js',
'lib/path/win32.js',
'lib/perf_hooks.js',
'lib/process.js',
'lib/punycode.js',
'lib/querystring.js',
'lib/readline.js',
'lib/repl.js',
'lib/stream.js',
'lib/stream/promises.js',
'lib/_stream_readable.js',
'lib/_stream_writable.js',
'lib/_stream_duplex.js',
'lib/_stream_transform.js',
'lib/_stream_passthrough.js',
'lib/_stream_wrap.js',
'lib/string_decoder.js',
'lib/sys.js',
'lib/timers/promises.js',
'lib/timers.js',
'lib/tls.js',
'lib/_tls_common.js',
'lib/_tls_wrap.js',
'lib/trace_events.js',
'lib/tty.js',
'lib/url.js',
'lib/util.js',
'lib/util/types.js',
'lib/v8.js',
'lib/vm.js',
'lib/wasi.js',
'lib/worker_threads.js',
'lib/zlib.js',
'lib/internal/abort_controller.js',
'lib/internal/assert.js',
'lib/internal/assert/assertion_error.js',
'lib/internal/assert/calltracker.js',
'lib/internal/async_hooks.js',
'lib/internal/blob.js',
'lib/internal/blocklist.js',
'lib/internal/buffer.js',
'lib/internal/cli_table.js',
'lib/internal/child_process.js',
'lib/internal/child_process/serialization.js',
'lib/internal/cluster/child.js',
'lib/internal/cluster/primary.js',
'lib/internal/cluster/round_robin_handle.js',
'lib/internal/cluster/shared_handle.js',
'lib/internal/cluster/utils.js',
'lib/internal/cluster/worker.js',
'lib/internal/console/constructor.js',
'lib/internal/console/global.js',
'lib/internal/crypto/aes.js',
'lib/internal/crypto/certificate.js',
'lib/internal/crypto/cipher.js',
'lib/internal/crypto/diffiehellman.js',
'lib/internal/crypto/dsa.js',
'lib/internal/crypto/ec.js',
'lib/internal/crypto/hash.js',
'lib/internal/crypto/hashnames.js',
'lib/internal/crypto/hkdf.js',
'lib/internal/crypto/keygen.js',
'lib/internal/crypto/keys.js',
'lib/internal/crypto/mac.js',
'lib/internal/crypto/pbkdf2.js',
'lib/internal/crypto/random.js',
'lib/internal/crypto/rsa.js',
'lib/internal/crypto/scrypt.js',
'lib/internal/crypto/sig.js',
'lib/internal/crypto/util.js',
'lib/internal/crypto/webcrypto.js',
'lib/internal/crypto/x509.js',
'lib/internal/constants.js',
'lib/internal/debugger/_inspect.js',
'lib/internal/debugger/inspect_client.js',
'lib/internal/debugger/inspect_repl.js',
'lib/internal/dgram.js',
'lib/internal/dns/promises.js',
'lib/internal/dns/utils.js',
'lib/internal/dtrace.js',
'lib/internal/encoding.js',
'lib/internal/errors.js',
'lib/internal/error_serdes.js',
'lib/internal/event_target.js',
'lib/internal/fixed_queue.js',
'lib/internal/freelist.js',
'lib/internal/freeze_intrinsics.js',
'lib/internal/fs/dir.js',
'lib/internal/fs/promises.js',
'lib/internal/fs/read_file_context.js',
'lib/internal/fs/rimraf.js',
'lib/internal/fs/streams.js',
'lib/internal/fs/sync_write_stream.js',
'lib/internal/fs/utils.js',
'lib/internal/fs/watchers.js',
'lib/internal/http.js',
'lib/internal/heap_utils.js',
'lib/internal/histogram.js',
'lib/internal/idna.js',
'lib/internal/inspector_async_hook.js',
'lib/internal/js_stream_socket.js',
'lib/internal/legacy/processbinding.js',
'lib/internal/linkedlist.js',
'lib/internal/main/check_syntax.js',
'lib/internal/main/eval_string.js',
'lib/internal/main/eval_stdin.js',
'lib/internal/main/inspect.js',
'lib/internal/main/print_help.js',
'lib/internal/main/prof_process.js',
'lib/internal/main/repl.js',
'lib/internal/main/run_main_module.js',
'lib/internal/main/worker_thread.js',
'lib/internal/modules/run_main.js',
'lib/internal/modules/package_json_reader.js',
'lib/internal/modules/cjs/helpers.js',
'lib/internal/modules/cjs/loader.js',
'lib/internal/modules/esm/loader.js',
'lib/internal/modules/esm/create_dynamic_module.js',
'lib/internal/modules/esm/get_format.js',
'lib/internal/modules/esm/get_source.js',
'lib/internal/modules/esm/module_job.js',
'lib/internal/modules/esm/module_map.js',
'lib/internal/modules/esm/resolve.js',
'lib/internal/modules/esm/transform_source.js',
'lib/internal/modules/esm/translators.js',
'lib/internal/net.js',
'lib/internal/options.js',
'lib/internal/perf/perf.js',
'lib/internal/perf/nodetiming.js',
'lib/internal/perf/usertiming.js',
'lib/internal/perf/observe.js',
'lib/internal/perf/event_loop_delay.js',
'lib/internal/perf/event_loop_utilization.js',
'lib/internal/perf/timerify.js',
'lib/internal/policy/manifest.js',
'lib/internal/policy/sri.js',
'lib/internal/priority_queue.js',
'lib/internal/process/esm_loader.js',
'lib/internal/process/execution.js',
'lib/internal/process/per_thread.js',
'lib/internal/process/policy.js',
'lib/internal/process/promises.js',
'lib/internal/process/warning.js',
'lib/internal/process/worker_thread_only.js',
'lib/internal/process/report.js',
'lib/internal/process/signal.js',
'lib/internal/process/task_queues.js',
'lib/internal/querystring.js',
'lib/internal/readline/callbacks.js',
'lib/internal/readline/emitKeypressEvents.js',
'lib/internal/readline/utils.js',
'lib/internal/repl.js',
'lib/internal/repl/await.js',
'lib/internal/repl/history.js',
'lib/internal/repl/utils.js',
'lib/internal/socketaddress.js',
'lib/internal/socket_list.js',
'lib/internal/source_map/prepare_stack_trace.js',
'lib/internal/source_map/source_map.js',
'lib/internal/source_map/source_map_cache.js',
'lib/internal/test/binding.js',
'lib/internal/test/transfer.js',
'lib/internal/timers.js',
'lib/internal/tls.js',
'lib/internal/trace_events_async_hooks.js',
'lib/internal/tty.js',
'lib/internal/url.js',
'lib/internal/util.js',
'lib/internal/util/comparisons.js',
'lib/internal/util/debuglog.js',
'lib/internal/util/inspect.js',
'lib/internal/util/inspector.js',
'lib/internal/util/iterable_weak_map.js',
'lib/internal/util/types.js',
'lib/internal/http2/core.js',
'lib/internal/http2/compat.js',
'lib/internal/http2/util.js',
'lib/internal/v8_prof_polyfill.js',
'lib/internal/v8_prof_processor.js',
'lib/internal/validators.js',
'lib/internal/stream_base_commons.js',
'lib/internal/vm/module.js',
'lib/internal/worker.js',
'lib/internal/worker/io.js',
'lib/internal/worker/js_transferable.js',
'lib/internal/watchdog.js',
'lib/internal/streams/lazy_transform.js',
'lib/internal/streams/add-abort-signal.js',
'lib/internal/streams/buffer_list.js',
'lib/internal/streams/duplexpair.js',
'lib/internal/streams/from.js',
'lib/internal/streams/legacy.js',
'lib/internal/streams/readable.js',
'lib/internal/streams/writable.js',
'lib/internal/streams/duplex.js',
'lib/internal/streams/passthrough.js',
'lib/internal/streams/transform.js',
'lib/internal/streams/destroy.js',
'lib/internal/streams/state.js',
'lib/internal/streams/pipeline.js',
'lib/internal/streams/end-of-stream.js',
'lib/internal/streams/utils.js',
'<!@(python tools/search_files.py --ext js lib)',
],
'deps_files': [
'deps/v8/tools/splaytree.mjs',
'deps/v8/tools/codemap.mjs',
'deps/v8/tools/consarray.mjs',
Expand Down Expand Up @@ -794,6 +558,7 @@
'deps/v8/include/v8.h',
# javascript files to make for an even more pleasant IDE experience
'<@(library_files)',
'<@(deps_files)',
# node.gyp is added by default, common.gypi is added for change detection
'common.gypi',
],
Expand Down Expand Up @@ -1047,14 +812,21 @@
# Put the code first so it's a dependency and can be used for invocation.
'tools/js2c.py',
'<@(library_files)',
'<@(deps_files)',
'config.gypi'
],
'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
],
'action': [
'python', '<@(_inputs)',
'--target', '<@(_outputs)',
'python',
'tools/js2c.py',
'--directory',
'lib',
'--target',
'<@(_outputs)',
'config.gypi',
'<@(deps_files)',
],
},
],
Expand Down Expand Up @@ -1598,6 +1370,7 @@
],
'sources': [
'<@(library_files)',
'<@(deps_files)',
'common.gypi',
],
'direct_dependent_settings': {
Expand Down
14 changes: 13 additions & 1 deletion tools/js2c.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import re
import functools
import codecs
import utils

def ReadFile(filename):
if is_verbose:
Expand Down Expand Up @@ -204,12 +205,23 @@ def main():
fromfile_prefix_chars='@'
)
parser.add_argument('--target', help='output file')
parser.add_argument(
'--directory',
default=None,
help='input file directory')
parser.add_argument('--verbose', action='store_true', help='output file')
parser.add_argument('sources', nargs='*', help='input files')
options = parser.parse_args()
global is_verbose
is_verbose = options.verbose
source_files = functools.reduce(SourceFileByExt, options.sources, {})
sources = options.sources
if options.directory is not None:
js_files = utils.SearchFiles(options.directory, 'js')
mjs_files = utils.SearchFiles(options.directory, 'mjs')
sources = js_files + mjs_files + options.sources

source_files = functools.reduce(SourceFileByExt, sources, {})

# Should have exactly 3 types: `.js`, `.mjs` and `.gypi`
assert len(source_files) == 3
# Currently config.gypi is the only `.gypi` file allowed
Expand Down
22 changes: 22 additions & 0 deletions tools/search_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env python

"""
This is a utility for recursively searching files under
a specified directory
"""

import argparse
import utils

def main():
parser = argparse.ArgumentParser(
description='Search files with a specific extension under a directory',
fromfile_prefix_chars='@'
)
parser.add_argument('--ext', required=True, help='extension to search for')
parser.add_argument('directory', help='input directory')
options = parser.parse_args()
print('\n'.join(utils.SearchFiles(options.directory, options.ext)))

if __name__ == "__main__":
main()
9 changes: 9 additions & 0 deletions tools/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


import glob
import platform
import re
import sys


# Reads a .list file into an array of strings
Expand Down Expand Up @@ -106,3 +108,10 @@ def GuessArchitecture():

def IsWindows():
return GuessOS() == 'win32'


def SearchFiles(dir, ext):
list = glob.glob(dir+ '/**/*.' + ext, recursive=True)
if sys.platform == 'win32':
list = [ x.replace('\\', '/')for x in list]
return list

0 comments on commit 412b101

Please sign in to comment.