Skip to content
Permalink
Browse files

process: specialize building and storage of process.config

Instead of treating config.gypi as a JavaScript file, specialize
the processing in js2c and make the serialized result a real JSON
string (with 'true' and 'false' converted to boolean values) so
we don't have to use a custom deserializer during bootstrap.

In addition, store the JSON string separately in NativeModuleLoader,
and keep it separate from the map of the builtin source code, so
we don't have to put it onto `NativeModule._source` and delete it
later, though we still preserve it in `process.binding('natives')`,
which we don't use anymore.

This patch also makes the map of builtin source code and the
config.gypi string available through side-effect-free getters
in C++.

PR-URL: #24816
Reviewed-By: Gus Caplan <me@gus.host>
  • Loading branch information...
joyeecheung authored and addaleax committed Dec 2, 2018
1 parent 6528ce6 commit 0e2fbe4ff4c1da4bd69eff70ba07cbab816333df
@@ -6,18 +6,16 @@
// cannot be tampered with even with --expose-internals.

const { NativeModule } = require('internal/bootstrap/loaders');
const { getSource, compileCodeCache } = internalBinding('native_module');
const { source, compileCodeCache } = internalBinding('native_module');
const { hasTracing } = process.binding('config');

const source = getSource();
const depsModule = Object.keys(source).filter(
(key) => NativeModule.isDepsModule(key) || key.startsWith('internal/deps')
);

// Modules with source code compiled in js2c that
// cannot be compiled with the code cache.
const cannotUseCache = [
'config',
'sys', // Deprecated.
'internal/v8_prof_polyfill',
'internal/v8_prof_processor',
@@ -169,10 +169,15 @@ function NativeModule(id) {
this.loading = false;
}

NativeModule._source = getInternalBinding('natives');
const {
source,
compileFunction
} = internalBinding('native_module');

NativeModule._source = source;
NativeModule._cache = {};

const config = getInternalBinding('config');
const config = internalBinding('config');

// Think of this as module.exports in this file even though it is not
// written in CommonJS style.
@@ -323,7 +328,6 @@ NativeModule.prototype.proxifyExports = function() {
this.exports = new Proxy(this.exports, handler);
};

const { compileFunction } = getInternalBinding('native_module');
NativeModule.prototype.compile = function() {
const id = this.id;

@@ -59,7 +59,7 @@ function startup() {
}

perThreadSetup.setupAssert();
perThreadSetup.setupConfig(NativeModule._source);
perThreadSetup.setupConfig();

if (isMainThread) {
mainThreadSetup.setupSignalHandlers(internalBinding);
@@ -143,17 +143,9 @@ function setupMemoryUsage(_memoryUsage) {
};
}

function setupConfig(_source) {
// NativeModule._source
// used for `process.config`, but not a real module
const config = _source.config;
delete _source.config;

process.config = JSON.parse(config, function(key, value) {
if (value === 'true') return true;
if (value === 'false') return false;
return value;
});
function setupConfig() {
// Serialized config.gypi
process.config = JSON.parse(internalBinding('native_module').config);
}


@@ -140,6 +140,7 @@ constexpr size_t kFsStatsBufferLength = kFsStatsFieldsNumber * 2;
V(channel_string, "channel") \
V(chunks_sent_since_last_write_string, "chunksSentSinceLastWrite") \
V(code_string, "code") \
V(config_string, "config") \
V(constants_string, "constants") \
V(crypto_dsa_string, "dsa") \
V(crypto_ec_string, "ec") \
@@ -312,7 +313,7 @@ constexpr size_t kFsStatsBufferLength = kFsStatsFieldsNumber * 2;
V(write_host_object_string, "_writeHostObject") \
V(write_queue_size_string, "writeQueueSize") \
V(x_forwarded_string, "x-forwarded-for") \
V(zero_return_string, "ZERO_RETURN") \
V(zero_return_string, "ZERO_RETURN")

#define ENVIRONMENT_STRONG_PERSISTENT_PROPERTIES(V) \
V(as_external, v8::External) \
@@ -390,6 +390,13 @@ void GetInternalBinding(const FunctionCallbackInfo<Value>& args) {
DefineConstants(env->isolate(), exports);
} else if (!strcmp(*module_v, "natives")) {
exports = per_process_loader.GetSourceObject(env->context());
// Legacy feature: process.binding('natives').config contains stringified
// config.gypi
CHECK(exports
->Set(env->context(),
env->config_string(),
per_process_loader.GetConfigString(env->isolate()))
.FromJust());
} else {
return ThrowIfNoSuchModule(env, *module_v);
}
@@ -9,6 +9,7 @@ using v8::Array;
using v8::ArrayBuffer;
using v8::ArrayBufferCreationMode;
using v8::Context;
using v8::DEFAULT;
using v8::EscapableHandleScope;
using v8::Function;
using v8::FunctionCallbackInfo;
@@ -19,11 +20,15 @@ using v8::Isolate;
using v8::Local;
using v8::Maybe;
using v8::MaybeLocal;
using v8::Name;
using v8::None;
using v8::Object;
using v8::PropertyCallbackInfo;
using v8::Script;
using v8::ScriptCompiler;
using v8::ScriptOrigin;
using v8::Set;
using v8::SideEffectType;
using v8::String;
using v8::Uint8Array;
using v8::Value;
@@ -70,25 +75,35 @@ void NativeModuleLoader::GetCacheUsage(
args.GetReturnValue().Set(result);
}

void NativeModuleLoader::GetSourceObject(
const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
args.GetReturnValue().Set(per_process_loader.GetSourceObject(env->context()));
void NativeModuleLoader::SourceObjectGetter(
Local<Name> property, const PropertyCallbackInfo<Value>& info) {
Local<Context> context = info.GetIsolate()->GetCurrentContext();
info.GetReturnValue().Set(per_process_loader.GetSourceObject(context));
}

void NativeModuleLoader::ConfigStringGetter(
Local<Name> property, const PropertyCallbackInfo<Value>& info) {
info.GetReturnValue().Set(
per_process_loader.GetConfigString(info.GetIsolate()));
}

Local<Object> NativeModuleLoader::GetSourceObject(
Local<Context> context) const {
return MapToObject(context, source_);
}

Local<String> NativeModuleLoader::GetConfigString(Isolate* isolate) const {
return config_.ToStringChecked(isolate);
}

Local<String> NativeModuleLoader::GetSource(Isolate* isolate,
const char* id) const {
const auto it = source_.find(id);
CHECK_NE(it, source_.end());
return it->second.ToStringChecked(isolate);
}

NativeModuleLoader::NativeModuleLoader() {
NativeModuleLoader::NativeModuleLoader() : config_(GetConfig()) {
LoadJavaScriptSource();
LoadJavaScriptHash();
LoadCodeCache();
@@ -321,8 +336,27 @@ void NativeModuleLoader::Initialize(Local<Object> target,
void* priv) {
Environment* env = Environment::GetCurrent(context);

env->SetMethod(
target, "getSource", NativeModuleLoader::GetSourceObject);
CHECK(target
->SetAccessor(env->context(),
env->config_string(),
ConfigStringGetter,
nullptr,
MaybeLocal<Value>(),
DEFAULT,
None,
SideEffectType::kHasNoSideEffect)
.FromJust());
CHECK(target
->SetAccessor(env->context(),
env->source_string(),
SourceObjectGetter,
nullptr,
MaybeLocal<Value>(),
DEFAULT,
None,
SideEffectType::kHasNoSideEffect)
.FromJust());

env->SetMethod(
target, "getCacheUsage", NativeModuleLoader::GetCacheUsage);
env->SetMethod(
@@ -40,6 +40,9 @@ class NativeModuleLoader {
v8::Local<v8::Context> context,
void* priv);
v8::Local<v8::Object> GetSourceObject(v8::Local<v8::Context> context) const;
// Returns config.gypi as a JSON string
v8::Local<v8::String> GetConfigString(v8::Isolate* isolate) const;

v8::Local<v8::String> GetSource(v8::Isolate* isolate, const char* id) const;

// Run a script with JS source bundled inside the binary as if it's wrapped
@@ -56,9 +59,15 @@ class NativeModuleLoader {

private:
static void GetCacheUsage(const v8::FunctionCallbackInfo<v8::Value>& args);
// For legacy process.binding('natives') which is mutable, and for
// internalBinding('native_module').source for internal use
static void GetSourceObject(const v8::FunctionCallbackInfo<v8::Value>& args);
// Passing map of builtin module source code into JS land as
// internalBinding('native_module').source
static void SourceObjectGetter(
v8::Local<v8::Name> property,
const v8::PropertyCallbackInfo<v8::Value>& info);
// Passing config.gypi into JS land as internalBinding('native_module').config
static void ConfigStringGetter(
v8::Local<v8::Name> property,
const v8::PropertyCallbackInfo<v8::Value>& info);
// Compile code cache for a specific native module
static void CompileCodeCache(const v8::FunctionCallbackInfo<v8::Value>& args);
// Compile a specific native module as a function
@@ -67,6 +76,7 @@ class NativeModuleLoader {
// Generated by tools/js2c.py as node_javascript.cc
void LoadJavaScriptSource(); // Loads data into source_
void LoadJavaScriptHash(); // Loads data into source_hash_
UnionBytes GetConfig(); // Return data for config.gypi

// Generated by tools/generate_code_cache.js as node_code_cache.cc when
// the build is configured with --code-cache-path=.... They are noops
@@ -94,6 +104,8 @@ class NativeModuleLoader {
bool has_code_cache_ = false;
NativeModuleRecordMap source_;
NativeModuleRecordMap code_cache_;
UnionBytes config_;

NativeModuleHashMap source_hash_;
NativeModuleHashMap code_cache_hash_;
};
@@ -55,21 +55,31 @@ class UnionBytes {
: is_one_byte_(false), two_bytes_(data), length_(length) {}
UnionBytes(const uint8_t* data, size_t length)
: is_one_byte_(true), one_bytes_(data), length_(length) {}

UnionBytes(const UnionBytes&) = default;
UnionBytes& operator=(const UnionBytes&) = default;
UnionBytes(UnionBytes&&) = default;
UnionBytes& operator=(UnionBytes&&) = default;

bool is_one_byte() const { return is_one_byte_; }
const uint16_t* two_bytes_data() const {
CHECK(!is_one_byte_);
CHECK_NE(two_bytes_, nullptr);
return two_bytes_;
}
const uint8_t* one_bytes_data() const {
CHECK(is_one_byte_);
CHECK_NE(one_bytes_, nullptr);
return one_bytes_;
}
v8::Local<v8::String> ToStringChecked(v8::Isolate* isolate) const {
if (is_one_byte_) {
CHECK_NE(one_bytes_, nullptr);
NonOwningExternalOneByteResource* source =
new NonOwningExternalOneByteResource(one_bytes_, length_);
return v8::String::NewExternalOneByte(isolate, source).ToLocalChecked();
} else {
CHECK_NE(two_bytes_, nullptr);
NonOwningExternalTwoByteResource* source =
new NonOwningExternalTwoByteResource(two_bytes_, length_);
return v8::String::NewExternalTwoByte(isolate, source).ToLocalChecked();
@@ -9,7 +9,7 @@ const common = require('../common');
const assert = require('assert');

const isMainThread = common.isMainThread;
const kMaxModuleCount = isMainThread ? 59 : 80;
const kMaxModuleCount = isMainThread ? 61 : 82;

assert(list.length <= kMaxModuleCount,
`Total length: ${list.length}\n` + list.join('\n')
@@ -193,6 +193,10 @@ def ReadMacros(lines):
{hash_initializers}
}}
UnionBytes NativeModuleLoader::GetConfig() {{
return UnionBytes(config_raw, arraysize(config_raw)); // config.gypi
}}
}} // namespace native_module
}} // namespace node
@@ -248,21 +252,23 @@ def JS2C(source, target):
definitions = []
initializers = []
hash_initializers = []
config_initializers = []

def AddModule(module, source):
var = '%s_raw' % (module.replace('-', '_').replace('/', '_'))
source_hash = hashlib.sha256(source).hexdigest()

def GetDefinition(var, source):
# Treat non-ASCII as UTF-8 and convert it to UTF-16.
if any(ord(c) > 127 for c in source):
source = map(ord, source.decode('utf-8').encode('utf-16be'))
source = [source[i] * 256 + source[i+1] for i in xrange(0, len(source), 2)]
source = ToCArray(source)
definition = TWO_BYTE_STRING.format(var=var, data=source)
return TWO_BYTE_STRING.format(var=var, data=source)
else:
source = ToCArray(map(ord, source), step=20)
definition = ONE_BYTE_STRING.format(var=var, data=source)
return ONE_BYTE_STRING.format(var=var, data=source)

def AddModule(module, source):
var = '%s_raw' % (module.replace('-', '_').replace('/', '_'))
source_hash = hashlib.sha256(source).hexdigest()
definition = GetDefinition(var, source)
initializer = INITIALIZER.format(module=module,
var=var)
hash_initializer = HASH_INITIALIZER.format(module=module,
@@ -292,11 +298,17 @@ def AddModule(module, source):

# if its a gypi file we're going to want it as json
# later on anyway, so get it out of the way now
if name.endswith(".gypi"):
if name.endswith('.gypi'):
# Currently only config.gypi is allowed
assert name == 'config.gypi'
lines = re.sub(r'\'true\'', 'true', lines)
lines = re.sub(r'\'false\'', 'false', lines)
lines = re.sub(r'#.*?\n', '', lines)
lines = re.sub(r'\'', '"', lines)

AddModule(name.split('.', 1)[0], lines)
definition = GetDefinition('config_raw', lines)
definitions.append(definition)
else:
AddModule(name.split('.', 1)[0], lines)

# Add deprecated aliases for deps without 'deps/'
if deprecated_deps is not None:
@@ -306,9 +318,11 @@ def AddModule(module, source):

# Emit result
output = open(str(target[0]), "w")
output.write(TEMPLATE.format(definitions=''.join(definitions),
initializers=''.join(initializers),
hash_initializers=''.join(hash_initializers)))
output.write(
TEMPLATE.format(definitions=''.join(definitions),
initializers=''.join(initializers),
hash_initializers=''.join(hash_initializers),
config_initializers=''.join(config_initializers)))
output.close()

def main():

0 comments on commit 0e2fbe4

Please sign in to comment.
You can’t perform that action at this time.