Skip to content
Permalink
Browse files

tools: implement mkcodecache as an executable

This patch implement a mkcodecache executable on top of the
`NativeModuleLoader` singleton.
This makes it possible to build a Node.js binary with embedded
code cache without building itself using the code cache stub -
the cache is now initialized by `NativeModuleEnv` instead which
can be refactored out of the mkcodecache dependencies.

PR-URL: #27161
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
  • Loading branch information...
joyeecheung authored and refack committed Apr 3, 2019
1 parent 1c26169 commit 4fd71935795fa7c284f5ed621551b65a28b8271c
@@ -115,7 +115,7 @@ with-code-cache:
$(PYTHON) ./configure $(CONFIG_FLAGS)
$(MAKE)
mkdir -p $(CODE_CACHE_DIR)
out/$(BUILDTYPE)/$(NODE_EXE) --expose-internals tools/generate_code_cache.js $(CODE_CACHE_FILE)
out/$(BUILDTYPE)/mkcodecache $(CODE_CACHE_FILE)
$(PYTHON) ./configure --code-cache-path $(CODE_CACHE_FILE) $(CONFIG_FLAGS)
$(MAKE)

@@ -1232,6 +1232,8 @@ LINT_CPP_FILES = $(filter-out $(LINT_CPP_EXCLUDE), $(wildcard \
test/node-api/*/*.h \
tools/icu/*.cc \
tools/icu/*.h \
tools/code_cache/*.cc \
tools/code_cache/*.h \
))

# Code blocks don't have newline at the end,
@@ -1103,6 +1103,59 @@
}],
],
}, # cctest
# TODO(joyeecheung): do not depend on node_lib,
# instead create a smaller static library node_lib_base that does
# just enough for node_native_module.cc and the cache builder to
# compile without compiling the generated code cache C++ file.
# So generate_code_cache -> mkcodecache -> node_lib_base,
# node_lib -> node_lib_base & generate_code_cache
{
'target_name': 'mkcodecache',
'type': 'executable',

'dependencies': [
'<(node_lib_target_name)',
'deps/histogram/histogram.gyp:histogram',
],

'includes': [
'node.gypi'
],

'include_dirs': [
'src',
'tools/msvs/genfiles',
'deps/v8/include',
'deps/cares/include',
'deps/uv/include',
],

'defines': [ 'NODE_WANT_INTERNALS=1' ],

'sources': [
'tools/code_cache/mkcodecache.cc',
'tools/code_cache/cache_builder.cc'
],

'conditions': [
[ 'node_report=="true"', {
'conditions': [
['OS=="win"', {
'libraries': [
'dbghelp.lib',
'PsApi.lib',
'Ws2_32.lib',
],
'dll_files': [
'dbghelp.dll',
'PsApi.dll',
'Ws2_32.dll',
],
}],
],
}],
],
}, # cache_builder
], # end targets

'conditions': [
@@ -4,7 +4,6 @@
namespace node {
namespace native_module {

using v8::ArrayBuffer;
using v8::Context;
using v8::DEFAULT;
using v8::Function;
@@ -18,11 +17,9 @@ using v8::Name;
using v8::None;
using v8::Object;
using v8::PropertyCallbackInfo;
using v8::ScriptCompiler;
using v8::Set;
using v8::SideEffectType;
using v8::String;
using v8::Uint8Array;
using v8::Value;

// TODO(joyeecheung): make these more general and put them into util.h
@@ -154,26 +151,6 @@ MaybeLocal<Function> NativeModuleEnv::LookupAndCompile(
return maybe;
}

// This is supposed to be run only by the main thread in
// tools/generate_code_cache.js
void NativeModuleEnv::GetCodeCache(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
Isolate* isolate = env->isolate();
CHECK(env->is_main_thread());

CHECK(args[0]->IsString());
node::Utf8Value id_v(isolate, args[0].As<String>());
const char* id = *id_v;

ScriptCompiler::CachedData* cached_data =
NativeModuleLoader::GetInstance()->GetCodeCache(id);
if (cached_data != nullptr) {
Local<ArrayBuffer> buf = ArrayBuffer::New(isolate, cached_data->length);
memcpy(buf->GetContents().Data(), cached_data->data, cached_data->length);
args.GetReturnValue().Set(Uint8Array::New(buf, 0, cached_data->length));
}
}

// TODO(joyeecheung): It is somewhat confusing that Class::Initialize
// is used to initilaize to the binding, but it is the current convention.
// Rename this across the code base to something that makes more sense.
@@ -216,7 +193,6 @@ void NativeModuleEnv::Initialize(Local<Object> target,
.Check();

env->SetMethod(target, "getCacheUsage", NativeModuleEnv::GetCacheUsage);
env->SetMethod(target, "getCodeCache", NativeModuleEnv::GetCodeCache);
env->SetMethod(target, "compileFunction", NativeModuleEnv::CompileFunction);
// internalBinding('native_module') should be frozen
target->SetIntegrityLevel(context, IntegrityLevel::kFrozen).FromJust();
@@ -52,7 +52,6 @@ class NativeModuleEnv {
const v8::PropertyCallbackInfo<v8::Value>& info);
// Compile a specific native module as a function
static void CompileFunction(const v8::FunctionCallbackInfo<v8::Value>& args);
static void GetCodeCache(const v8::FunctionCallbackInfo<v8::Value>& args);
};

} // namespace native_module
@@ -1,28 +1,40 @@
'use strict';

// This test verifies that the binary is compiled with code cache and the
// cache is used when built in modules are compiled.
// This test verifies the code cache generator can generate a C++
// file that contains the code cache. This can be removed once we
// actually build that C++ file into our binary.

const common = require('../common');

const tmpdir = require('../common/tmpdir');
const { spawnSync } = require('child_process');
const assert = require('assert');
const path = require('path');
const fs = require('fs');
const readline = require('readline');

const generator = path.join(
__dirname, '..', '..', 'tools', 'generate_code_cache.js'
);
console.log('Looking for mkcodecache executable');
let buildDir;
const stat = fs.statSync(process.execPath);
if (stat.isSymbolicLink()) {
console.log('Binary is a symbolic link');
buildDir = path.dirname(fs.readlinkSync(process.execPath));
} else {
buildDir = path.dirname(process.execPath);
}

const ext = common.isWindows ? '.exe' : '';
const generator = path.join(buildDir, `mkcodecache${ext}`);
if (!fs.existsSync(generator)) {
common.skip('Could not find mkcodecache');
}

console.log(`mkcodecache is ${generator}`);

tmpdir.refresh();
const dest = path.join(tmpdir.path, 'cache.cc');

// Run tools/generate_code_cache.js
const child = spawnSync(
process.execPath,
['--expose-internals', generator, dest]
);
// Run mkcodecache
const child = spawnSync(generator, [dest]);
assert.ifError(child.error);
if (child.status !== 0) {
console.log(child.stderr.toString());
@@ -0,0 +1,165 @@
#include "cache_builder.h"
#include <iostream>
#include <map>
#include <sstream>
#include <vector>
#include <cstdlib>
#include "util.h"

#include "node_native_module.h"

namespace node {
namespace native_module {

using v8::Context;
using v8::Function;
using v8::Isolate;
using v8::Local;
using v8::MaybeLocal;
using v8::ScriptCompiler;

static std::string GetDefName(const std::string& id) {
char buf[64] = {0};
size_t size = id.size();
CHECK_LT(size, sizeof(buf));
for (size_t i = 0; i < size; ++i) {
char ch = id[i];
buf[i] = (ch == '-' || ch == '/') ? '_' : ch;
}
return buf;
}

static std::string FormatSize(size_t size) {
char buf[64] = {0};
if (size < 1024) {
snprintf(buf, sizeof(buf), "%.2fB", static_cast<double>(size));
} else if (size < 1024 * 1024) {
snprintf(buf, sizeof(buf), "%.2fKB", static_cast<double>(size / 1024));
} else {
snprintf(
buf, sizeof(buf), "%.2fMB", static_cast<double>(size / 1024 / 1024));
}
return buf;
}

static std::string GetDefinition(const std::string& id,
size_t size,
const uint8_t* data) {
std::stringstream ss;
ss << "static const uint8_t " << GetDefName(id) << "[] = {\n";
for (size_t i = 0; i < size; ++i) {
uint8_t ch = data[i];
ss << std::to_string(ch) << (i == size - 1 ? '\n' : ',');
}
ss << "};";
return ss.str();
}

static std::string GetInitializer(const std::string& id) {
std::string def_name = GetDefName(id);
char buf[256] = {0};
snprintf(buf,
sizeof(buf),
"code_cache->emplace(\n"
" \"%s\",\n"
" std::make_unique<v8::ScriptCompiler::CachedData>"
"(%s, static_cast<int>(arraysize(%s)), policy)\n"
");",
id.c_str(),
def_name.c_str(),
def_name.c_str());
return buf;
}

static std::string GenerateCodeCache(
std::map<std::string, ScriptCompiler::CachedData*> data,
std::vector<std::string> ids,
bool log_progress) {
std::stringstream ss;
ss << R"(#include <cinttypes>
#include "node_native_module_env.h"
// This file is generated by tools/mkcodecache
// and is used when configure is run with \`--code-cache-path\`
namespace node {
namespace native_module {
)";

size_t total = 0;
for (const auto& x : data) {
const std::string& id = x.first;
ScriptCompiler::CachedData* cached_data = x.second;
total += cached_data->length;
std::string def = GetDefinition(id, cached_data->length, cached_data->data);
ss << def << "\n\n";
if (log_progress) {
std::cout << "Generated cache for " << id
<< ", size = " << FormatSize(cached_data->length)
<< ", total = " << FormatSize(total) << "\n";
}
}

ss << R"(void NativeModuleEnv::InitializeCodeCache() {
NativeModuleCacheMap* code_cache =
NativeModuleLoader::GetInstance()->code_cache();
if (!code_cache->empty()) {
return;
}
auto policy = v8::ScriptCompiler::CachedData::BufferPolicy::BufferNotOwned;
)";

for (const auto& x : data) {
const std::string& id = x.first;
ss << GetInitializer(id) << "\n\n";
}

ss << R"(}
} // namespace native_module
} // namespace node
)";
return ss.str();
}

std::string CodeCacheBuilder::Generate(Local<Context> context) {
NativeModuleLoader* loader = NativeModuleLoader::GetInstance();
std::vector<std::string> ids = loader->GetModuleIds();

std::vector<std::string> modules;
modules.reserve(ids.size());

std::map<std::string, ScriptCompiler::CachedData*> data;

NativeModuleLoader::Result result;
for (const auto& id : ids) {
// TODO(joyeecheung): we can only compile the modules that can be
// required here because the parameters for other types of builtins
// are still very flexible. We should look into auto-generating
// the paramters from the source somehow.
if (loader->CanBeRequired(id.c_str())) {
modules.push_back(id);
USE(loader->CompileAsModule(context, id.c_str(), &result));
ScriptCompiler::CachedData* cached_data =
loader->GetCodeCache(id.c_str());
if (cached_data == nullptr) {
// TODO(joyeecheung): display syntax errors
std::cerr << "Failed to complile " << id << "\n";
} else {
data.emplace(id, cached_data);
}
}
}

char env_buf[32];
size_t env_size = sizeof(env_buf);
int ret = uv_os_getenv("NODE_DEBUG", env_buf, &env_size);
bool log_progress = false;
if (ret == 0 && strcmp(env_buf, "mkcodecache") == 0) {
log_progress = true;
}
return GenerateCodeCache(data, modules, log_progress);
}

} // namespace native_module
} // namespace node
@@ -0,0 +1,16 @@
#ifndef TOOLS_CODE_CACHE_CACHE_BUILDER_H_
#define TOOLS_CODE_CACHE_CACHE_BUILDER_H_

#include <string>
#include "v8.h"

namespace node {
namespace native_module {
class CodeCacheBuilder {
public:
static std::string Generate(v8::Local<v8::Context> context);
};
} // namespace native_module
} // namespace node

#endif // TOOLS_CODE_CACHE_CACHE_BUILDER_H_

0 comments on commit 4fd7193

Please sign in to comment.
You can’t perform that action at this time.