Skip to content

Sourcemap support for Binaryen C/JS #1392

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kripken merged 2 commits into from
Jan 29, 2018
Merged

Sourcemap support for Binaryen C/JS #1392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a5b107e
Initial source map support for C/JS
dcodeIO Jan 27, 2018
c7bc4cf
Also test getDebugInfoFileName
dcodeIO Jan 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
271 changes: 136 additions & 135 deletions bin/binaryen.js
View file
Open in desktop

Large diffs are not rendered by default.

40 changes: 20 additions & 20 deletions bin/wasm.js
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions build-js.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,8 +540,11 @@ export_function "_BinaryenSetDebugInfo"
export_function "_BinaryenModuleRunPasses"
export_function "_BinaryenModuleAutoDrop"
export_function "_BinaryenModuleWrite"
export_function "_BinaryenModuleWriteWithSourceMap"
export_function "_BinaryenModuleRead"
export_function "_BinaryenModuleInterpret"
export_function "_BinaryenModuleAddDebugInfoFileName"
export_function "_BinaryenModuleGetDebugInfoFileName"

# 'FunctionType' operations
export_function "_BinaryenFunctionTypeGetName"
Expand All @@ -560,6 +563,7 @@ export_function "_BinaryenFunctionGetVar"
export_function "_BinaryenFunctionGetBody"
export_function "_BinaryenFunctionOptimize"
export_function "_BinaryenFunctionRunPasses"
export_function "_BinaryenFunctionSetDebugLocation"

# 'Import' operations
export_function "_BinaryenImportGetKind"
Expand Down
71 changes: 65 additions & 6 deletions src/binaryen-c.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2100,19 +2100,43 @@ void BinaryenModuleAutoDrop(BinaryenModuleRef module) {
passRunner.run();
}

size_t BinaryenModuleWrite(BinaryenModuleRef module, char* output, size_t outputSize) {
if (tracing) {
std::cout << " // BinaryenModuleWrite\n";
}

static BinaryenBufferSizes writeModule(BinaryenModuleRef module, char* output, size_t outputSize, const char* sourceMapUrl, char* sourceMap, size_t sourceMapSize) {
Module* wasm = (Module*)module;
BufferWithRandomAccess buffer(false);
WasmBinaryWriter writer(wasm, buffer, false);
writer.setNamesSection(globalPassOptions.debugInfo);
std::ostringstream os;
if (sourceMapUrl) {
writer.setSourceMap(&os, sourceMapUrl);
}
writer.write();
size_t bytes = std::min(buffer.size(), outputSize);
std::copy_n(buffer.begin(), bytes, output);
return bytes;
size_t sourceMapBytes = 0;
if (sourceMapUrl) {
auto str = os.str();
sourceMapBytes = std::min(str.length(), sourceMapSize);
std::copy_n(str.c_str(), sourceMapBytes, sourceMap);
}
return { bytes, sourceMapBytes };
}

size_t BinaryenModuleWrite(BinaryenModuleRef module, char* output, size_t outputSize) {
if (tracing) {
std::cout << " // BinaryenModuleWrite\n";
}

return writeModule((Module*)module, output, outputSize, nullptr, nullptr, 0).outputBytes;
}

BinaryenBufferSizes BinaryenModuleWriteWithSourceMap(BinaryenModuleRef module, const char* url, char* output, size_t outputSize, char* sourceMap, size_t sourceMapSize) {
if (tracing) {
std::cout << " // BinaryenModuleWriteWithSourceMap\n";
}

assert(url);
assert(sourceMap);
return writeModule((Module*)module, output, outputSize, url, sourceMap, sourceMapSize);
}

BinaryenModuleRef BinaryenModuleRead(char* input, size_t inputSize) {
Expand Down Expand Up @@ -2144,6 +2168,26 @@ void BinaryenModuleInterpret(BinaryenModuleRef module) {
ModuleInstance instance(*wasm, &interface);
}

BinaryenIndex BinaryenModuleAddDebugInfoFileName(BinaryenModuleRef module, const char* filename) {
if (tracing) {
std::cout << " BinaryenModuleAddDebugInfoFileName(the_module, \"" << filename << "\");\n";
}

Module* wasm = (Module*)module;
BinaryenIndex index = wasm->debugInfoFileNames.size();
wasm->debugInfoFileNames.push_back(filename);
return index;
}

const char* BinaryenModuleGetDebugInfoFileName(BinaryenModuleRef module, BinaryenIndex index) {
if (tracing) {
std::cout << " BinaryenModuleGetDebugInfoFileName(the_module, \"" << index << "\");\n";
}

Module* wasm = (Module*)module;
return index < wasm->debugInfoFileNames.size() ? wasm->debugInfoFileNames.at(index).c_str() : nullptr;
}

//
// ======== FunctionType Operations ========
//
Expand Down Expand Up @@ -2275,6 +2319,21 @@ void BinaryenFunctionRunPasses(BinaryenFunctionRef func, BinaryenModuleRef modul
}
passRunner.runOnFunction((Function*)func);
}
void BinaryenFunctionSetDebugLocation(BinaryenFunctionRef func, BinaryenExpressionRef expr, BinaryenIndex fileIndex, BinaryenIndex lineNumber, BinaryenIndex columnNumber) {
if (tracing) {
std::cout << " BinaryenFunctionSetDebugLocation(functions[" << functions[func] << "], expressions[" << expressions[expr] << "], " << fileIndex << ", " << lineNumber << ", " << columnNumber << ");\n";
}

auto* fn = (Function*)func;
auto* ex = (Expression*)expr;

Function::DebugLocation loc;
loc.fileIndex = fileIndex;
loc.lineNumber = lineNumber;
loc.columnNumber = columnNumber;

fn->debugLocations[ex] = loc;
}

//
// =========== Import operations ===========
Expand Down
20 changes: 20 additions & 0 deletions src/binaryen-c.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,16 @@ void BinaryenModuleAutoDrop(BinaryenModuleRef module);
// @return how many bytes were written. This will be less than or equal to outputSize
size_t BinaryenModuleWrite(BinaryenModuleRef module, char* output, size_t outputSize);

typedef struct BinaryenBufferSizes {
size_t outputBytes;
size_t sourceMapBytes;
} BinaryenBufferSizes;

// Serialize a module into binary form including its source map. Uses the currently set
// global debugInfo option.
// @returns how many bytes were written. This will be less than or equal to outputSize
BinaryenBufferSizes BinaryenModuleWriteWithSourceMap(BinaryenModuleRef module, const char* url, char* output, size_t outputSize, char* sourceMap, size_t sourceMapSize);
Copy link
Member

@kripken kripken Jan 29, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

returning a struct has some odd ABI issues in C that I'd rather avoid. instead, how about returning the output size as in BinaryenModuleWrite, and adding a parameter with a pointer to write the source map bytes into?

Copy link
Contributor Author

@dcodeIO dcodeIO Jan 29, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's actually exactly how output works (seen from the JS-side), just with two such buffers. What's done there is that the user pre-allocates a buffer or two on the JS side and provides their pointers including their maximum length. The function then returns the number of bytes written so that the JS side can trim the buffer, and in this specific case it has to return two (hence the struct).

There is a FIXME on the JS side because these buffers are currently capped to 1MB. I guess if we solve this one, the API might change anyway. From JS perspective, it would be ideal not to pre-allocate anything and just receive a properly sized buffer incl. its length from the function. With the ABI stuff, that'd look like

struct {
  void* buffer;
  size_t bufferSize;
}

or something, and I don't currently have a better idea (can't use passing by reference from JS for example, except there's also an ABI for that?). Any idea?

Copy link
Contributor Author

@dcodeIO dcodeIO Jan 29, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe, if we could write just the source map without having to also write the binary, we could split that, making it two functions that return just size_t. Might require some changes to the binary writer, though, as source map generation is currently piggybacked on top of writing a binary (what about a specific sourcemap writer?). Wouldn't fix the hard-wired limit, though.

Copy link
Member

@kripken kripken Jan 29, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, an API variant that mallocs the buffer internally and returns it, and requires the caller to free it later, would be a good way to solve the 1MB limit. Alternatively the zlib API has a method that can upper bound the size needed for a buffer. That's less easy for us, but maybe there's a way to that, which would be better (avoid malloc, the user might have another preferred allocation method).


// Deserialize a module from binary form.
BinaryenModuleRef BinaryenModuleRead(char* input, size_t inputSize);

Expand All @@ -705,6 +715,13 @@ BinaryenModuleRef BinaryenModuleRead(char* input, size_t inputSize);
// and then destroying the instance.
void BinaryenModuleInterpret(BinaryenModuleRef module);

// Adds a debug info file name to the module and returns its index.
BinaryenIndex BinaryenModuleAddDebugInfoFileName(BinaryenModuleRef module, const char* filename);

// Gets the name of the debug info file at the specified index. Returns `NULL` if it
// does not exist.
const char* BinaryenModuleGetDebugInfoFileName(BinaryenModuleRef module, BinaryenIndex index);

//
// ======== FunctionType Operations ========
//
Expand Down Expand Up @@ -747,6 +764,9 @@ void BinaryenFunctionOptimize(BinaryenFunctionRef func, BinaryenModuleRef module
// optimize and shrink level.
void BinaryenFunctionRunPasses(BinaryenFunctionRef func, BinaryenModuleRef module, const char **passes, BinaryenIndex numPasses);

// Sets the debug location of the specified `Expression` within the specified `Function`.
void BinaryenFunctionSetDebugLocation(BinaryenFunctionRef func, BinaryenExpressionRef expr, BinaryenIndex fileIndex, BinaryenIndex lineNumber, BinaryenIndex columnNumber);

//
// ========== Import Operations ==========
//
Expand Down
64 changes: 45 additions & 19 deletions src/js/binaryen.js-post.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,8 +286,8 @@ Module['Module'] = function(module) {
// need to make their own Literals, as the C API handles them by value,
// which means we would leak them. Instead, this is the only API that
// accepts Literals, so fuse it with Literal creation
var literal = _malloc(16); // a single literal in memory. the LLVM C ABI
// makes us pass pointers to this.
var temp = _malloc(16); // a single literal in memory. the LLVM C ABI
// makes us pass pointers to this.

this['i32'] = {
'load': function(offset, align, ptr) {
Expand Down Expand Up @@ -315,8 +315,8 @@ Module['Module'] = function(module) {
return Module['_BinaryenStore'](module, 2, offset, align, ptr, value, Module['i32']);
},
'const': function(x) {
Module['_BinaryenLiteralInt32'](literal, x);
return Module['_BinaryenConst'](module, literal);
Module['_BinaryenLiteralInt32'](temp, x);
return Module['_BinaryenConst'](module, temp);
},
'clz': function(value) {
return Module['_BinaryenUnary'](module, Module['ClzInt32'], value);
Expand Down Expand Up @@ -556,8 +556,8 @@ Module['Module'] = function(module) {
return Module['_BinaryenStore'](module, 4, offset, align, ptr, value, Module['i64']);
},
'const': function(x, y) {
Module['_BinaryenLiteralInt64'](literal, x, y);
return Module['_BinaryenConst'](module, literal);
Module['_BinaryenLiteralInt64'](temp, x, y);
return Module['_BinaryenConst'](module, temp);
},
'clz': function(value) {
return Module['_BinaryenUnary'](module, Module['ClzInt64'], value);
Expand Down Expand Up @@ -802,12 +802,12 @@ Module['Module'] = function(module) {
return Module['_BinaryenStore'](module, 4, offset, align, ptr, value, Module['f32']);
},
'const': function(x) {
Module['_BinaryenLiteralFloat32'](literal, x);
return Module['_BinaryenConst'](module, literal);
Module['_BinaryenLiteralFloat32'](temp, x);
return Module['_BinaryenConst'](module, temp);
},
'const_bits': function(x) {
Module['_BinaryenLiteralFloat32Bits'](literal, x);
return Module['_BinaryenConst'](module, literal);
Module['_BinaryenLiteralFloat32Bits'](temp, x);
return Module['_BinaryenConst'](module, temp);
},
'neg': function(value) {
return Module['_BinaryenUnary'](module, Module['NegFloat32'], value);
Expand Down Expand Up @@ -901,12 +901,12 @@ Module['Module'] = function(module) {
return Module['_BinaryenStore'](module, 8, offset, align, ptr, value, Module['f64']);
},
'const': function(x) {
Module['_BinaryenLiteralFloat64'](literal, x);
return Module['_BinaryenConst'](module, literal);
Module['_BinaryenLiteralFloat64'](temp, x);
return Module['_BinaryenConst'](module, temp);
},
'const_bits': function(x, y) {
Module['_BinaryenLiteralFloat64Bits'](literal, x, y);
return Module['_BinaryenConst'](module, literal);
Module['_BinaryenLiteralFloat64Bits'](temp, x, y);
return Module['_BinaryenConst'](module, temp);
},
'neg': function(value) {
return Module['_BinaryenUnary'](module, Module['NegFloat64'], value);
Expand Down Expand Up @@ -1185,16 +1185,42 @@ Module['Module'] = function(module) {
Module['_BinaryenModuleDispose'](module);
};
var MAX = 1024*1024; // TODO: fix this hard-wired limit
var writeBuffer = null;
this['emitBinary'] = function() {
if (!writeBuffer) writeBuffer = _malloc(MAX);
var bytes = Module['_BinaryenModuleWrite'](module, writeBuffer, MAX);
var outputBuffer = null;
var sourceMapBuffer = null;
this['emitBinary'] = function(sourceMapUrl) {
if (!outputBuffer) outputBuffer = _malloc(MAX);
var bytes = Module['_BinaryenModuleWrite'](module, outputBuffer, MAX);
assert(bytes < MAX, 'FIXME: hardcoded limit on module size'); // we should not use the whole buffer
return new Uint8Array(HEAPU8.subarray(writeBuffer, writeBuffer + bytes));
return new Uint8Array(HEAPU8.subarray(outputBuffer, outputBuffer + bytes));
};
this['emitBinaryWithSourceMap'] = function(sourceMapUrl) {
if (!outputBuffer) outputBuffer = _malloc(MAX);
if (!sourceMapBuffer) sourceMapBuffer = _malloc(MAX);
return preserveStack(function() {
Module['_BinaryenModuleWriteWithSourceMap'](temp, module, strToStack(sourceMapUrl), outputBuffer, MAX, sourceMapBuffer, MAX);
Copy link
Member

@kripken kripken Jan 29, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(looks like this depends on the ABI issues mentioned above, it assumes the output is pointed to by an extra param at the beginning? I'm actually surprised it's not at the end.)

Copy link
Contributor Author

@dcodeIO dcodeIO Jan 29, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just reused what's done for BinaryenLiteral here, which is also a returned struct, above, and reused its temporary buffer. I actually don't know how exactly this works :)

Copy link
Member

@kripken kripken Jan 29, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, I see. Yeah, we already do depend on those ABI details I guess... no big deal to add this, then.

var outputBytes = HEAPU32[temp >>> 2];
var sourceMapBytes = HEAPU32[(temp + 1) >>> 2];
assert(outputBytes < MAX && sourceMapBytes < MAX, 'FIXME: hardcoded limit on module size'); // see above
return {
'binary': new Uint8Array(HEAPU8.subarray(outputBuffer, outputBuffer + outputBytes)),
'sourceMap': Pointer_stringify(sourceMapBuffer)
};
});
};
this['interpret'] = function() {
return Module['_BinaryenModuleInterpret'](module);
};
this['addDebugInfoFileName'] = function(filename) {
return preserveStack(function() {
return Module['_BinaryenModuleAddDebugInfoFileName'](module, strToStack(filename));
});
};
this['getDebugInfoFileName'] = function(index) {
return Pointer_stringify(Module['_BinaryenModuleGetDebugInfoFileName'](module, index));
};
this['setDebugLocation'] = function(func, expr, fileIndex, lineNumber, columnNumber) {
return Module['_BinaryenFunctionSetDebugLocation'](func, expr, fileIndex, lineNumber, columnNumber);
};
};

// 'Relooper' interface
Expand Down
39 changes: 39 additions & 0 deletions test/binaryen.js/sourcemap.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
var module = new Binaryen.Module();

var signature = module.addFunctionType("i", Binaryen.i32, []);

var fileIndex = module.addDebugInfoFileName("module.c");

console.log(module.getDebugInfoFileName(fileIndex));
console.log();

var expr = module.i32.const(1);
var body = module.block("", [
expr
], Binaryen.i32);

var func = module.addFunction("main", signature, [], body);

module.setDebugLocation(func, expr, fileIndex, 1, 2);
module.setDebugLocation(func, body, fileIndex, 0, 3);

var output = module.emitBinaryWithSourceMap("module.wasm.map");

function dumpBinary(buffer) {
var hex = [], o, b, h;
for (var i = 0; i < buffer.length; ++i) {
o = i.toString(16);
while (o.length < 3) o = "0" + o;
if ((i & 15) === 0) hex.push((i ? "\n" : "") + o + ":");
if ((b = buffer[i]) >= 0x21 && b <= 0x7e)
h = String.fromCharCode(b) + ' ';
else if ((h = b.toString(16)).length < 2)
h = "0" + h;
hex.push(h);
}
console.log(hex.join(" "));
}

dumpBinary(output.binary);
console.log();
console.log(output.sourceMap);
8 changes: 8 additions & 0 deletions test/binaryen.js/sourcemap.js.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
module.c

000: 00 a s m 01 00 00 00 01 05 01 ` 00 01 7f 03
010: 02 01 00 0a 06 01 04 00 A 01 0b 00 ! 10 s o
020: u r c e M a p p i n g U R L 0f m
030: o d u l e . w a s m . m a p

{"version":3,"sources":["module.c"],"names":[],"mappings":"gCAAE"}