From d1540ad5788ece8f8bd03a319843d4999c17a175 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Thu, 9 May 2024 22:50:08 -0700 Subject: [PATCH 01/18] initial sourcemap loading not complete yet, needs a better comment parser --- src/bun.js/bindings/ZigSourceProvider.cpp | 22 ++- src/bun.js/bindings/ZigSourceProvider.h | 15 +- src/bun.js/bindings/exports.zig | 1 + src/bun.js/bindings/headers-handwritten.h | 1 + src/bun.js/javascript.zig | 117 ++++++++++-- src/bun.js/module_loader.zig | 3 +- src/js_printer.zig | 2 +- src/sourcemap/sourcemap.zig | 212 ++++++++++++++-------- 8 files changed, 267 insertions(+), 106 deletions(-) diff --git a/src/bun.js/bindings/ZigSourceProvider.cpp b/src/bun.js/bindings/ZigSourceProvider.cpp index b5a6eae7728f4..6dd9a87310cc0 100644 --- a/src/bun.js/bindings/ZigSourceProvider.cpp +++ b/src/bun.js/bindings/ZigSourceProvider.cpp @@ -66,10 +66,14 @@ JSC::SourceID sourceIDForSourceURL(const WTF::String& sourceURL) } extern "C" bool BunTest__shouldGenerateCodeCoverage(BunString sourceURL); - -Ref SourceProvider::create(Zig::GlobalObject* globalObject, ResolvedSource& resolvedSource, JSC::SourceProviderSourceType sourceType, bool isBuiltin) -{ - +extern "C" void Bun__addSourceProviderSourceMap(void* bun_vm, void* opaque_source_provider, BunString* specifier); + +Ref SourceProvider::create( + Zig::GlobalObject* globalObject, + ResolvedSource& resolvedSource, + JSC::SourceProviderSourceType sourceType, + bool isBuiltin +) { auto string = resolvedSource.source_code.toWTFString(BunString::ZeroCopy); auto sourceURLString = resolvedSource.source_url.toWTFString(BunString::ZeroCopy); @@ -99,6 +103,10 @@ Ref SourceProvider::create(Zig::GlobalObject* globalObject, Reso ByteRangeMapping__generate(Bun::toString(provider->sourceURL()), Bun::toString(provider->source().toStringWithoutCopying()), provider->asID()); } + if (resolvedSource.already_bundled) { + Bun__addSourceProviderSourceMap(globalObject->bunVM(), provider.ptr(), &resolvedSource.source_url); + } + return provider; } @@ -229,4 +237,10 @@ int SourceProvider::readCache(JSC::VM& vm, const JSC::SourceCode& sourceCode) // return 0; // } } + +extern "C" BunString ZigSourceProvider__getSourceSlice(SourceProvider* provider) { + // TODO(@paperdave): does this clone the data? i sure hope not. + return Bun::toStringRef(provider->source().toString()); +} + }; // namespace Zig diff --git a/src/bun.js/bindings/ZigSourceProvider.h b/src/bun.js/bindings/ZigSourceProvider.h index 6c04c14b26dc6..186846c4d6fc9 100644 --- a/src/bun.js/bindings/ZigSourceProvider.h +++ b/src/bun.js/bindings/ZigSourceProvider.h @@ -1,5 +1,7 @@ #include "headers.h" #include "root.h" +#include <_types/_uint64_t.h> +#include #pragma once @@ -36,7 +38,11 @@ class SourceProvider final : public JSC::SourceProvider { using SourceOrigin = JSC::SourceOrigin; public: - static Ref create(Zig::GlobalObject*, ResolvedSource& resolvedSource, JSC::SourceProviderSourceType sourceType = JSC::SourceProviderSourceType::Module, bool isBuiltIn = false); + static Ref create( + Zig::GlobalObject*, + ResolvedSource& resolvedSource, + JSC::SourceProviderSourceType sourceType = JSC::SourceProviderSourceType::Module, + bool isBuiltIn = false); ~SourceProvider(); unsigned hash() const override; StringView source() const override { return StringView(m_source.get()); } @@ -67,17 +73,12 @@ class SourceProvider final : public JSC::SourceProvider { : Base(sourceOrigin, WTFMove(sourceURL), String(), taintedness, startPosition, sourceType) , m_source(sourceImpl) { - m_resolvedSource = resolvedSource; } RefPtr m_cachedBytecode; Ref m_source; - bool did_free_source_code = false; - Zig::GlobalObject* m_globalObjectForSourceProviderMap; unsigned m_hash = 0; - - // JSC::SourceCodeKey key; }; -} // namespace Zig \ No newline at end of file +} // namespace Zig diff --git a/src/bun.js/bindings/exports.zig b/src/bun.js/bindings/exports.zig index 11a1a42e97844..31375ae373e96 100644 --- a/src/bun.js/bindings/exports.zig +++ b/src/bun.js/bindings/exports.zig @@ -218,6 +218,7 @@ pub const ResolvedSource = extern struct { /// This is for source_code source_code_needs_deref: bool = true, + already_bundled: bool = false, pub const Tag = @import("ResolvedSourceTag").ResolvedSourceTag; }; diff --git a/src/bun.js/bindings/headers-handwritten.h b/src/bun.js/bindings/headers-handwritten.h index c68a1d3dab752..1caa74af617ee 100644 --- a/src/bun.js/bindings/headers-handwritten.h +++ b/src/bun.js/bindings/headers-handwritten.h @@ -99,6 +99,7 @@ typedef struct ResolvedSource { JSC::EncodedJSValue jsvalue_for_export; uint32_t tag; bool needsDeref; + bool already_bundled; } ResolvedSource; static const uint32_t ResolvedSourceTagPackageJSONTypeModule = 1; typedef union ErrorableResolvedSourceResult { diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 36b028ab35af1..f498469c7bca2 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -118,9 +118,13 @@ const MappingList = SourceMap.Mapping.List; const uv = bun.windows.libuv; pub const SavedSourceMap = struct { + /// This is a pointer to the map located on the VirtualMachine struct + map: *HashTable, + mutex: bun.Lock = bun.Lock.init(), + pub const vlq_offset = 24; - // For bun.js, we store the number of mappings and how many bytes the final list is at the beginning of the array + // For the runtime, we store the number of mappings and how many bytes the final list is at the beginning of the array // The first 8 bytes are the length of the array // The second 8 bytes are the number of mappings pub const SavedMappings = struct { @@ -172,16 +176,70 @@ pub const SavedSourceMap = struct { } }; + /// ParsedSourceMap is the canonical form for sourcemaps, + /// + /// but `SavedMappings` and `SourceProviderMap` are much cheaper to construct. + /// In `fn get`, this value gets converted to ParsedSourceMap always pub const Value = TaggedPointerUnion(.{ ParsedSourceMap, SavedMappings, + SourceProviderMap, }); - pub const HashTable = std.HashMap(u64, *anyopaque, IdentityContext(u64), 80); - /// This is a pointer to the map located on the VirtualMachine struct - map: *HashTable, + /// This is a pointer to a ZigSourceProvider that may or may not have a `//# sourceMappingURL` comment + /// when we want to lookup this data, we will then resolve it to a ParsedSourceMap if it does. + /// + /// This is used for files that were pre-bundled with `bun build --target=bun --sourcemap` + pub const SourceProviderMap = opaque { + extern fn ZigSourceProvider__getSourceSlice(*SourceProviderMap) bun.String; + + pub fn toParsedSourceMap(provider: *SourceProviderMap) ?*ParsedSourceMap { + const bun_str = ZigSourceProvider__getSourceSlice(provider); + defer bun_str.deref(); + bun.assert(bun_str.tag == .WTFStringImpl); + const wtf = bun_str.value.WTFStringImpl; + + // TODO: do not use toUTF8() + const utf8 = wtf.toUTF8(bun.default_allocator); + defer utf8.deinit(); + const bytes = utf8.slice(); + + // TODO: use smarter technique + if (std.mem.indexOf(u8, bytes, "//# sourceMappingURL=")) |index| { + const end = std.mem.indexOfAnyPos(u8, bytes, index + "//# sourceMappingURL=".len, " \n\r") orelse bytes.len; + const url = bytes[index + "//# sourceMappingURL=".len .. end]; + + var sfb = std.heap.stackFallback(32768, bun.default_allocator); + var arena = bun.ArenaAllocator.init(sfb.get()); + defer arena.deinit(); + + switch (bun.sourcemap.parseUrl(bun.default_allocator, arena.allocator(), url)) { + .fail => { + return null; + }, + .success => |parsed| { + const ptr = bun.default_allocator.create(ParsedSourceMap) catch bun.outOfMemory(); + ptr.* = parsed; + return ptr; + }, + } + } - mutex: bun.Lock = bun.Lock.init(), + return null; + } + + pub fn deinit(provider: *SourceProviderMap) void { + _ = provider; + @panic("TODO"); + } + }; + + pub fn putZigSourceProvider(this: *SavedSourceMap, opaque_source_provider: *anyopaque, path: []const u8) void { + const source_provider: *SourceProviderMap = @ptrCast(opaque_source_provider); + this.putValue(path, Value.init(source_provider)) catch bun.outOfMemory(); + } + + pub const HashTable = std.HashMap(u64, *anyopaque, IdentityContext(u64), 80); pub fn onSourceMapChunk(this: *SavedSourceMap, chunk: SourceMap.Chunk, source: logger.Source) anyerror!void { try this.putMappings(source, chunk.buffer); @@ -211,26 +269,31 @@ pub const SavedSourceMap = struct { } pub fn putMappings(this: *SavedSourceMap, source: logger.Source, mappings: MutableString) !void { + try this.putValue(source.path.text, Value.init(bun.cast(*SavedMappings, mappings.list.items.ptr))); + } + + fn putValue(this: *SavedSourceMap, path: []const u8, value: Value) !void { this.mutex.lock(); defer this.mutex.unlock(); - const entry = try this.map.getOrPut(bun.hash(source.path.text)); + const entry = try this.map.getOrPut(bun.hash(path)); if (entry.found_existing) { - var value = Value.from(entry.value_ptr.*); - if (value.get(ParsedSourceMap)) |source_map_| { - var source_map: *ParsedSourceMap = source_map_; + var old_value = Value.from(entry.value_ptr.*); + if (old_value.get(ParsedSourceMap)) |parsed_source_map| { + var source_map: *ParsedSourceMap = parsed_source_map; source_map.deinit(default_allocator); - } else if (value.get(SavedMappings)) |saved_mappings| { + } else if (old_value.get(SavedMappings)) |saved_mappings| { var saved = SavedMappings{ .data = @as([*]u8, @ptrCast(saved_mappings)) }; - saved.deinit(); + } else if (old_value.get(SourceProviderMap)) |provider| { + provider.deinit(); } } - - entry.value_ptr.* = Value.init(bun.cast(*SavedMappings, mappings.list.items.ptr)).ptr(); + entry.value_ptr.* = value.ptr(); } pub fn get(this: *SavedSourceMap, path: string) ?ParsedSourceMap { - const mapping = this.map.getEntry(bun.hash(path)) orelse return null; + const hash = bun.hash(path); + const mapping = this.map.getEntry(hash) orelse return null; switch (Value.from(mapping.value_ptr.*).tag()) { Value.Tag.ParsedSourceMap => { return Value.from(mapping.value_ptr.*).as(ParsedSourceMap).*; @@ -246,7 +309,23 @@ pub const SavedSourceMap = struct { mapping.value_ptr.* = Value.init(result).ptr(); return result.*; }, - else => return null, + Value.Tag.SourceProviderMap => { + var ptr = Value.from(mapping.value_ptr.*).as(SourceProviderMap); + if (ptr.toParsedSourceMap()) |result| { + mapping.value_ptr.* = Value.init(result).ptr(); + return result.*; + } else { + // does not have a valid source map + _ = this.map.remove(hash); + return null; + } + }, + else => { + if (Environment.allow_assert) { + @panic("Corrupt pointer tag"); + } + return null; + }, } } @@ -3825,4 +3904,12 @@ pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime }; } +export fn Bun__addSourceProviderSourceMap(bun_vm: *anyopaque, opaque_source_provider: *anyopaque, specifier: *bun.String) void { + const vm: *VirtualMachine = @alignCast(@ptrCast(bun_vm)); + var sfb = std.heap.stackFallback(4096, bun.default_allocator); + const slice = specifier.toUTF8(sfb.get()); + defer slice.deinit(); + vm.source_mappings.putZigSourceProvider(opaque_source_provider, slice.slice()); +} + pub export var isBunTest: bool = false; diff --git a/src/bun.js/module_loader.zig b/src/bun.js/module_loader.zig index 1e08e4b6e4dc4..7d03dd286fe53 100644 --- a/src/bun.js/module_loader.zig +++ b/src/bun.js/module_loader.zig @@ -592,6 +592,7 @@ pub const RuntimeTranspilerStore = struct { .source_code = bun.String.createLatin1(parse_result.source.contents), .specifier = duped, .source_url = duped.createIfDifferent(path.text), + .already_bundled = true, .hash = 0, }; this.resolved_source.source_code.ensureHash(); @@ -1796,7 +1797,7 @@ pub const ModuleLoader = struct { .source_code = bun.String.createLatin1(parse_result.source.contents), .specifier = input_specifier, .source_url = input_specifier.createIfDifferent(path.text), - + .already_bundled = true, .hash = 0, }; } diff --git a/src/js_printer.zig b/src/js_printer.zig index 4ba133c1a8918..42fffe91d8cf5 100644 --- a/src/js_printer.zig +++ b/src/js_printer.zig @@ -5865,7 +5865,7 @@ pub fn getSourceMapBuilder( return .{ .source_map = SourceMap.Chunk.Builder.SourceMapper.init( opts.allocator, - is_bun_platform, + is_bun_platform and generate_source_map == .lazy, ), .cover_lines_without_mappings = true, .approximate_input_line_count = tree.approximate_newline_count, diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig index be4eca94e6a49..8613ede77fd27 100644 --- a/src/sourcemap/sourcemap.zig +++ b/src/sourcemap/sourcemap.zig @@ -39,78 +39,95 @@ sources_content: []string, mapping: Mapping.List = .{}, allocator: std.mem.Allocator, -pub fn parse( - allocator: std.mem.Allocator, - json_source: *const Logger.Source, - log: *Logger.Log, -) !SourceMap { - var json = try bun.JSON.ParseJSONUTF8(json_source, log, allocator); - var mappings = bun.sourcemap.Mapping.List{}; - - if (json.get("version")) |version| { - if (version.data != .e_number or version.data.e_number.value != 3.0) { - return error.@"Unsupported sourcemap version"; - } - } - - if (json.get("mappings")) |mappings_str| { - if (mappings_str.data != .e_string) { - return error.@"Invalid sourcemap mappings"; - } - - var parsed = bun.sourcemap.Mapping.parse(allocator, try mappings_str.data.e_string.toUTF8(allocator), null, std.math.maxInt(i32)); - if (parsed == .fail) { - try log.addMsg(bun.logger.Msg{ - .data = parsed.fail.toData("sourcemap.json"), - .kind = .err, - }); - return error.@"Failed to parse sourcemap mappings"; - } - - mappings = parsed.success; - } - - var sources = std.ArrayList(bun.string).init(allocator); - var sources_content = std.ArrayList(string).init(allocator); +/// `source` must be in UTF-8 and can be freed after this call. +/// The mappings are owned by the `alloc` allocator. +/// Temporary allocations are made to the `arena` allocator, which +/// should be an arena allocator (those allocations are not freed). +pub fn parseUrl( + alloc: std.mem.Allocator, + arena: std.mem.Allocator, + source: []const u8, +) Mapping.ParseResult { + const data_prefix = "data:application/json"; + if (bun.strings.hasPrefixComptime(source, data_prefix) and source.len > (data_prefix.len + 1)) try_data_url: { + const json_bytes = switch (source[data_prefix.len]) { + ';' => json_bytes: { + const encoding = bun.sliceTo(source[data_prefix.len + 1 ..], ','); + if (!bun.strings.eqlComptime(encoding, "base64")) break :try_data_url; + const base64_data = source[data_prefix.len + ";base64,".len ..]; + + const len = bun.base64.decodeLen(base64_data); + const bytes = arena.alloc(u8, len) catch bun.outOfMemory(); + const result = bun.base64.decode(bytes, base64_data); + if (result.fail) return .{ .fail = .{ + .msg = "Invalid Base64", + .err = error.InvalidBase64, + .value = 0, + .loc = .{ .start = 0 }, + } }; + break :json_bytes bytes[0..result.written]; + }, + ',' => source[data_prefix.len + 1 ..], + else => break :try_data_url, + }; - if (json.get("sourcesContent")) |mappings_str| { - if (mappings_str.data != .e_array) { - return error.@"Invalid sourcemap sources"; - } + const json_src = bun.logger.Source.initPathString("url.json", json_bytes); + var log = bun.logger.Log.init(arena); + defer log.deinit(); + + var json = bun.JSON.ParseJSONUTF8(&json_src, &log, arena) catch { + return .{ .fail = .{ + .msg = "Invalid JSON", + .err = error.InvalidJSON, + .value = 0, + .loc = .{ .start = 0 }, + } }; + }; - try sources_content.ensureTotalCapacityPrecise(mappings_str.data.e_array.items.len); - for (mappings_str.data.e_array.items.slice()) |source| { - if (source.data != .e_string) { - return error.@"Invalid sourcemap source"; + if (json.get("version")) |version| { + if (version.data != .e_number or version.data.e_number.value != 3.0) { + return .{ .fail = .{ + .msg = "Unsupported source map version", + .err = error.UnsupportedVersion, + .value = 0, + .loc = .{ .start = 0 }, + } }; } - - try source.data.e_string.toUTF8(allocator); - sources_content.appendAssumeCapacity(source.data.e_string.slice()); - } - } - - if (json.get("sources")) |mappings_str| { - if (mappings_str.data != .e_array) { - return error.@"Invalid sourcemap sources"; } - try sources.ensureTotalCapacityPrecise(mappings_str.data.e_array.items.len); - for (mappings_str.data.e_array.items.slice()) |source| { - if (source.data != .e_string) { - return error.@"Invalid sourcemap source"; - } + const mappings_str = json.get("mappings") orelse { + return .{ .fail = .{ + .msg = "Missing source mappings", + .err = error.UnsupportedVersion, + .value = 0, + .loc = .{ .start = 0 }, + } }; + }; - try source.data.e_string.toUTF8(allocator); - sources.appendAssumeCapacity(source.data.e_string.slice()); + if (mappings_str.data != .e_string) { + return .{ .fail = .{ + .msg = "Missing source mappings", + .err = error.UnsupportedVersion, + .value = 0, + .loc = .{ .start = 0 }, + } }; } + + return Mapping.parse( + alloc, + mappings_str.data.e_string.slice(arena), + null, + std.math.maxInt(i32), + std.math.maxInt(i32), + ); } - return SourceMap{ - .mapping = mappings, - .allocator = allocator, - .sources_content = sources_content.items, - .sources = sources.items, - }; + return .{ .fail = .{ + .msg = "Unsupported source map type", + .err = error.UnsupportedFormat, + .value = 0, + .loc = .{ .start = 0 }, + } }; } pub const Mapping = struct { @@ -559,14 +576,14 @@ pub const SourceMapPieces = struct { const potential_start_of_run = current; - current = decodeVLQ(mappings, current).start; - current = decodeVLQ(mappings, current).start; - current = decodeVLQ(mappings, current).start; + current = decodeVLQAssumeValid(mappings, current).start; + current = decodeVLQAssumeValid(mappings, current).start; + current = decodeVLQAssumeValid(mappings, current).start; if (current < mappings.len) { const c = mappings[current]; if (c != ',' and c != ';') { - current = decodeVLQ(mappings, current).start; + current = decodeVLQAssumeValid(mappings, current).start; } } @@ -594,7 +611,8 @@ pub const SourceMapPieces = struct { assert(shift.before.lines == shift.after.lines); const shift_column_delta = shift.after.columns - shift.before.columns; - const encode = encodeVLQ(decode_result.value + shift_column_delta - prev_shift_column_delta); + const vlq_value = decode_result.value + shift_column_delta - prev_shift_column_delta; + const encode = encodeVLQ(vlq_value); j.push(encode.bytes[0..encode.len]); prev_shift_column_delta = shift_column_delta; @@ -637,14 +655,16 @@ pub fn appendSourceMapChunk(j: *Joiner, allocator: std.mem.Allocator, prev_end_s // Strip off the first mapping from the buffer. The first mapping should be // for the start of the original file (the printer always generates one for // the start of the file). + // + // Bun has a 24-byte header for source map meta-data var i: usize = 0; - const generated_column_ = decodeVLQ(source_map, 0); + const generated_column_ = decodeVLQAssumeValid(source_map, i); i = generated_column_.start; - const source_index_ = decodeVLQ(source_map, i); + const source_index_ = decodeVLQAssumeValid(source_map, i); i = source_index_.start; - const original_line_ = decodeVLQ(source_map, i); + const original_line_ = decodeVLQAssumeValid(source_map, i); i = original_line_.start; - const original_column_ = decodeVLQ(source_map, i); + const original_column_ = decodeVLQAssumeValid(source_map, i); i = original_column_.start; source_map = source_map[i..]; @@ -658,7 +678,12 @@ pub fn appendSourceMapChunk(j: *Joiner, allocator: std.mem.Allocator, prev_end_s start_state.original_column += original_column_.value; j.append( - appendMappingToBuffer(MutableString.initEmpty(allocator), j.lastByte(), prev_end_state, start_state).list.items, + appendMappingToBuffer( + MutableString.initEmpty(allocator), + j.lastByte(), + prev_end_state, + start_state, + ).list.items, 0, allocator, ); @@ -694,9 +719,7 @@ pub const VLQ = struct { } }; -pub fn encodeVLQWithLookupTable( - value: i32, -) VLQ { +pub fn encodeVLQWithLookupTable(value: i32) VLQ { return if (value >= 0 and value <= 255) vlq_lookup_table[@as(usize, @intCast(value))] else @@ -797,6 +820,39 @@ pub fn decodeVLQ(encoded: []const u8, start: usize) VLQResult { return VLQResult{ .start = start + encoded_.len, .value = 0 }; } +pub fn decodeVLQAssumeValid(encoded: []const u8, start: usize) VLQResult { + var shift: u8 = 0; + var vlq: u32 = 0; + + // hint to the compiler what the maximum value is + const encoded_ = encoded[start..][0..@min(encoded.len - start, comptime (vlq_max_in_bytes + 1))]; + + // inlining helps for the 1 or 2 byte case, hurts a little for larger + comptime var i: usize = 0; + inline while (i < vlq_max_in_bytes + 1) : (i += 1) { + bun.assert(encoded_[i] < std.math.maxInt(u7)); // invalid base64 character + const index = @as(u32, base64_lut[@as(u7, @truncate(encoded_[i]))]); + bun.assert(index != std.math.maxInt(u7)); // invalid base64 character + + // decode a byte + vlq |= (index & 31) << @as(u5, @truncate(shift)); + shift += 5; + + // Stop if there's no continuation bit + if ((index & 32) == 0) { + return VLQResult{ + .start = start + comptime (i + 1), + .value = if ((vlq & 1) == 0) + @as(i32, @intCast(vlq >> 1)) + else + -@as(i32, @intCast((vlq >> 1))), + }; + } + } + + return VLQResult{ .start = start + encoded_.len, .value = 0 }; +} + pub const LineOffsetTable = struct { /// The source map specification is very loose and does not specify what /// column numbers actually mean. The popular "source-map" library from Mozilla @@ -1368,9 +1424,9 @@ pub const Chunk = struct { pub fn addSourceMapping(b: *ThisBuilder, loc: Logger.Loc, output: []const u8) void { if ( - // exclude generated code from source + // don't insert mappings for same location twice b.prev_loc.eql(loc) or - // don't insert mappings for same location twice + // exclude generated code from source loc.start == Logger.Loc.Empty.start) return; From 9e295c911f0f4d8afd38cfaffdf5887c538331c5 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Fri, 10 May 2024 23:51:52 -0700 Subject: [PATCH 02/18] more work on this --- src/bun.js/bindings/bindings.zig | 10 + src/bun.js/javascript.zig | 130 ++++----- src/sourcemap/CodeCoverage.zig | 5 +- src/sourcemap/sourcemap.zig | 460 +++++++++++++++++++++++++++---- 4 files changed, 469 insertions(+), 136 deletions(-) diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index 01a23a31bca99..e83a3e16fb779 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -421,6 +421,16 @@ pub const ZigString = extern struct { }; } + /// Creates a `Slice` from bytes without an allocator, aka a string view. + /// This slice is only valid for the lifetime of the given input, and + /// will never be freed by this slice. + pub fn initStatic(input: []const u8) Slice { + return .{ + .ptr = input.ptr, + .len = @as(u32, @truncate(input.len)), + }; + } + pub fn toZigString(this: Slice) ZigString { if (this.isAllocated()) return ZigString.initUTF8(this.ptr[0..this.len]); diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index f498469c7bca2..0d750d029c880 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -114,6 +114,7 @@ export var has_bun_garbage_collector_flag_enabled = false; const SourceMap = @import("../sourcemap/sourcemap.zig"); const ParsedSourceMap = SourceMap.Mapping.ParsedSourceMap; const MappingList = SourceMap.Mapping.List; +const SourceProviderMap = SourceMap.SourceProviderMap; const uv = bun.windows.libuv; @@ -186,54 +187,6 @@ pub const SavedSourceMap = struct { SourceProviderMap, }); - /// This is a pointer to a ZigSourceProvider that may or may not have a `//# sourceMappingURL` comment - /// when we want to lookup this data, we will then resolve it to a ParsedSourceMap if it does. - /// - /// This is used for files that were pre-bundled with `bun build --target=bun --sourcemap` - pub const SourceProviderMap = opaque { - extern fn ZigSourceProvider__getSourceSlice(*SourceProviderMap) bun.String; - - pub fn toParsedSourceMap(provider: *SourceProviderMap) ?*ParsedSourceMap { - const bun_str = ZigSourceProvider__getSourceSlice(provider); - defer bun_str.deref(); - bun.assert(bun_str.tag == .WTFStringImpl); - const wtf = bun_str.value.WTFStringImpl; - - // TODO: do not use toUTF8() - const utf8 = wtf.toUTF8(bun.default_allocator); - defer utf8.deinit(); - const bytes = utf8.slice(); - - // TODO: use smarter technique - if (std.mem.indexOf(u8, bytes, "//# sourceMappingURL=")) |index| { - const end = std.mem.indexOfAnyPos(u8, bytes, index + "//# sourceMappingURL=".len, " \n\r") orelse bytes.len; - const url = bytes[index + "//# sourceMappingURL=".len .. end]; - - var sfb = std.heap.stackFallback(32768, bun.default_allocator); - var arena = bun.ArenaAllocator.init(sfb.get()); - defer arena.deinit(); - - switch (bun.sourcemap.parseUrl(bun.default_allocator, arena.allocator(), url)) { - .fail => { - return null; - }, - .success => |parsed| { - const ptr = bun.default_allocator.create(ParsedSourceMap) catch bun.outOfMemory(); - ptr.* = parsed; - return ptr; - }, - } - } - - return null; - } - - pub fn deinit(provider: *SourceProviderMap) void { - _ = provider; - @panic("TODO"); - } - }; - pub fn putZigSourceProvider(this: *SavedSourceMap, opaque_source_provider: *anyopaque, path: []const u8) void { const source_provider: *SourceProviderMap = @ptrCast(opaque_source_provider); this.putValue(path, Value.init(source_provider)) catch bun.outOfMemory(); @@ -291,12 +244,12 @@ pub const SavedSourceMap = struct { entry.value_ptr.* = value.ptr(); } - pub fn get(this: *SavedSourceMap, path: string) ?ParsedSourceMap { + pub fn get(this: *SavedSourceMap, path: string, contents: SourceMap.SourceContentHandling) ?*ParsedSourceMap { const hash = bun.hash(path); const mapping = this.map.getEntry(hash) orelse return null; switch (Value.from(mapping.value_ptr.*).tag()) { Value.Tag.ParsedSourceMap => { - return Value.from(mapping.value_ptr.*).as(ParsedSourceMap).*; + return Value.from(mapping.value_ptr.*).as(ParsedSourceMap); }, Value.Tag.SavedMappings => { var saved = SavedMappings{ .data = @as([*]u8, @ptrCast(Value.from(mapping.value_ptr.*).as(ParsedSourceMap))) }; @@ -307,13 +260,13 @@ pub const SavedSourceMap = struct { return null; }; mapping.value_ptr.* = Value.init(result).ptr(); - return result.*; + return result; }, Value.Tag.SourceProviderMap => { var ptr = Value.from(mapping.value_ptr.*).as(SourceProviderMap); - if (ptr.toParsedSourceMap()) |result| { + if (ptr.toParsedSourceMap(contents)) |result| { mapping.value_ptr.* = Value.init(result).ptr(); - return result.*; + return result; } else { // does not have a valid source map _ = this.map.remove(hash); @@ -334,12 +287,20 @@ pub const SavedSourceMap = struct { path: []const u8, line: i32, column: i32, - ) ?SourceMap.Mapping { + source_handling: SourceMap.SourceContentHandling, + ) ?SourceMap.Mapping.Lookup { this.mutex.lock(); defer this.mutex.unlock(); - const parsed_mappings = this.get(path) orelse return null; - return SourceMap.Mapping.find(parsed_mappings.mappings, line, column); + const parsed_mapping = this.get(path, source_handling) orelse + return null; + const mapping = SourceMap.Mapping.find(parsed_mapping.mappings, line, column) orelse + return null; + + return .{ + .mapping = mapping, + .source_map = parsed_mapping, + }; } }; const uws = bun.uws; @@ -621,6 +582,7 @@ pub const VirtualMachine = struct { /// only use it through /// source_mappings saved_source_map_table: SavedSourceMap.HashTable = undefined, + source_mappings: SavedSourceMap = undefined, arena: *Arena = undefined, has_loaded: bool = false, @@ -670,8 +632,6 @@ pub const VirtualMachine = struct { ref_strings: JSC.RefString.Map = undefined, ref_strings_mutex: Lock = undefined, - source_mappings: SavedSourceMap = undefined, - active_tasks: usize = 0, rare_data: ?*JSC.RareData = null, @@ -2807,7 +2767,12 @@ pub const VirtualMachine = struct { sourceURL.slice(), @max(frame.position.line, 0), @max(frame.position.column_start, 0), - )) |mapping| { + .no_source_contents, + )) |lookup| { + if (lookup.displaySourceURLIfNeeded()) |source_url| { + frame.source_url = source_url; + } + const mapping = lookup.mapping; frame.position.line = mapping.original.lines; frame.position.column_start = mapping.original.columns; frame.remapped = true; @@ -2896,27 +2861,43 @@ pub const VirtualMachine = struct { var top_source_url = top.source_url.toUTF8(bun.default_allocator); defer top_source_url.deinit(); - const mapping_ = if (top.remapped) - SourceMap.Mapping{ - .generated = .{}, - .original = .{ - .lines = @max(top.position.line, 0), - .columns = @max(top.position.column_start, 0), + const maybe_lookup = if (top.remapped) + SourceMap.Mapping.Lookup{ + .mapping = .{ + .generated = .{}, + .original = .{ + .lines = @max(top.position.line, 0), + .columns = @max(top.position.column_start, 0), + }, + .source_index = 0, }, - .source_index = 0, + // this pointer is not read if `top.remapped` + .source_map = undefined, } else this.source_mappings.resolveMapping( top_source_url.slice(), @max(top.position.line, 0), @max(top.position.column_start, 0), + .source_contents, ); - if (mapping_) |mapping| { - var log = logger.Log.init(default_allocator); - var original_source = fetchWithoutOnLoadPlugins(this, this.global, top.source_url, bun.String.empty, &log, .print_source) catch return; - must_reset_parser_arena_later.* = true; - const code = original_source.source_code.toUTF8(bun.default_allocator); + if (maybe_lookup) |lookup| { + const mapping = lookup.mapping; + + const code = code: { + if (top.remapped or lookup.source_map.external_source_names.len == 0) { + var log = logger.Log.init(default_allocator); + var original_source = fetchWithoutOnLoadPlugins(this, this.global, top.source_url, bun.String.empty, &log, .print_source) catch return; + must_reset_parser_arena_later.* = true; + break :code original_source.source_code.toUTF8(bun.default_allocator); + } else { + break :code ZigString.Slice.initStatic( + lookup.getSourceCode() orelse + @panic("TODO: what if this failed"), + ); + } + }; defer code.deinit(); top.position.line = mapping.original.lines; @@ -2972,7 +2953,12 @@ pub const VirtualMachine = struct { source_url.slice(), @max(frame.position.line, 0), @max(frame.position.column_start, 0), - )) |mapping| { + .no_source_contents, + )) |lookup| { + if (lookup.displaySourceURLIfNeeded()) |src| { + frame.source_url = src; + } + const mapping = lookup.mapping; frame.position.line = mapping.original.lines; frame.remapped = true; frame.position.column_start = mapping.original.columns; diff --git a/src/sourcemap/CodeCoverage.zig b/src/sourcemap/CodeCoverage.zig index 03c2e0363df99..0576ffa65b2a3 100644 --- a/src/sourcemap/CodeCoverage.zig +++ b/src/sourcemap/CodeCoverage.zig @@ -369,9 +369,8 @@ pub const ByteRangeMapping = struct { var executable_lines: Bitset = Bitset{}; var lines_which_have_executed: Bitset = Bitset{}; - const parsed_mappings_ = bun.JSC.VirtualMachine.get().source_mappings.get( - source_url.slice(), - ); + const parsed_mappings_ = bun.JSC.VirtualMachine.get() + .source_mappings.get(source_url.slice(), .no_source_contents); var functions = std.ArrayListUnmanaged(CodeCoverageReport.Block){}; try functions.ensureTotalCapacityPrecise(allocator, function_blocks.len); diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig index 8613ede77fd27..73e5f2e0e7061 100644 --- a/src/sourcemap/sourcemap.zig +++ b/src/sourcemap/sourcemap.zig @@ -39,95 +39,268 @@ sources_content: []string, mapping: Mapping.List = .{}, allocator: std.mem.Allocator, +const ParseUrlResultType = enum { + all, + mappings_only, + sources_only, +}; + +/// Parses an inline source map url like `data:application/json,....` +/// Currently does not handle non-inline source maps. +/// /// `source` must be in UTF-8 and can be freed after this call. /// The mappings are owned by the `alloc` allocator. /// Temporary allocations are made to the `arena` allocator, which -/// should be an arena allocator (those allocations are not freed). +/// should be an arena allocator (caller is assumed to call `deinit`). pub fn parseUrl( alloc: std.mem.Allocator, arena: std.mem.Allocator, source: []const u8, + result: ParseUrlResultType, ) Mapping.ParseResult { - const data_prefix = "data:application/json"; - if (bun.strings.hasPrefixComptime(source, data_prefix) and source.len > (data_prefix.len + 1)) try_data_url: { - const json_bytes = switch (source[data_prefix.len]) { - ';' => json_bytes: { - const encoding = bun.sliceTo(source[data_prefix.len + 1 ..], ','); - if (!bun.strings.eqlComptime(encoding, "base64")) break :try_data_url; - const base64_data = source[data_prefix.len + ";base64,".len ..]; - - const len = bun.base64.decodeLen(base64_data); - const bytes = arena.alloc(u8, len) catch bun.outOfMemory(); - const result = bun.base64.decode(bytes, base64_data); - if (result.fail) return .{ .fail = .{ - .msg = "Invalid Base64", - .err = error.InvalidBase64, - .value = 0, - .loc = .{ .start = 0 }, - } }; - break :json_bytes bytes[0..result.written]; - }, - ',' => source[data_prefix.len + 1 ..], - else => break :try_data_url, - }; + const json_bytes = json_bytes: { + const data_prefix = "data:application/json"; + if (bun.strings.hasPrefixComptime(source, data_prefix) and source.len > (data_prefix.len + 1)) try_data_url: { + switch (source[data_prefix.len]) { + ';' => { + const encoding = bun.sliceTo(source[data_prefix.len + 1 ..], ','); + if (!bun.strings.eqlComptime(encoding, "base64")) break :try_data_url; + const base64_data = source[data_prefix.len + ";base64,".len ..]; + + const len = bun.base64.decodeLen(base64_data); + const bytes = arena.alloc(u8, len) catch bun.outOfMemory(); + const decoded = bun.base64.decode(bytes, base64_data); + if (decoded.fail) return .{ .fail = .{ + .msg = "Invalid Base64", + .err = error.InvalidBase64, + .value = 0, + .loc = .{ .start = 0 }, + } }; + break :json_bytes bytes[0..decoded.written]; + }, + ',' => break :json_bytes source[data_prefix.len + 1 ..], + else => break :try_data_url, + } + } + + return .{ .fail = .{ + .msg = "Unsupported source map type", + .err = error.UnsupportedFormat, + .value = 0, + .loc = .{ .start = 0 }, + } }; + }; + return parseJSON(alloc, arena, json_bytes, result); +} + +/// Parses a JSON source-map +/// +/// `source` must be in UTF-8 and can be freed after this call. +/// The mappings are owned by the `alloc` allocator. +/// Temporary allocations are made to the `arena` allocator, which +/// should be an arena allocator (caller is assumed to call `deinit`). +pub fn parseJSON( + alloc: std.mem.Allocator, + arena: std.mem.Allocator, + source: []const u8, + result: ParseUrlResultType, +) Mapping.ParseResult { + const json_src = bun.logger.Source.initPathString("", source); + var log = bun.logger.Log.init(arena); + defer log.deinit(); + + var json = bun.JSON.ParseJSON(&json_src, &log, arena) catch { + return .{ .fail = .{ + .msg = "Invalid JSON", + .err = error.InvalidJSON, + .value = 0, + .loc = .{ .start = 0 }, + } }; + }; - const json_src = bun.logger.Source.initPathString("url.json", json_bytes); - var log = bun.logger.Log.init(arena); - defer log.deinit(); + // the allocator given to the JS parser is not respected for all parts + // of the parse, so we need to remember to reset the ast store + defer { + bun.JSAst.Expr.Data.Store.reset(); + bun.JSAst.Stmt.Data.Store.reset(); + } - var json = bun.JSON.ParseJSONUTF8(&json_src, &log, arena) catch { + if (json.get("version")) |version| { + if (version.data != .e_number or version.data.e_number.value != 3.0) { return .{ .fail = .{ - .msg = "Invalid JSON", - .err = error.InvalidJSON, + .msg = "Unsupported source map version", + .err = error.UnsupportedVersion, .value = 0, .loc = .{ .start = 0 }, } }; - }; - - if (json.get("version")) |version| { - if (version.data != .e_number or version.data.e_number.value != 3.0) { - return .{ .fail = .{ - .msg = "Unsupported source map version", - .err = error.UnsupportedVersion, - .value = 0, - .loc = .{ .start = 0 }, - } }; - } } + } + + const mappings_str = json.get("mappings") orelse { + return .{ .fail = .{ + .msg = "Missing source mappings", + .err = error.InvalidSourceMap, + .value = 0, + .loc = .{ .start = 0 }, + } }; + }; + + if (mappings_str.data != .e_string) { + return .{ .fail = .{ + .msg = "Missing source mappings", + .err = error.InvalidSourceMap, + .value = 0, + .loc = .{ .start = 0 }, + } }; + } + + const sources_content = switch ((json.get("sourcesContent") orelse { + return .{ .fail = .{ + .msg = "Missing sourcesContent", + .err = error.InvalidSourceMap, + .value = 0, + .loc = .{ .start = 0 }, + } }; + }).data) { + .e_array => |arr| arr, + else => return .{ .fail = .{ + .msg = "Missing sourcesContent", + .err = error.InvalidSourceMap, + .value = 0, + .loc = .{ .start = 0 }, + } }, + }; + + const sources_paths = switch ((json.get("sources") orelse { + return .{ .fail = .{ + .msg = "Missing sources", + .err = error.InvalidSourceMap, + .value = 0, + .loc = .{ .start = 0 }, + } }; + }).data) { + .e_array => |arr| arr, + else => return .{ .fail = .{ + .msg = "Missing sources", + .err = error.InvalidSourceMap, + .value = 0, + .loc = .{ .start = 0 }, + } }, + }; + + if (sources_content.items.len != sources_paths.items.len) { + return .{ .fail = .{ + .msg = "sources.length != sourcesContent.length", + .err = error.InvalidSourceMap, + .value = 0, + .loc = .{ .start = 0 }, + } }; + } + + const source_contents_slice = if (result != .mappings_only) + alloc.alloc(?[]const u8, sources_content.items.len) catch bun.outOfMemory() + else + undefined; + const source_paths_slice = alloc.alloc([]const u8, sources_content.items.len) catch bun.outOfMemory(); + + var i: usize = 0; + for (sources_paths.items.slice()) |item| { + if (item.data != .e_string) { + for (source_paths_slice[0..i]) |slice| alloc.free(slice); + alloc.free(source_paths_slice); + if (result != .mappings_only) alloc.free(source_contents_slice); - const mappings_str = json.get("mappings") orelse { return .{ .fail = .{ - .msg = "Missing source mappings", - .err = error.UnsupportedVersion, + .msg = "Expected string in sources", + .err = error.InvalidSourceMap, .value = 0, .loc = .{ .start = 0 }, } }; - }; + } + + const utf16_decode = bun.js_lexer.decodeUTF8(item.data.e_string.string(arena) catch bun.outOfMemory(), arena) catch bun.outOfMemory(); + defer arena.free(utf16_decode); + source_paths_slice[i] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch { + for (source_paths_slice[0..i]) |slice| alloc.free(slice); + alloc.free(source_paths_slice); + if (result != .mappings_only) alloc.free(source_contents_slice); - if (mappings_str.data != .e_string) { return .{ .fail = .{ - .msg = "Missing source mappings", - .err = error.UnsupportedVersion, + .msg = "Expected string in sources", + .err = error.InvalidSourceMap, .value = 0, .loc = .{ .start = 0 }, } }; - } + }; - return Mapping.parse( - alloc, - mappings_str.data.e_string.slice(arena), - null, - std.math.maxInt(i32), - std.math.maxInt(i32), - ); + i += 1; } - return .{ .fail = .{ - .msg = "Unsupported source map type", - .err = error.UnsupportedFormat, - .value = 0, - .loc = .{ .start = 0 }, - } }; + i = 0; + if (result != .mappings_only) { + for (sources_content.items.slice()) |item| { + if (item.data != .e_string) { + for (source_paths_slice) |slice| alloc.free(slice); + for (source_contents_slice[0..i]) |slice| + if (slice) |s| alloc.free(s); + alloc.free(source_paths_slice); + alloc.free(source_contents_slice); + + return .{ .fail = .{ + .msg = "Expected string in sources", + .err = error.InvalidSourceMap, + .value = 0, + .loc = .{ .start = 0 }, + } }; + } + + const utf16_decode = bun.js_lexer.decodeUTF8(item.data.e_string.string(arena) catch bun.outOfMemory(), arena) catch bun.outOfMemory(); + defer arena.free(utf16_decode); + source_contents_slice[i] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch { + for (source_paths_slice) |slice| alloc.free(slice); + for (source_contents_slice[0..i]) |slice| + if (slice) |s| alloc.free(s); + alloc.free(source_paths_slice); + alloc.free(source_contents_slice); + + return .{ .fail = .{ + .msg = "Expected string in sources", + .err = error.InvalidSourceMap, + .value = 0, + .loc = .{ .start = 0 }, + } }; + }; + + i += 1; + } + } + + switch (Mapping.parse( + alloc, + mappings_str.data.e_string.slice(arena), + null, + std.math.maxInt(i32), + std.math.maxInt(i32), + )) { + .fail => |f| { + for (source_paths_slice) |slice| alloc.free(slice); + if (result != .mappings_only) + for (source_contents_slice) |slice| + if (slice) |s| alloc.free(s); + alloc.free(source_paths_slice); + if (result != .mappings_only) alloc.free(source_contents_slice); + return .{ .fail = f }; + }, + .success => |map_const| { + var map = map_const; + map.external_source_names = source_paths_slice; + map.source_contents = if (result != .mappings_only) + Mapping.ParsedSourceMap.SourceContentPtr.fromSources(source_contents_slice.ptr) + else + .{ .data = 0, .state = .unloaded }; + return .{ .success = map }; + }, + } } pub const Mapping = struct { @@ -135,6 +308,43 @@ pub const Mapping = struct { original: LineColumnOffset, source_index: i32, + pub const Lookup = struct { + mapping: Mapping, + source_map: *ParsedSourceMap, + + /// This creates a bun.String if the source remap *changes* the source url, + /// a case that happens only when the source map points to another file. + pub fn displaySourceURLIfNeeded(lookup: Lookup) ?bun.String { + // See doc comment on `external_source_names` + if (lookup.source_map.external_source_names.len == 0) return null; + + // if crash here, then sourcemap is corrupt and remapping code did not catch this + assert(lookup.mapping.source_index < lookup.source_map.external_source_names.len); + + const name = lookup.source_map.external_source_names[@intCast(lookup.mapping.source_index)]; + return bun.String.init(name); + } + + /// Only valid if `lookup.source_map.isExternal()` + /// This has the possibility of invoking a call to the filesystem. + pub fn getSourceCode(lookup: Lookup) ?[]const u8 { + assert(lookup.source_map.isExternal()); + + if (lookup.source_map.source_contents.state == .unloaded) { + @panic("TODO: transition from .unloaded -> load source maps contents"); + } + + const sources = lookup.source_map.externalSourceContents(); + const i = lookup.mapping.source_index; + if (i >= sources.len or i < 0) return null; // invalid source map + if (sources[@intCast(i)]) |s| { + return s; + } + + @panic("TODO: load source from disk using name"); + } + }; + pub const List = std.MultiArrayList(Mapping); pub inline fn generatedLine(mapping: Mapping) i32 { @@ -415,14 +625,142 @@ pub const Mapping = struct { pub const ParsedSourceMap = struct { input_line_count: usize = 0, mappings: Mapping.List = .{}, + /// If this is empty, this implies that the source code is a single file + /// transpiled on-demand. If there are items, then it means this is a file + /// loaded without transpilation but with external sources. This array + /// maps `source_index` to the correct filename. + external_source_names: []const []const u8 = &.{}, + /// This is 64 bits encoding three usizes' worth of data. + /// + /// In .state = .unloaded, we do not have sources loaded in memory. + /// For most operations, having source code is not needed, only the mapping + /// list + filenames. To get source files from this state, `.data` will + /// be a SourceProviderMap + /// + /// When source files are loaded, this stores a pointer to a slice + /// of ?[]const u8, where null means the specified file is 1. not in the + /// sourcemap directly; and 2. not yet loaded. + /// + /// Only valid if `external_source_names` has entries. + source_contents: SourceContentPtr = .{ .state = .invalid, .data = 0 }, + + const SourceContentPtr = packed struct { + state: State, + data: u62, + + const State = enum(u2) { loaded, unloaded, invalid }; + + fn fromProvider(p: *SourceProviderMap) SourceContentPtr { + return .{ .state = .unloaded, .data = @intCast(@intFromPtr(p)) }; + } + + fn fromSources(p: [*]const ?[]const u8) SourceContentPtr { + return .{ .state = .loaded, .data = @intCast(@intFromPtr(p)) }; + } + + fn provider(sc: SourceContentPtr) *SourceProviderMap { + bun.assert(sc.state == .unloaded); + return @ptrFromInt(sc.data); + } + + fn sources(sc: SourceContentPtr) [*]const ?[]const u8 { + bun.assert(sc.state == .loaded); + return @ptrFromInt(sc.data); + } + }; + + pub fn isExternal(psm: *ParsedSourceMap) bool { + return psm.external_source_names.len != 0; + } + + pub fn externalSourceContents(psm: *ParsedSourceMap) []const ?[]const u8 { + return psm.source_contents.sources()[0..psm.external_source_names.len]; + } pub fn deinit(this: *ParsedSourceMap, allocator: std.mem.Allocator) void { this.mappings.deinit(allocator); + // if (this.external_source_names.len > 0) { + // bun.assert(this.external_source_contents.len > 0); + // for (this.external_source_names) |name| allocator.free(name); + // for (this.external_source_contents) |content| allocator.free(content); + // allocator.free(this.external_source_names); + // allocator.free(this.external_source_contents); + // } allocator.destroy(this); } }; }; +/// For some sourcemap loading code, this enum is used as a hint if it should +/// bother loading source code into memory. Most uses of source maps only care +/// about filenames and source mappings, and we should avoid loading contents +/// whenever possible. +pub const SourceContentHandling = enum { + no_source_contents, + source_contents, +}; + +/// This is a pointer to a ZigSourceProvider that may or may not have a `//# sourceMappingURL` comment +/// when we want to lookup this data, we will then resolve it to a ParsedSourceMap if it does. +/// +/// This is used for files that were pre-bundled with `bun build --target=bun --sourcemap` +pub const SourceProviderMap = opaque { + extern fn ZigSourceProvider__getSourceSlice(*SourceProviderMap) bun.String; + + pub fn toParsedSourceMap(provider: *SourceProviderMap, contents: SourceContentHandling) ?*Mapping.ParsedSourceMap { + const bun_str = ZigSourceProvider__getSourceSlice(provider); + defer bun_str.deref(); + bun.assert(bun_str.tag == .WTFStringImpl); + const wtf = bun_str.value.WTFStringImpl; + + // TODO: do not use toUTF8() + const utf8 = wtf.toUTF8(bun.default_allocator); + defer utf8.deinit(); + const bytes = utf8.slice(); + + // TODO: use smarter technique + if (std.mem.indexOf(u8, bytes, "//# sourceMappingURL=")) |index| { + const end = std.mem.indexOfAnyPos(u8, bytes, index + "//# sourceMappingURL=".len, " \n\r") orelse bytes.len; + const url = bytes[index + "//# sourceMappingURL=".len .. end]; + + var sfb = std.heap.stackFallback(32768, bun.default_allocator); + var arena = bun.ArenaAllocator.init(sfb.get()); + defer arena.deinit(); + + switch (parseUrl( + bun.default_allocator, + arena.allocator(), + url, + // only aquire sourcemap if the load request desires one + switch (contents) { + .no_source_contents => .mappings_only, + .source_contents => .all, + }, + )) { + .fail => { + return null; + }, + .success => |parsed| { + const ptr = bun.default_allocator.create(Mapping.ParsedSourceMap) catch bun.outOfMemory(); + ptr.* = parsed; + if (ptr.source_contents.state == .unloaded) + ptr.source_contents = Mapping.ParsedSourceMap.SourceContentPtr.fromProvider(provider); + return ptr; + }, + } + } + + // TODO: look for file of same filename. + + return null; + } + + pub fn deinit(provider: *SourceProviderMap) void { + _ = provider; + @panic("TODO"); + } +}; + pub const LineColumnOffset = struct { lines: i32 = 0, columns: i32 = 0, From 3cf3838886d3ae14b4f53eaa724f40f1195d6730 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Fri, 10 May 2024 23:54:02 -0700 Subject: [PATCH 03/18] tidy --- src/bun.js/javascript.zig | 4 ++-- src/sourcemap/sourcemap.zig | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 0d750d029c880..eec18b4fc6f04 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -238,7 +238,7 @@ pub const SavedSourceMap = struct { var saved = SavedMappings{ .data = @as([*]u8, @ptrCast(saved_mappings)) }; saved.deinit(); } else if (old_value.get(SourceProviderMap)) |provider| { - provider.deinit(); + _ = provider; // do nothing } } entry.value_ptr.* = value.ptr(); @@ -268,7 +268,7 @@ pub const SavedSourceMap = struct { mapping.value_ptr.* = Value.init(result).ptr(); return result; } else { - // does not have a valid source map + // does not have a valid source map. let's not try again _ = this.map.remove(hash); return null; } diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig index 73e5f2e0e7061..04a3ccb62448c 100644 --- a/src/sourcemap/sourcemap.zig +++ b/src/sourcemap/sourcemap.zig @@ -687,6 +687,7 @@ pub const Mapping = struct { // allocator.free(this.external_source_contents); // } allocator.destroy(this); + @panic("TODO: this destructor"); } }; }; @@ -754,11 +755,6 @@ pub const SourceProviderMap = opaque { return null; } - - pub fn deinit(provider: *SourceProviderMap) void { - _ = provider; - @panic("TODO"); - } }; pub const LineColumnOffset = struct { From 87eaf36c6e5c4965e84f7134e3a7ca688787b593 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Mon, 13 May 2024 17:55:53 -0700 Subject: [PATCH 04/18] work --- src/bun.js/bindings/BunString.cpp | 18 +- src/bun.js/bindings/ZigGlobalObject.cpp | 2 +- src/bun.js/bindings/ZigSourceProvider.cpp | 3 +- src/bun.js/bindings/headers-handwritten.h | 4 + src/bun.js/javascript.zig | 81 +-- src/sourcemap/sourcemap.zig | 645 ++++++++++++---------- src/string_immutable.zig | 5 + 7 files changed, 423 insertions(+), 335 deletions(-) diff --git a/src/bun.js/bindings/BunString.cpp b/src/bun.js/bindings/BunString.cpp index c156d6a57f412..097401cd21542 100644 --- a/src/bun.js/bindings/BunString.cpp +++ b/src/bun.js/bindings/BunString.cpp @@ -196,6 +196,22 @@ BunString toStringRef(WTF::StringImpl* wtfString) return { BunStringTag::WTFStringImpl, { .wtf = wtfString } }; } +BunString toStringView(StringView view) { + view.is8Bit(); + return { + BunStringTag::StaticZigString, + { .zig = { + .ptr = (const LChar*)(view.is8Bit() + ? (size_t)view.rawCharacters() + : ((size_t)view.rawCharacters() + (static_cast(1) << 63)) + ), + .len = view.length() + } } + }; +} + + + } extern "C" JSC::EncodedJSValue BunString__toJS(JSC::JSGlobalObject* globalObject, const BunString* bunString) @@ -582,4 +598,4 @@ extern "C" void JSC__JSValue__putBunString( WTF::String str = key->tag == BunStringTag::Empty ? WTF::String(""_s) : key->toWTFString(); Identifier id = Identifier::fromString(vm, str); target->putDirect(vm, id, value, 0); -} \ No newline at end of file +} diff --git a/src/bun.js/bindings/ZigGlobalObject.cpp b/src/bun.js/bindings/ZigGlobalObject.cpp index d28902f37c811..2a4d1b280cc71 100644 --- a/src/bun.js/bindings/ZigGlobalObject.cpp +++ b/src/bun.js/bindings/ZigGlobalObject.cpp @@ -413,7 +413,7 @@ WTF::String Bun::formatStackTrace(JSC::VM& vm, JSC::JSGlobalObject* globalObject sb.append(" at ("_s); - sb.append(sourceURLForFrame); + sb.append(remappedFrame.source_url.toWTFString()); if (remappedFrame.remapped) { errorInstance->putDirect(vm, Identifier::fromString(vm, "originalLine"_s), jsNumber(originalLine.oneBasedInt()), 0); diff --git a/src/bun.js/bindings/ZigSourceProvider.cpp b/src/bun.js/bindings/ZigSourceProvider.cpp index 6dd9a87310cc0..7cca068a49dec 100644 --- a/src/bun.js/bindings/ZigSourceProvider.cpp +++ b/src/bun.js/bindings/ZigSourceProvider.cpp @@ -239,8 +239,7 @@ int SourceProvider::readCache(JSC::VM& vm, const JSC::SourceCode& sourceCode) } extern "C" BunString ZigSourceProvider__getSourceSlice(SourceProvider* provider) { - // TODO(@paperdave): does this clone the data? i sure hope not. - return Bun::toStringRef(provider->source().toString()); + return Bun::toStringView(provider->source()); } }; // namespace Zig diff --git a/src/bun.js/bindings/headers-handwritten.h b/src/bun.js/bindings/headers-handwritten.h index 1caa74af617ee..bef00c36bdeab 100644 --- a/src/bun.js/bindings/headers-handwritten.h +++ b/src/bun.js/bindings/headers-handwritten.h @@ -279,6 +279,10 @@ BunString toStringRef(JSC::JSGlobalObject* globalObject, JSC::JSValue value); BunString toStringRef(WTF::String& wtfString); BunString toStringRef(const WTF::String& wtfString); BunString toStringRef(WTF::StringImpl* wtfString); + +// This creates a detached string view, which cannot be ref/unref. +// Be very careful using this, and ensure the memory owner does not get destroyed. +BunString toStringView(WTF::StringView view); } using Uint8Array_alias = JSC::JSUint8Array; diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index eec18b4fc6f04..cde7b73ce76ac 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -143,37 +143,32 @@ pub const SavedSourceMap = struct { default_allocator.free(this.data[0..this.len()]); } - pub fn toMapping(this: SavedMappings, allocator: Allocator, path: string) anyerror!ParsedSourceMap { - const result = SourceMap.Mapping.parse( + pub fn toMapping(this: SavedMappings, allocator: Allocator, path: string) !ParsedSourceMap { + var err_data: SourceMap.Mapping.ParseError = undefined; + return SourceMap.Mapping.parse( allocator, this.data[vlq_offset..this.len()], @as(usize, @bitCast(this.data[8..16].*)), 1, @as(usize, @bitCast(this.data[16..24].*)), - ); - switch (result) { - .fail => |fail| { - if (Output.enable_ansi_colors_stderr) { - try fail.toData(path).writeFormat( - Output.errorWriter(), - logger.Kind.warn, - true, - ); - } else { - try fail.toData(path).writeFormat( - Output.errorWriter(), - logger.Kind.warn, - - false, - ); - } + &err_data, + ) catch |err| { + if (Output.enable_ansi_colors_stderr) { + try err_data.toData(path).writeFormat( + Output.errorWriter(), + logger.Kind.warn, + true, + ); + } else { + try err_data.toData(path).writeFormat( + Output.errorWriter(), + logger.Kind.warn, - return fail.err; - }, - .success => |success| { - return success; - }, - } + false, + ); + } + return err; + }; } }; @@ -264,7 +259,7 @@ pub const SavedSourceMap = struct { }, Value.Tag.SourceProviderMap => { var ptr = Value.from(mapping.value_ptr.*).as(SourceProviderMap); - if (ptr.toParsedSourceMap(contents)) |result| { + if (ptr.toParsedSourceMap(path, contents)) |result| { mapping.value_ptr.* = Value.init(result).ptr(); return result; } else { @@ -2769,7 +2764,8 @@ pub const VirtualMachine = struct { @max(frame.position.column_start, 0), .no_source_contents, )) |lookup| { - if (lookup.displaySourceURLIfNeeded()) |source_url| { + if (lookup.displaySourceURLIfNeeded(sourceURL.slice())) |source_url| { + frame.source_url.deref(); frame.source_url = source_url; } const mapping = lookup.mapping; @@ -2785,6 +2781,7 @@ pub const VirtualMachine = struct { pub fn remapZigException(this: *VirtualMachine, exception: *ZigException, error_instance: JSValue, exception_list: ?*ExceptionList, must_reset_parser_arena_later: *bool) void { error_instance.toZigException(this.global, exception); + // defer this so that it copies correctly defer { if (exception_list) |list| { @@ -2885,18 +2882,22 @@ pub const VirtualMachine = struct { if (maybe_lookup) |lookup| { const mapping = lookup.mapping; + if (lookup.displaySourceURLIfNeeded(top_source_url.slice())) |src| { + top.source_url.deref(); + top.source_url = src; + } + const code = code: { - if (top.remapped or lookup.source_map.external_source_names.len == 0) { - var log = logger.Log.init(default_allocator); - var original_source = fetchWithoutOnLoadPlugins(this, this.global, top.source_url, bun.String.empty, &log, .print_source) catch return; - must_reset_parser_arena_later.* = true; - break :code original_source.source_code.toUTF8(bun.default_allocator); - } else { - break :code ZigString.Slice.initStatic( - lookup.getSourceCode() orelse - @panic("TODO: what if this failed"), - ); + if (!top.remapped and lookup.source_map.isExternal()) { + if (lookup.getSourceCode(top_source_url.slice())) |src| { + break :code ZigString.Slice.initStatic(src); + } } + + var log = logger.Log.init(default_allocator); + var original_source = fetchWithoutOnLoadPlugins(this, this.global, top.source_url, bun.String.empty, &log, .print_source) catch return; + must_reset_parser_arena_later.* = true; + break :code original_source.source_code.toUTF8(bun.default_allocator); }; defer code.deinit(); @@ -2955,7 +2956,8 @@ pub const VirtualMachine = struct { @max(frame.position.column_start, 0), .no_source_contents, )) |lookup| { - if (lookup.displaySourceURLIfNeeded()) |src| { + if (lookup.displaySourceURLIfNeeded(source_url.slice())) |src| { + frame.source_url.deref(); frame.source_url = src; } const mapping = lookup.mapping; @@ -3890,8 +3892,7 @@ pub fn NewHotReloader(comptime Ctx: type, comptime EventLoopType: type, comptime }; } -export fn Bun__addSourceProviderSourceMap(bun_vm: *anyopaque, opaque_source_provider: *anyopaque, specifier: *bun.String) void { - const vm: *VirtualMachine = @alignCast(@ptrCast(bun_vm)); +export fn Bun__addSourceProviderSourceMap(vm: *VirtualMachine, opaque_source_provider: *anyopaque, specifier: *bun.String) void { var sfb = std.heap.stackFallback(4096, bun.default_allocator); const slice = specifier.toUTF8(sfb.get()); defer slice.deinit(); diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig index 04a3ccb62448c..535dc3e39aac0 100644 --- a/src/sourcemap/sourcemap.zig +++ b/src/sourcemap/sourcemap.zig @@ -43,6 +43,20 @@ const ParseUrlResultType = enum { all, mappings_only, sources_only, + + pub fn GetSourceMapType(comptime t: ParseUrlResultType) type { + return switch (t) { + .all, .mappings_only => *Mapping.ParsedSourceMap, + .sources_only => []?[]const u8, + }; + } + + pub fn ParseType(comptime t: ParseUrlResultType) type { + return switch (t) { + .all, .mappings_only => Mapping.ParsedSourceMap, + .sources_only => []?[]const u8, + }; + } }; /// Parses an inline source map url like `data:application/json,....` @@ -56,8 +70,9 @@ pub fn parseUrl( alloc: std.mem.Allocator, arena: std.mem.Allocator, source: []const u8, - result: ParseUrlResultType, -) Mapping.ParseResult { + comptime result: ParseUrlResultType, + err: *Mapping.ParseError, +) !result.ParseType() { const json_bytes = json_bytes: { const data_prefix = "data:application/json"; if (bun.strings.hasPrefixComptime(source, data_prefix) and source.len > (data_prefix.len + 1)) try_data_url: { @@ -70,12 +85,13 @@ pub fn parseUrl( const len = bun.base64.decodeLen(base64_data); const bytes = arena.alloc(u8, len) catch bun.outOfMemory(); const decoded = bun.base64.decode(bytes, base64_data); - if (decoded.fail) return .{ .fail = .{ - .msg = "Invalid Base64", - .err = error.InvalidBase64, - .value = 0, - .loc = .{ .start = 0 }, - } }; + if (decoded.fail) { + return err.fail(error.InvalidBase64, .{ + .msg = "Invalid Base64", + .value = 0, + .loc = .{ .start = 0 }, + }); + } break :json_bytes bytes[0..decoded.written]; }, ',' => break :json_bytes source[data_prefix.len + 1 ..], @@ -83,14 +99,14 @@ pub fn parseUrl( } } - return .{ .fail = .{ + return err.fail(error.UnsupportedFormat, .{ .msg = "Unsupported source map type", - .err = error.UnsupportedFormat, .value = 0, .loc = .{ .start = 0 }, - } }; + }); }; - return parseJSON(alloc, arena, json_bytes, result); + + return parseJSON(alloc, arena, json_bytes, result, err); } /// Parses a JSON source-map @@ -103,19 +119,15 @@ pub fn parseJSON( alloc: std.mem.Allocator, arena: std.mem.Allocator, source: []const u8, - result: ParseUrlResultType, -) Mapping.ParseResult { + comptime result: ParseUrlResultType, + err: *Mapping.ParseError, +) !result.ParseType() { const json_src = bun.logger.Source.initPathString("", source); var log = bun.logger.Log.init(arena); defer log.deinit(); var json = bun.JSON.ParseJSON(&json_src, &log, arena) catch { - return .{ .fail = .{ - .msg = "Invalid JSON", - .err = error.InvalidJSON, - .value = 0, - .loc = .{ .start = 0 }, - } }; + return err.fail(error.InvalidJSON, .{}); }; // the allocator given to the JS parser is not respected for all parts @@ -127,95 +139,64 @@ pub fn parseJSON( if (json.get("version")) |version| { if (version.data != .e_number or version.data.e_number.value != 3.0) { - return .{ .fail = .{ - .msg = "Unsupported source map version", - .err = error.UnsupportedVersion, - .value = 0, - .loc = .{ .start = 0 }, - } }; + return err.fail(error.UnsupportedVersion, .{}); } } const mappings_str = json.get("mappings") orelse { - return .{ .fail = .{ - .msg = "Missing source mappings", - .err = error.InvalidSourceMap, - .value = 0, - .loc = .{ .start = 0 }, - } }; + return err.fail(error.UnsupportedVersion, .{}); }; if (mappings_str.data != .e_string) { - return .{ .fail = .{ - .msg = "Missing source mappings", - .err = error.InvalidSourceMap, - .value = 0, - .loc = .{ .start = 0 }, - } }; + return err.fail(error.InvalidSourceMap, .{}); } - const sources_content = switch ((json.get("sourcesContent") orelse { - return .{ .fail = .{ - .msg = "Missing sourcesContent", - .err = error.InvalidSourceMap, - .value = 0, - .loc = .{ .start = 0 }, - } }; - }).data) { + const sources_content = switch ((json.get("sourcesContent") orelse return error.InvalidSourceMap).data) { .e_array => |arr| arr, - else => return .{ .fail = .{ - .msg = "Missing sourcesContent", - .err = error.InvalidSourceMap, - .value = 0, - .loc = .{ .start = 0 }, - } }, + else => return err.fail(error.InvalidSourceMap, .{}), }; - const sources_paths = switch ((json.get("sources") orelse { - return .{ .fail = .{ - .msg = "Missing sources", - .err = error.InvalidSourceMap, - .value = 0, - .loc = .{ .start = 0 }, - } }; - }).data) { + const sources_paths = switch ((json.get("sources") orelse return error.InvalidSourceMap).data) { .e_array => |arr| arr, - else => return .{ .fail = .{ - .msg = "Missing sources", - .err = error.InvalidSourceMap, - .value = 0, - .loc = .{ .start = 0 }, - } }, + else => return err.fail(error.InvalidSourceMap, .{}), }; if (sources_content.items.len != sources_paths.items.len) { - return .{ .fail = .{ + return err.fail(error.InvalidSourceMap, .{ .msg = "sources.length != sourcesContent.length", - .err = error.InvalidSourceMap, - .value = 0, - .loc = .{ .start = 0 }, - } }; + }); } + var i: usize = 0; + const source_contents_slice = if (result != .mappings_only) - alloc.alloc(?[]const u8, sources_content.items.len) catch bun.outOfMemory() - else - undefined; - const source_paths_slice = alloc.alloc([]const u8, sources_content.items.len) catch bun.outOfMemory(); + alloc.alloc(?[]const u8, sources_content.items.len) catch bun.outOfMemory(); + errdefer if (result != .mappings_only) { + for (source_contents_slice[0..i]) |item_nullable| if (item_nullable) |item| { + if (item.len > 0) + alloc.free(item); + }; + alloc.free(source_contents_slice); + }; - var i: usize = 0; - for (sources_paths.items.slice()) |item| { + const source_paths_slice = if (result != .sources_only) + alloc.alloc([]const u8, sources_content.items.len) catch bun.outOfMemory(); + errdefer if (result != .sources_only) { + for (source_paths_slice[0..i]) |item| alloc.free(item); + alloc.free(source_paths_slice); + }; + + if (result != .sources_only) for (sources_paths.items.slice()) |item| { if (item.data != .e_string) { for (source_paths_slice[0..i]) |slice| alloc.free(slice); alloc.free(source_paths_slice); if (result != .mappings_only) alloc.free(source_contents_slice); - return .{ .fail = .{ + return err.fail(error.InvalidSourceMap, .{ .msg = "Expected string in sources", - .err = error.InvalidSourceMap, .value = 0, .loc = .{ .start = 0 }, - } }; + }); } const utf16_decode = bun.js_lexer.decodeUTF8(item.data.e_string.string(arena) catch bun.outOfMemory(), arena) catch bun.outOfMemory(); @@ -225,82 +206,64 @@ pub fn parseJSON( alloc.free(source_paths_slice); if (result != .mappings_only) alloc.free(source_contents_slice); - return .{ .fail = .{ + return err.fail(error.InvalidSourceMap, .{ .msg = "Expected string in sources", - .err = error.InvalidSourceMap, .value = 0, .loc = .{ .start = 0 }, - } }; + }); }; i += 1; - } + }; - i = 0; + var j: usize = 0; if (result != .mappings_only) { for (sources_content.items.slice()) |item| { + defer j += 1; + if (item.data != .e_string) { - for (source_paths_slice) |slice| alloc.free(slice); - for (source_contents_slice[0..i]) |slice| - if (slice) |s| alloc.free(s); - alloc.free(source_paths_slice); - alloc.free(source_contents_slice); + source_contents_slice[j] = null; + continue; + } - return .{ .fail = .{ - .msg = "Expected string in sources", - .err = error.InvalidSourceMap, - .value = 0, - .loc = .{ .start = 0 }, - } }; + const str = item.data.e_string.string(arena) catch bun.outOfMemory(); + if (str.len == 0) { + source_contents_slice[j] = ""; + continue; } - const utf16_decode = bun.js_lexer.decodeUTF8(item.data.e_string.string(arena) catch bun.outOfMemory(), arena) catch bun.outOfMemory(); + const utf16_decode = bun.js_lexer.decodeUTF8(str, arena) catch bun.outOfMemory(); defer arena.free(utf16_decode); - source_contents_slice[i] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch { - for (source_paths_slice) |slice| alloc.free(slice); - for (source_contents_slice[0..i]) |slice| - if (slice) |s| alloc.free(s); - alloc.free(source_paths_slice); - alloc.free(source_contents_slice); - - return .{ .fail = .{ + + source_contents_slice[j] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch { + return err.fail(error.InvalidSourceMap, .{ .msg = "Expected string in sources", - .err = error.InvalidSourceMap, .value = 0, .loc = .{ .start = 0 }, - } }; + }); }; - - i += 1; + bun.assert(source_contents_slice[j].?.len > 0); } } - switch (Mapping.parse( + if (result == .sources_only) { + return source_contents_slice; + } + + var map = try Mapping.parse( alloc, mappings_str.data.e_string.slice(arena), null, std.math.maxInt(i32), std.math.maxInt(i32), - )) { - .fail => |f| { - for (source_paths_slice) |slice| alloc.free(slice); - if (result != .mappings_only) - for (source_contents_slice) |slice| - if (slice) |s| alloc.free(s); - alloc.free(source_paths_slice); - if (result != .mappings_only) alloc.free(source_contents_slice); - return .{ .fail = f }; - }, - .success => |map_const| { - var map = map_const; - map.external_source_names = source_paths_slice; - map.source_contents = if (result != .mappings_only) - Mapping.ParsedSourceMap.SourceContentPtr.fromSources(source_contents_slice.ptr) - else - .{ .data = 0, .state = .unloaded }; - return .{ .success = map }; - }, - } + err, + ); + map.external_source_names = source_paths_slice; + map.source_contents = if (result != .mappings_only) + Mapping.ParsedSourceMap.SourceContentPtr.fromSources(source_contents_slice.ptr) + else + .{ .data = 0, .state = .unloaded }; + return map; } pub const Mapping = struct { @@ -314,34 +277,73 @@ pub const Mapping = struct { /// This creates a bun.String if the source remap *changes* the source url, /// a case that happens only when the source map points to another file. - pub fn displaySourceURLIfNeeded(lookup: Lookup) ?bun.String { + pub fn displaySourceURLIfNeeded(lookup: Lookup, base_filename: []const u8) ?bun.String { // See doc comment on `external_source_names` - if (lookup.source_map.external_source_names.len == 0) return null; - - // if crash here, then sourcemap is corrupt and remapping code did not catch this - assert(lookup.mapping.source_index < lookup.source_map.external_source_names.len); + if (lookup.source_map.external_source_names.len == 0) + return null; + if (lookup.mapping.source_index >= lookup.source_map.external_source_names.len) + return null; const name = lookup.source_map.external_source_names[@intCast(lookup.mapping.source_index)]; + + if (std.fs.path.isAbsolute(base_filename)) { + const dir = bun.path.dirname(base_filename, .auto); + return bun.String.init(bun.path.joinAbs(dir, .auto, name)); + } + return bun.String.init(name); } /// Only valid if `lookup.source_map.isExternal()` /// This has the possibility of invoking a call to the filesystem. - pub fn getSourceCode(lookup: Lookup) ?[]const u8 { + pub fn getSourceCode(lookup: Lookup, base_filename: []const u8) ?[]const u8 { assert(lookup.source_map.isExternal()); + if (lookup.mapping.source_index >= lookup.source_map.external_source_names.len) + return null; if (lookup.source_map.source_contents.state == .unloaded) { - @panic("TODO: transition from .unloaded -> load source maps contents"); + const map = lookup.source_map.source_contents.provider().getSourceMap( + base_filename, + if (lookup.source_map.source_contents.load_from_separate_file) + .is_external_map + else + .is_inline_map, + .sources_only, + ) orelse return null; + assert(map.len == lookup.source_map.external_source_names.len); + lookup.source_map.source_contents = Mapping.ParsedSourceMap.SourceContentPtr.fromSources(map.ptr); } const sources = lookup.source_map.externalSourceContents(); const i = lookup.mapping.source_index; if (i >= sources.len or i < 0) return null; // invalid source map if (sources[@intCast(i)]) |s| { + if (s.len == 0) return null; return s; } - @panic("TODO: load source from disk using name"); + const name = lookup.source_map.external_source_names[@intCast(i)]; + + var buf: bun.PathBuffer = undefined; + const normalized = bun.path.joinAbsStringBufZ( + bun.path.dirname(base_filename, .auto), + &buf, + &.{name}, + .loose, + ); + const bytes = switch (bun.sys.File.readFrom( + std.fs.cwd(), + normalized, + bun.default_allocator, + )) { + .result => |r| r, + .err => { + sources[@intCast(i)] = ""; + return null; + }, + }; + sources[@intCast(i)] = bytes; + return bytes; } }; @@ -401,17 +403,20 @@ pub const Mapping = struct { return null; } + /// On error, `err` is populated with error information. pub fn parse( allocator: std.mem.Allocator, bytes: []const u8, estimated_mapping_count: ?usize, sources_count: i32, input_line_count: usize, - ) ParseResult { + err: *ParseError, + ) !ParsedSourceMap { var mapping = Mapping.List{}; if (estimated_mapping_count) |count| { - mapping.ensureTotalCapacity(allocator, count) catch unreachable; + mapping.ensureTotalCapacity(allocator, count) catch bun.outOfMemory(); } + errdefer mapping.deinit(allocator); var generated = LineColumnOffset{ .lines = 0, .columns = 0 }; var original = LineColumnOffset{ .lines = 0, .columns = 0 }; @@ -444,28 +449,22 @@ pub const Mapping = struct { const generated_column_delta = decodeVLQ(remain, 0); if (generated_column_delta.start == 0) { - return .{ - .fail = .{ - .msg = "Missing generated column value", - .err = error.MissingGeneratedColumnValue, - .value = generated.columns, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }, - }; + return err.fail(error.MissingGeneratedColumnValue, .{ + .msg = "Missing generated column value", + .value = generated.columns, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }); } needs_sort = needs_sort or generated_column_delta.value < 0; generated.columns += generated_column_delta.value; if (generated.columns < 0) { - return .{ - .fail = .{ - .msg = "Invalid generated column value", - .err = error.InvalidGeneratedColumnValue, - .value = generated.columns, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }, - }; + return err.fail(error.InvalidGeneratedColumnValue, .{ + .msg = "Invalid generated column value", + .value = generated.columns, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }); } remain = remain[generated_column_delta.start..]; @@ -491,81 +490,59 @@ pub const Mapping = struct { // Read the original source const source_index_delta = decodeVLQ(remain, 0); if (source_index_delta.start == 0) { - return .{ - .fail = .{ - .msg = "Invalid source index delta", - .err = error.InvalidSourceIndexDelta, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }, - }; + return err.fail(error.InvalidSourceIndexDelta, .{ + .msg = "Invalid source index delta", + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }); } source_index += source_index_delta.value; if (source_index < 0 or source_index > sources_count) { - return .{ - .fail = .{ - .msg = "Invalid source index value", - .err = error.InvalidSourceIndexValue, - .value = source_index, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }, - }; + return err.fail(error.InvalidSourceIndexValue, .{ + .msg = "Invalid source index value", + .value = source_index, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }); } remain = remain[source_index_delta.start..]; - // // "AAAA" is extremely common - // if (strings.hasPrefixComptime(remain, "AAAA;")) { - - // } - // Read the original line const original_line_delta = decodeVLQ(remain, 0); if (original_line_delta.start == 0) { - return .{ - .fail = .{ - .msg = "Missing original line", - .err = error.MissingOriginalLine, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }, - }; + return err.fail(error.MissingOriginalLine, .{ + .msg = "Missing original line", + .err = error.MissingOriginalLine, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }); } original.lines += original_line_delta.value; if (original.lines < 0) { - return .{ - .fail = .{ - .msg = "Invalid original line value", - .err = error.InvalidOriginalLineValue, - .value = original.lines, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }, - }; + return err.fail(error.InvalidOriginalLineValue, .{ + .msg = "Invalid original line value", + .value = original.lines, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }); } remain = remain[original_line_delta.start..]; // Read the original column const original_column_delta = decodeVLQ(remain, 0); if (original_column_delta.start == 0) { - return .{ - .fail = .{ - .msg = "Missing original column value", - .err = error.MissingOriginalColumnValue, - .value = original.columns, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }, - }; + return err.fail(error.MissingOriginalColumnValue, .{ + .msg = "Missing original column value", + .value = original.columns, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }); } original.columns += original_column_delta.value; if (original.columns < 0) { - return .{ - .fail = .{ - .msg = "Invalid original column value", - .err = error.InvalidOriginalColumnValue, - .value = original.columns, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }, - }; + return err.fail(error.InvalidOriginalColumnValue, .{ + .msg = "Invalid original column value", + .value = original.columns, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }); } remain = remain[original_column_delta.start..]; @@ -576,14 +553,11 @@ pub const Mapping = struct { }, ';' => {}, else => |c| { - return .{ - .fail = .{ - .msg = "Invalid character after mapping", - .err = error.InvalidSourceMap, - .value = @as(i32, @intCast(c)), - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }, - }; + return err.fail(error.InvalidSourceMap, .{ + .msg = "Invalid character after mapping", + .value = @as(i32, @intCast(c)), + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }); }, } } @@ -591,35 +565,38 @@ pub const Mapping = struct { .generated = generated, .original = original, .source_index = source_index, - }) catch unreachable; + }) catch bun.outOfMemory(); } - return ParseResult{ - .success = .{ - .mappings = mapping, - .input_line_count = input_line_count, - }, + return .{ + .mappings = mapping, + .input_line_count = input_line_count, }; } - pub const ParseResult = union(enum) { - fail: struct { - loc: Logger.Loc, - err: anyerror, - value: i32 = 0, - msg: []const u8 = "", - - pub fn toData(this: @This(), path: []const u8) Logger.Data { - return Logger.Data{ - .location = Logger.Location{ - .file = path, - .offset = this.loc.toUsize(), - }, - .text = this.msg, - }; - } - }, - success: ParsedSourceMap, + pub const ParseError = struct { + loc: Logger.Loc = Logger.Loc.Empty, + err: anyerror = error.Unexpected, + value: i32 = 0, + msg: []const u8 = "", + + pub fn fail(undefined_ptr: *ParseError, comptime err_tag: anytype, data: ParseError) @Type(.{ + .ErrorSet = &.{.{ .name = @errorName(err_tag) }}, + }) { + undefined_ptr.* = data; + undefined_ptr.err = err_tag; + return err_tag; + } + + pub fn toData(this: @This(), path: []const u8) Logger.Data { + return .{ + .location = .{ + .file = path, + .offset = this.loc.toUsize(), + }, + .text = this.msg, + }; + } }; pub const ParsedSourceMap = struct { @@ -642,11 +619,16 @@ pub const Mapping = struct { /// sourcemap directly; and 2. not yet loaded. /// /// Only valid if `external_source_names` has entries. + /// + /// `.load_from_map_file = true` means the file should load via a `.map` + /// file adacent to the source file, where `false` means there is an + /// inline source map to use. source_contents: SourceContentPtr = .{ .state = .invalid, .data = 0 }, - const SourceContentPtr = packed struct { + const SourceContentPtr = packed struct(u64) { + load_from_separate_file: bool = false, state: State, - data: u62, + data: u61, const State = enum(u2) { loaded, unloaded, invalid }; @@ -654,7 +636,7 @@ pub const Mapping = struct { return .{ .state = .unloaded, .data = @intCast(@intFromPtr(p)) }; } - fn fromSources(p: [*]const ?[]const u8) SourceContentPtr { + fn fromSources(p: [*]?[]const u8) SourceContentPtr { return .{ .state = .loaded, .data = @intCast(@intFromPtr(p)) }; } @@ -663,7 +645,7 @@ pub const Mapping = struct { return @ptrFromInt(sc.data); } - fn sources(sc: SourceContentPtr) [*]const ?[]const u8 { + fn sources(sc: SourceContentPtr) [*]?[]const u8 { bun.assert(sc.state == .loaded); return @ptrFromInt(sc.data); } @@ -673,21 +655,32 @@ pub const Mapping = struct { return psm.external_source_names.len != 0; } - pub fn externalSourceContents(psm: *ParsedSourceMap) []const ?[]const u8 { + pub fn externalSourceContents(psm: *ParsedSourceMap) []?[]const u8 { return psm.source_contents.sources()[0..psm.external_source_names.len]; } pub fn deinit(this: *ParsedSourceMap, allocator: std.mem.Allocator) void { this.mappings.deinit(allocator); - // if (this.external_source_names.len > 0) { - // bun.assert(this.external_source_contents.len > 0); - // for (this.external_source_names) |name| allocator.free(name); - // for (this.external_source_contents) |content| allocator.free(content); - // allocator.free(this.external_source_names); - // allocator.free(this.external_source_contents); - // } + + if (this.external_source_names.len > 0) { + for (this.external_source_names) |name| allocator.free(name); + allocator.free(this.external_source_names); + } + + switch (this.source_contents.state) { + .invalid, .unloaded => {}, + .loaded => { + bun.assert(this.external_source_names.len > 0); + const slice = this.externalSourceContents(); + for (slice) |maybe_item| if (maybe_item) |item| { + if (item.len > 0) + allocator.free(item); + }; + allocator.free(slice); + }, + } + allocator.destroy(this); - @panic("TODO: this destructor"); } }; }; @@ -701,6 +694,12 @@ pub const SourceContentHandling = enum { source_contents, }; +pub const SourceMapLoadHint = enum { + none, + is_inline_map, + is_external_map, +}; + /// This is a pointer to a ZigSourceProvider that may or may not have a `//# sourceMappingURL` comment /// when we want to lookup this data, we will then resolve it to a ParsedSourceMap if it does. /// @@ -708,52 +707,116 @@ pub const SourceContentHandling = enum { pub const SourceProviderMap = opaque { extern fn ZigSourceProvider__getSourceSlice(*SourceProviderMap) bun.String; - pub fn toParsedSourceMap(provider: *SourceProviderMap, contents: SourceContentHandling) ?*Mapping.ParsedSourceMap { - const bun_str = ZigSourceProvider__getSourceSlice(provider); - defer bun_str.deref(); - bun.assert(bun_str.tag == .WTFStringImpl); - const wtf = bun_str.value.WTFStringImpl; + fn findSourceMappingURL(comptime T: type, source: []const T, alloc: std.mem.Allocator) ?bun.JSC.ZigString.Slice { + const needle = comptime bun.strings.literal(T, "//# sourceMappingURL="); + const found = bun.strings.indexOfT(T, source, needle) orelse return null; + const end = std.mem.indexOfScalarPos(T, source, found + needle.len, '\n') orelse source.len; + const url = std.mem.trimRight(T, source[found + needle.len .. end], &.{ ' ', '\r' }); + return switch (T) { + u8 => bun.JSC.ZigString.Slice.initStatic(url), + u16 => bun.JSC.ZigString.Slice.init( + alloc, + bun.strings.toUTF8Alloc(alloc, url) catch bun.outOfMemory(), + ), + else => @compileError("Not Supported"), + }; + } - // TODO: do not use toUTF8() - const utf8 = wtf.toUTF8(bun.default_allocator); - defer utf8.deinit(); - const bytes = utf8.slice(); + /// The last two arguments to this specify loading hints + pub fn getSourceMap( + provider: *SourceProviderMap, + source_filename: []const u8, + load_hint: SourceMapLoadHint, + comptime result: ParseUrlResultType, + ) ?result.GetSourceMapType() { + var sfb = std.heap.stackFallback(65536, bun.default_allocator); + var arena = bun.ArenaAllocator.init(sfb.get()); + defer arena.deinit(); + + var load_from_separate_file = false; + + const parsed = parsed: { + // try to get an inline source map + if (load_hint != .is_external_map) try_inline: { + const source = ZigSourceProvider__getSourceSlice(provider); + defer source.deref(); + bun.assert(source.tag == .StaticZigString); + + const found_url = (if (source.is8Bit()) + findSourceMappingURL(u8, source.latin1(), arena.allocator()) + else + findSourceMappingURL(u16, source.utf16(), arena.allocator())) orelse + break :try_inline; + defer found_url.deinit(); + + var err_data: Mapping.ParseError = undefined; + break :parsed parseUrl( + bun.default_allocator, + arena.allocator(), + found_url.slice(), + result, + &err_data, + ) catch return null; + } - // TODO: use smarter technique - if (std.mem.indexOf(u8, bytes, "//# sourceMappingURL=")) |index| { - const end = std.mem.indexOfAnyPos(u8, bytes, index + "//# sourceMappingURL=".len, " \n\r") orelse bytes.len; - const url = bytes[index + "//# sourceMappingURL=".len .. end]; + // try to load a .map file + if (load_hint != .is_inline_map) try_external: { + var load_path_buf: bun.PathBuffer = undefined; + if (source_filename.len + 4 > load_path_buf.len) + break :try_external; + @memcpy(load_path_buf[0..source_filename.len], source_filename); + @memcpy(load_path_buf[source_filename.len..][0..4], ".map"); + + const data = switch (bun.sys.File.readFrom( + std.fs.cwd(), + load_path_buf[0 .. source_filename.len + 4], + arena.allocator(), + )) { + .err => break :try_external, + .result => |data| data, + }; - var sfb = std.heap.stackFallback(32768, bun.default_allocator); - var arena = bun.ArenaAllocator.init(sfb.get()); - defer arena.deinit(); + load_from_separate_file = true; - switch (parseUrl( - bun.default_allocator, - arena.allocator(), - url, - // only aquire sourcemap if the load request desires one - switch (contents) { - .no_source_contents => .mappings_only, - .source_contents => .all, - }, - )) { - .fail => { - return null; - }, - .success => |parsed| { - const ptr = bun.default_allocator.create(Mapping.ParsedSourceMap) catch bun.outOfMemory(); - ptr.* = parsed; - if (ptr.source_contents.state == .unloaded) - ptr.source_contents = Mapping.ParsedSourceMap.SourceContentPtr.fromProvider(provider); - return ptr; - }, + var err_data: Mapping.ParseError = undefined; + break :parsed parseJSON( + bun.default_allocator, + arena.allocator(), + data, + result, + &err_data, + ) catch return null; } - } - // TODO: look for file of same filename. + return null; + }; - return null; + if (result == .sources_only) return parsed; + + const ptr = bun.default_allocator.create(Mapping.ParsedSourceMap) catch bun.outOfMemory(); + ptr.* = parsed; + if (ptr.source_contents.state == .unloaded) { + ptr.source_contents = Mapping.ParsedSourceMap.SourceContentPtr.fromProvider(provider); + ptr.source_contents.load_from_separate_file = load_from_separate_file; + } + return ptr; + } + + pub fn toParsedSourceMap( + provider: *SourceProviderMap, + source_filename: []const u8, + source_handling: SourceContentHandling, + ) ?*Mapping.ParsedSourceMap { + return switch (source_handling) { + inline else => |tag| provider.getSourceMap( + source_filename, + .none, + switch (tag) { + .no_source_contents => .mappings_only, + .source_contents => .all, + }, + ), + }; } }; diff --git a/src/string_immutable.zig b/src/string_immutable.zig index cf43e1f43c358..f47d0f174b985 100644 --- a/src/string_immutable.zig +++ b/src/string_immutable.zig @@ -312,6 +312,11 @@ pub inline fn indexOf(self: string, str: string) ?usize { return @as(usize, @intCast(i)); } +pub fn indexOfT(comptime T: type, haystack: []const T, needle: []const T) ?usize { + if (T == u8) return indexOf(haystack, needle); + return std.mem.indexOf(T, haystack, needle); +} + pub fn split(self: string, delimiter: string) SplitIterator { return SplitIterator{ .buffer = self, From 1407575233f807e5c7db40ef9e80aa6bc2ad128c Mon Sep 17 00:00:00 2001 From: dave caruso Date: Mon, 13 May 2024 18:44:03 -0700 Subject: [PATCH 05/18] wow --- src/bun.js/bindings/ZigSourceProvider.cpp | 15 +++++++--- src/bun.js/bindings/ZigSourceProvider.h | 7 ++--- src/bun.js/javascript.zig | 36 ++++++++++++++++++++++- src/bun_js.zig | 27 +++++++++++++++++ 4 files changed, 76 insertions(+), 9 deletions(-) diff --git a/src/bun.js/bindings/ZigSourceProvider.cpp b/src/bun.js/bindings/ZigSourceProvider.cpp index 7cca068a49dec..48ddef08ea6bb 100644 --- a/src/bun.js/bindings/ZigSourceProvider.cpp +++ b/src/bun.js/bindings/ZigSourceProvider.cpp @@ -66,7 +66,8 @@ JSC::SourceID sourceIDForSourceURL(const WTF::String& sourceURL) } extern "C" bool BunTest__shouldGenerateCodeCoverage(BunString sourceURL); -extern "C" void Bun__addSourceProviderSourceMap(void* bun_vm, void* opaque_source_provider, BunString* specifier); +extern "C" void Bun__addSourceProviderSourceMap(void* bun_vm, SourceProvider* opaque_source_provider, BunString* specifier); +extern "C" void Bun__removeSourceProviderSourceMap(void* bun_vm, SourceProvider* opaque_source_provider, BunString* specifier); Ref SourceProvider::create( Zig::GlobalObject* globalObject, @@ -110,6 +111,13 @@ Ref SourceProvider::create( return provider; } +SourceProvider::~SourceProvider() { + if(m_resolvedSource.already_bundled) { + BunString str = Bun::toString(sourceURL()); + Bun__removeSourceProviderSourceMap(m_globalObject->bunVM(), this, &str); + } +} + unsigned SourceProvider::hash() const { if (m_hash) { @@ -147,9 +155,8 @@ void SourceProvider::cacheBytecode(const BytecodeCacheGenerator& generator) if (update) m_cachedBytecode->addGlobalUpdate(*update); } -SourceProvider::~SourceProvider() -{ -} + + void SourceProvider::commitCachedBytecode() { // if (!m_resolvedSource.bytecodecache_fd || !m_cachedBytecode || !m_cachedBytecode->hasUpdates()) diff --git a/src/bun.js/bindings/ZigSourceProvider.h b/src/bun.js/bindings/ZigSourceProvider.h index 186846c4d6fc9..36ca682bc8b33 100644 --- a/src/bun.js/bindings/ZigSourceProvider.h +++ b/src/bun.js/bindings/ZigSourceProvider.h @@ -46,13 +46,10 @@ class SourceProvider final : public JSC::SourceProvider { ~SourceProvider(); unsigned hash() const override; StringView source() const override { return StringView(m_source.get()); } + RefPtr cachedBytecode() { - // if (m_resolvedSource.bytecodecache_fd == 0) { return nullptr; - // } - - // return m_cachedBytecode; }; void updateCache(const UnlinkedFunctionExecutable* executable, const SourceCode&, @@ -71,11 +68,13 @@ class SourceProvider final : public JSC::SourceProvider { const SourceOrigin& sourceOrigin, WTF::String&& sourceURL, const TextPosition& startPosition, JSC::SourceProviderSourceType sourceType) : Base(sourceOrigin, WTFMove(sourceURL), String(), taintedness, startPosition, sourceType) + , m_globalObject(globalObject) , m_source(sourceImpl) { m_resolvedSource = resolvedSource; } + Zig::GlobalObject* m_globalObject; RefPtr m_cachedBytecode; Ref m_source; unsigned m_hash = 0; diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index cde7b73ce76ac..f6ab16bd14b30 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -182,11 +182,26 @@ pub const SavedSourceMap = struct { SourceProviderMap, }); + pub const MissingSourceMapNoteInfo = struct { + pub var storage: bun.PathBuffer = undefined; + pub var path: ?[]const u8 = ""; + }; + pub fn putZigSourceProvider(this: *SavedSourceMap, opaque_source_provider: *anyopaque, path: []const u8) void { const source_provider: *SourceProviderMap = @ptrCast(opaque_source_provider); this.putValue(path, Value.init(source_provider)) catch bun.outOfMemory(); } + pub fn removeZigSourceProvider(this: *SavedSourceMap, opaque_source_provider: *anyopaque, path: []const u8) void { + const entry = this.map.getEntry(bun.hash(path)) orelse return; + const old_value = Value.from(entry.value_ptr.*); + if (old_value.get(SourceProviderMap)) |prov| { + if (@intFromPtr(prov) == @intFromPtr(opaque_source_provider)) { + this.map.removeByPtr(entry.key_ptr); + } + } + } + pub const HashTable = std.HashMap(u64, *anyopaque, IdentityContext(u64), 80); pub fn onSourceMapChunk(this: *SavedSourceMap, chunk: SourceMap.Chunk, source: logger.Source) anyerror!void { @@ -265,6 +280,12 @@ pub const SavedSourceMap = struct { } else { // does not have a valid source map. let's not try again _ = this.map.remove(hash); + + // Store path for a user note. + const storage = MissingSourceMapNoteInfo.storage[0..path.len]; + @memcpy(storage, path); + MissingSourceMapNoteInfo.path = storage; + return null; } }, @@ -2779,7 +2800,13 @@ pub const VirtualMachine = struct { } } - pub fn remapZigException(this: *VirtualMachine, exception: *ZigException, error_instance: JSValue, exception_list: ?*ExceptionList, must_reset_parser_arena_later: *bool) void { + pub fn remapZigException( + this: *VirtualMachine, + exception: *ZigException, + error_instance: JSValue, + exception_list: ?*ExceptionList, + must_reset_parser_arena_later: *bool, + ) void { error_instance.toZigException(this.global, exception); // defer this so that it copies correctly @@ -3899,4 +3926,11 @@ export fn Bun__addSourceProviderSourceMap(vm: *VirtualMachine, opaque_source_pro vm.source_mappings.putZigSourceProvider(opaque_source_provider, slice.slice()); } +export fn Bun__removeSourceProviderSourceMap(vm: *VirtualMachine, opaque_source_provider: *anyopaque, specifier: *bun.String) void { + var sfb = std.heap.stackFallback(4096, bun.default_allocator); + const slice = specifier.toUTF8(sfb.get()); + defer slice.deinit(); + vm.source_mappings.removeZigSourceProvider(opaque_source_provider, slice.slice()); +} + pub export var isBunTest: bool = false; diff --git a/src/bun_js.zig b/src/bun_js.zig index 8ccb36c6be8ed..e15e24c1a44b5 100644 --- a/src/bun_js.zig +++ b/src/bun_js.zig @@ -284,6 +284,15 @@ pub const Run = struct { vm.onExit(); if (run.any_unhandled) { + const MissingSourceMapNoteInfo = bun.JSC.SavedSourceMap.MissingSourceMapNoteInfo; + if (MissingSourceMapNoteInfo.path) |note| { + Output.note( + "missing sourcemaps for {s}", + .{note}, + ); + Output.note("consider bundling with '--sourcemap' to get an unminified traces", .{}); + } + Output.prettyErrorln( "\n{s}", .{Global.unhandled_error_bun_version_string}, @@ -315,6 +324,15 @@ pub const Run = struct { vm.exit_handler.exit_code = 1; vm.onExit(); if (run.any_unhandled) { + const MissingSourceMapNoteInfo = bun.JSC.SavedSourceMap.MissingSourceMapNoteInfo; + if (MissingSourceMapNoteInfo.path) |note| { + Output.note( + "missing sourcemaps for {s}", + .{note}, + ); + Output.note("consider bundling with '--sourcemap' to get an unminified traces", .{}); + } + Output.prettyErrorln( "\n{s}", .{Global.unhandled_error_bun_version_string}, @@ -417,6 +435,15 @@ pub const Run = struct { if (this.any_unhandled and this.vm.exit_handler.exit_code == 0) { this.vm.exit_handler.exit_code = 1; + const MissingSourceMapNoteInfo = bun.JSC.SavedSourceMap.MissingSourceMapNoteInfo; + if (MissingSourceMapNoteInfo.path) |note| { + Output.note( + "missing sourcemaps for {s}", + .{note}, + ); + Output.note("consider bundling with '--sourcemap' to get an unminified traces", .{}); + } + Output.prettyErrorln( "\n{s}", .{Global.unhandled_error_bun_version_string}, From 95468bc3d9de648c76da4cc1dfd42a73d16f779a Mon Sep 17 00:00:00 2001 From: dave caruso Date: Mon, 13 May 2024 18:47:35 -0700 Subject: [PATCH 06/18] woops --- src/bun.js/javascript.zig | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index f6ab16bd14b30..f966ed34b79f5 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -2895,7 +2895,7 @@ pub const VirtualMachine = struct { }, .source_index = 0, }, - // this pointer is not read if `top.remapped` + // undefined is fine, because this pointer is never read if `top.remapped == true` .source_map = undefined, } else @@ -2909,9 +2909,11 @@ pub const VirtualMachine = struct { if (maybe_lookup) |lookup| { const mapping = lookup.mapping; - if (lookup.displaySourceURLIfNeeded(top_source_url.slice())) |src| { - top.source_url.deref(); - top.source_url = src; + if (!top.remapped) { + if (lookup.displaySourceURLIfNeeded(top_source_url.slice())) |src| { + top.source_url.deref(); + top.source_url = src; + } } const code = code: { From 5b9374130e5347239a193e8e7f31021e1d9fc509 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Tue, 14 May 2024 17:46:10 -0700 Subject: [PATCH 07/18] add some tests --- src/bun.js/bindings/ZigSourceProvider.h | 2 - src/bun.js/bindings/bindings.zig | 10 -- src/bun.js/javascript.zig | 29 +++++- src/bun_js.zig | 27 +----- src/sourcemap/sourcemap.zig | 6 +- test/bundler/bundler_bun.test.ts | 57 +++++++++++ test/bundler/expectBundled.ts | 16 ++- test/cli/hot/hot.test.ts | 124 ++++++++++++++++++++++++ 8 files changed, 227 insertions(+), 44 deletions(-) diff --git a/src/bun.js/bindings/ZigSourceProvider.h b/src/bun.js/bindings/ZigSourceProvider.h index 36ca682bc8b33..d8ccaa69ee528 100644 --- a/src/bun.js/bindings/ZigSourceProvider.h +++ b/src/bun.js/bindings/ZigSourceProvider.h @@ -1,7 +1,5 @@ #include "headers.h" #include "root.h" -#include <_types/_uint64_t.h> -#include #pragma once diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index 6cdd6059e27bf..fe2613dfd7ea0 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -421,16 +421,6 @@ pub const ZigString = extern struct { }; } - /// Creates a `Slice` from bytes without an allocator, aka a string view. - /// This slice is only valid for the lifetime of the given input, and - /// will never be freed by this slice. - pub fn initStatic(input: []const u8) Slice { - return .{ - .ptr = input.ptr, - .len = @as(u32, @truncate(input.len)), - }; - } - pub fn toZigString(this: Slice) ZigString { if (this.isAllocated()) return ZigString.initUTF8(this.ptr[0..this.len]); diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index f966ed34b79f5..e83b38afaaed0 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -184,7 +184,17 @@ pub const SavedSourceMap = struct { pub const MissingSourceMapNoteInfo = struct { pub var storage: bun.PathBuffer = undefined; - pub var path: ?[]const u8 = ""; + pub var path: ?[]const u8 = null; + + pub fn print() void { + if (path) |note| { + Output.note( + "missing sourcemaps for {s}", + .{note}, + ); + Output.note("consider bundling with '--sourcemap' to get an unminified traces", .{}); + } + } }; pub fn putZigSourceProvider(this: *SavedSourceMap, opaque_source_provider: *anyopaque, path: []const u8) void { @@ -199,6 +209,17 @@ pub const SavedSourceMap = struct { if (@intFromPtr(prov) == @intFromPtr(opaque_source_provider)) { this.map.removeByPtr(entry.key_ptr); } + } else if (old_value.get(ParsedSourceMap)) |map| { + if (map.source_contents.state == .unloaded and + @intFromPtr(map.source_contents.provider()) == @intFromPtr(opaque_source_provider)) + { + this.map.removeByPtr(entry.key_ptr); + } else { + // not possible to know for sure this is the same map. also not worth + // knowing, because what will happen is one of: + // - a hot reload happens and replaces this entry. + // - the process exits, freeing the memory. + } } } @@ -222,6 +243,8 @@ pub const SavedSourceMap = struct { } else if (value.get(SavedMappings)) |saved_mappings| { var saved = SavedMappings{ .data = @as([*]u8, @ptrCast(saved_mappings)) }; saved.deinit(); + } else if (value.get(SourceProviderMap)) |provider| { + _ = provider; // do nothing, we did not hold a ref to ZigSourceProvider } } @@ -248,7 +271,7 @@ pub const SavedSourceMap = struct { var saved = SavedMappings{ .data = @as([*]u8, @ptrCast(saved_mappings)) }; saved.deinit(); } else if (old_value.get(SourceProviderMap)) |provider| { - _ = provider; // do nothing + _ = provider; // do nothing, we did not hold a ref to ZigSourceProvider } } entry.value_ptr.* = value.ptr(); @@ -2919,7 +2942,7 @@ pub const VirtualMachine = struct { const code = code: { if (!top.remapped and lookup.source_map.isExternal()) { if (lookup.getSourceCode(top_source_url.slice())) |src| { - break :code ZigString.Slice.initStatic(src); + break :code ZigString.Slice.fromUTF8NeverFree(src); } } diff --git a/src/bun_js.zig b/src/bun_js.zig index e15e24c1a44b5..1147d08d72248 100644 --- a/src/bun_js.zig +++ b/src/bun_js.zig @@ -284,14 +284,7 @@ pub const Run = struct { vm.onExit(); if (run.any_unhandled) { - const MissingSourceMapNoteInfo = bun.JSC.SavedSourceMap.MissingSourceMapNoteInfo; - if (MissingSourceMapNoteInfo.path) |note| { - Output.note( - "missing sourcemaps for {s}", - .{note}, - ); - Output.note("consider bundling with '--sourcemap' to get an unminified traces", .{}); - } + bun.JSC.SavedSourceMap.MissingSourceMapNoteInfo.print(); Output.prettyErrorln( "\n{s}", @@ -324,14 +317,7 @@ pub const Run = struct { vm.exit_handler.exit_code = 1; vm.onExit(); if (run.any_unhandled) { - const MissingSourceMapNoteInfo = bun.JSC.SavedSourceMap.MissingSourceMapNoteInfo; - if (MissingSourceMapNoteInfo.path) |note| { - Output.note( - "missing sourcemaps for {s}", - .{note}, - ); - Output.note("consider bundling with '--sourcemap' to get an unminified traces", .{}); - } + bun.JSC.SavedSourceMap.MissingSourceMapNoteInfo.print(); Output.prettyErrorln( "\n{s}", @@ -435,14 +421,7 @@ pub const Run = struct { if (this.any_unhandled and this.vm.exit_handler.exit_code == 0) { this.vm.exit_handler.exit_code = 1; - const MissingSourceMapNoteInfo = bun.JSC.SavedSourceMap.MissingSourceMapNoteInfo; - if (MissingSourceMapNoteInfo.path) |note| { - Output.note( - "missing sourcemaps for {s}", - .{note}, - ); - Output.note("consider bundling with '--sourcemap' to get an unminified traces", .{}); - } + bun.JSC.SavedSourceMap.MissingSourceMapNoteInfo.print(); Output.prettyErrorln( "\n{s}", diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig index 535dc3e39aac0..3b620434cb4cc 100644 --- a/src/sourcemap/sourcemap.zig +++ b/src/sourcemap/sourcemap.zig @@ -640,12 +640,12 @@ pub const Mapping = struct { return .{ .state = .loaded, .data = @intCast(@intFromPtr(p)) }; } - fn provider(sc: SourceContentPtr) *SourceProviderMap { + pub fn provider(sc: SourceContentPtr) *SourceProviderMap { bun.assert(sc.state == .unloaded); return @ptrFromInt(sc.data); } - fn sources(sc: SourceContentPtr) [*]?[]const u8 { + pub fn sources(sc: SourceContentPtr) [*]?[]const u8 { bun.assert(sc.state == .loaded); return @ptrFromInt(sc.data); } @@ -713,7 +713,7 @@ pub const SourceProviderMap = opaque { const end = std.mem.indexOfScalarPos(T, source, found + needle.len, '\n') orelse source.len; const url = std.mem.trimRight(T, source[found + needle.len .. end], &.{ ' ', '\r' }); return switch (T) { - u8 => bun.JSC.ZigString.Slice.initStatic(url), + u8 => bun.JSC.ZigString.Slice.fromUTF8NeverFree(url), u16 => bun.JSC.ZigString.Slice.init( alloc, bun.strings.toUTF8Alloc(alloc, url) catch bun.outOfMemory(), diff --git a/test/bundler/bundler_bun.test.ts b/test/bundler/bundler_bun.test.ts index 5f2413df54936..1aff814d39cf6 100644 --- a/test/bundler/bundler_bun.test.ts +++ b/test/bundler/bundler_bun.test.ts @@ -42,4 +42,61 @@ describe("bundler", () => { }, run: { stdout: "Hello, world!", setCwd: true }, }); + itBundled("bun/TargetBunNoSourcemapMessage", { + target: "bun", + files: { + "/entry.ts": /* js */ ` + // this file has comments and weird whitespace, intentionally + // to make it obvious if sourcemaps were generated and mapped properly + if (true) code(); + function code() { + // hello world + throw new + Error("Hello World"); + } + `, + }, + run: { + exitCode: 1, + verify({ stdio }) { + expect(stdio).toInclude("\nnote: missing sourcemaps for "); + expect(stdio).toInclude("\nnote: consider bundling with '--sourcemap' to get an unminified traces\n"); + }, + }, + }); + itBundled("bun/TargetBunSourcemapInline", { + target: "bun", + files: { + "/entry.ts": /* js */ ` + // this file has comments and weird whitespace, intentionally + // to make it obvious if sourcemaps were generated and mapped properly + if (true) code(); + function code() { + // hello world + throw new + Error("Hello World"); + } + `, + }, + sourceMap: "inline", + run: { + exitCode: 1, + verify({ stdio }) { + assert( + stdio.startsWith( + stdio, + `1 | // this file has comments and weird whitespace, intentionally +2 | // to make it obvious if sourcemaps were generated and mapped properly +3 | if (true) code(); +4 | function code() { +5 | // hello world +6 | throw new + ^ +error: Hello World`, + ), + ); + expect(stdio).toInclude("/entry.ts:6:19"); + }, + }, + }); }); diff --git a/test/bundler/expectBundled.ts b/test/bundler/expectBundled.ts index a28dbcc3f9eec..c926857fa8184 100644 --- a/test/bundler/expectBundled.ts +++ b/test/bundler/expectBundled.ts @@ -307,6 +307,10 @@ export interface BundlerTestRunOptions { runtime?: "bun" | "node"; setCwd?: boolean; + /** Expect a certain non-zero exit code */ + exitCode?: number; + /** Run a function with stdout and stderr. Use expect to assert exact outputs */ + validate?: (ctx: { stdout: string; stderr: string }) => void; } /** given when you do itBundled('id', (this object) => BundlerTestInput) */ @@ -1259,7 +1263,7 @@ for (const [key, blob] of build.outputs) { throw new Error(prefix + "run.file is required when there is more than one entrypoint."); } - const { success, stdout, stderr } = Bun.spawnSync({ + const { success, stdout, stderr, exitCode } = Bun.spawnSync({ cmd: [ ...(compile ? [] : [(run.runtime ?? "bun") === "bun" ? bunExe() : "node"]), ...(run.bunArgs ?? []), @@ -1326,7 +1330,15 @@ for (const [key, blob] of build.outputs) { } } } else if (!success) { - throw new Error(prefix + "Runtime failed\n" + stdout!.toUnixString() + "\n" + stderr!.toUnixString()); + if (run.exitCode) { + expect(exitCode).toBe(run.exitCode); + } else { + throw new Error(prefix + "Runtime failed\n" + stdout!.toUnixString() + "\n" + stderr!.toUnixString()); + } + } + + if (run.validate) { + run.validate({ stderr, stdout }); } if (run.stdout !== undefined) { diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index 487ea2da8a60a..efb2916b30267 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -320,3 +320,127 @@ it("should hot reload when a file is renamed() into place", async () => { runner?.kill?.(9); } }); + +it("should work with sourcemap generation", async () => { + writeFileSync( + hotRunnerRoot, + `// source content +// +// +throw new Error('0');`, + ); + await using runner = spawn({ + cmd: [bunExe(), "--hot", "run", hotRunnerRoot], + env: bunEnv, + cwd, + stdout: "ignore", + stderr: "pipe", + stdin: "ignore", + }); + let reloadCounter = 0; + function onReload() { + writeFileSync( + hotRunnerRoot, + `// source content +// etc etc +// etc etc +${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, + ); + } + let str = ""; + for await (const chunk of runner.stderr) { + str += new TextDecoder().decode(chunk); + var any = false; + if (!/error: .*[0-9]\n.*?\n/g.test(str)) continue; + + let it = str.split("\n"); + let line; + while ((line = it.shift())) { + if (!line.includes("error")) continue; + reloadCounter++; + str = ""; + + if (reloadCounter === 100) { + runner.kill(); + break; + } + + expect(line).toContain(`error: ${reloadCounter - 1}`); + let next = it.shift()!; + const col = next.match(/\s*at.*?:4:(\d+)$/)![1]; + expect(Number(col)).toBe(1 + "throw ".length + (reloadCounter - 1) * 2); + any = true; + } + + if (any) await onReload(); + } + expect(reloadCounter).toBe(100); +}); + +it("should work with sourcemap loading", async () => { + let bundleIn = join(cwd, "bundle_in.ts"); + rmSync(hotRunnerRoot); + writeFileSync( + bundleIn, + `// source content +// +// +throw new Error('0');`, + ); + await using bundler = spawn({ + cmd: [bunExe(), "build", "--watch", bundleIn, "--target=bun", "--sourcemap", "--outfile", hotRunnerRoot], + env: bunEnv, + cwd, + stdout: "inherit", + stderr: "inherit", + stdin: "ignore", + }); + await using runner = spawn({ + cmd: [bunExe(), "--hot", "run", hotRunnerRoot], + env: bunEnv, + cwd, + stdout: "ignore", + stderr: "pipe", + stdin: "ignore", + }); + let reloadCounter = 0; + function onReload() { + writeFileSync( + bundleIn, + `// source content +// etc etc +// etc etc +${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, + ); + } + let str = ""; + for await (const chunk of runner.stderr) { + str += new TextDecoder().decode(chunk); + var any = false; + if (!/error: .*[0-9]\n.*?\n/g.test(str)) continue; + + let it = str.split("\n"); + let line; + while ((line = it.shift())) { + if (!line.includes("error")) continue; + reloadCounter++; + str = ""; + + if (reloadCounter === 100) { + runner.kill(); + break; + } + + expect(line).toContain(`error: ${reloadCounter - 1}`); + let next = it.shift()!; + expect(next).toInclude("bundle_in.ts"); + const col = next.match(/\s*at.*?:4:(\d+)$/)![1]; + expect(Number(col)).toBe(1 + "throw ".length + (reloadCounter - 1) * 2); + any = true; + } + + if (any) await onReload(); + } + expect(reloadCounter).toBe(100); + bundler.kill(); +}); From c871a4dc52d1d688dc483f948eb2a24af203ad60 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Tue, 14 May 2024 18:29:59 -0700 Subject: [PATCH 08/18] some stuff --- src/bun.js/bindings/BunString.cpp | 5 +---- src/string.zig | 12 ++++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/bun.js/bindings/BunString.cpp b/src/bun.js/bindings/BunString.cpp index 097401cd21542..4949985b18999 100644 --- a/src/bun.js/bindings/BunString.cpp +++ b/src/bun.js/bindings/BunString.cpp @@ -197,9 +197,8 @@ BunString toStringRef(WTF::StringImpl* wtfString) } BunString toStringView(StringView view) { - view.is8Bit(); return { - BunStringTag::StaticZigString, + BunStringTag::ZigString, { .zig = { .ptr = (const LChar*)(view.is8Bit() ? (size_t)view.rawCharacters() @@ -210,8 +209,6 @@ BunString toStringView(StringView view) { }; } - - } extern "C" JSC::EncodedJSValue BunString__toJS(JSC::JSGlobalObject* globalObject, const BunString* bunString) diff --git a/src/string.zig b/src/string.zig index c3cd95d4287e1..c5f2a471fd0a2 100644 --- a/src/string.zig +++ b/src/string.zig @@ -266,10 +266,22 @@ pub const StringImplAllocator = struct { }; pub const Tag = enum(u8) { + /// String is not valid. Observed on some failed operations. + /// To prevent crashes, this value acts similarly to .Empty (such as length = 0) Dead = 0, + /// String is backed by a WTF::StringImpl from JavaScriptCore. + /// Can be in either `latin1` or `utf16le` encodings. WTFStringImpl = 1, + /// Memory is backed by an allocation in Bun's Zig codebase. When converted to JSValue, + /// it has to be cloned into a WTF::String. This string should not be cloned in any + /// other circumstance. + /// Can be in either `utf8` or `utf16le` encodings. ZigString = 2, + /// Static memory that is guarenteed to never be freed. When converted to WTF::String, + /// the memory is not cloned, but instead referenced with WTF::ExternalStringImpl. + /// Can be in either `utf8` or `utf16le` encodings. StaticZigString = 3, + /// String is "" Empty = 4, }; From 56ade8bfcb9b69a4ab844f81f968148215d17f4b Mon Sep 17 00:00:00 2001 From: dave caruso Date: Wed, 15 May 2024 00:44:16 -0700 Subject: [PATCH 09/18] lol --- src/bun.js/bindings/BunString.cpp | 9 +-- src/bun.js/javascript.zig | 8 +- src/js_lexer.zig | 2 +- src/sourcemap/sourcemap.zig | 17 ++--- src/string.zig | 6 +- test/bundler/bundler_bun.test.ts | 18 ++--- test/bundler/expectBundled.ts | 24 +++--- test/cli/hot/hot.test.ts | 120 ++++++++++++++++++++++++++++-- 8 files changed, 155 insertions(+), 49 deletions(-) diff --git a/src/bun.js/bindings/BunString.cpp b/src/bun.js/bindings/BunString.cpp index 4949985b18999..bf8013ee7309a 100644 --- a/src/bun.js/bindings/BunString.cpp +++ b/src/bun.js/bindings/BunString.cpp @@ -1,4 +1,5 @@ +#include "helpers.h" #include "root.h" #include "headers-handwritten.h" #include @@ -199,13 +200,7 @@ BunString toStringRef(WTF::StringImpl* wtfString) BunString toStringView(StringView view) { return { BunStringTag::ZigString, - { .zig = { - .ptr = (const LChar*)(view.is8Bit() - ? (size_t)view.rawCharacters() - : ((size_t)view.rawCharacters() + (static_cast(1) << 63)) - ), - .len = view.length() - } } + { .zig = toZigString(view) } }; } diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index e83b38afaaed0..815d4f81151b9 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -203,6 +203,9 @@ pub const SavedSourceMap = struct { } pub fn removeZigSourceProvider(this: *SavedSourceMap, opaque_source_provider: *anyopaque, path: []const u8) void { + this.mutex.lock(); + defer this.mutex.unlock(); + const entry = this.map.getEntry(bun.hash(path)) orelse return; const old_value = Value.from(entry.value_ptr.*); if (old_value.get(SourceProviderMap)) |prov| { @@ -213,6 +216,7 @@ pub const SavedSourceMap = struct { if (map.source_contents.state == .unloaded and @intFromPtr(map.source_contents.provider()) == @intFromPtr(opaque_source_provider)) { + map.deinit(default_allocator); this.map.removeByPtr(entry.key_ptr); } else { // not possible to know for sure this is the same map. also not worth @@ -234,6 +238,8 @@ pub const SavedSourceMap = struct { pub fn deinit(this: *SavedSourceMap) void { { this.mutex.lock(); + defer this.mutex.unlock(); + var iter = this.map.valueIterator(); while (iter.next()) |val| { var value = Value.from(val.*); @@ -247,8 +253,6 @@ pub const SavedSourceMap = struct { _ = provider; // do nothing, we did not hold a ref to ZigSourceProvider } } - - this.mutex.unlock(); } this.map.deinit(); diff --git a/src/js_lexer.zig b/src/js_lexer.zig index 96e396c8de7a9..50142410481a5 100644 --- a/src/js_lexer.zig +++ b/src/js_lexer.zig @@ -77,7 +77,7 @@ pub const JSONOptions = struct { always_decode_escape_sequences: bool = false, }; -pub fn decodeUTF8(bytes: string, allocator: std.mem.Allocator) ![]const u16 { +pub fn decodeStringLiteralEscapeSequencesToUTF16(bytes: string, allocator: std.mem.Allocator) ![]const u16 { var log = logger.Log.init(allocator); defer log.deinit(); const source = logger.Source.initEmptyFile(""); diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig index 3b620434cb4cc..e74e32ad397cf 100644 --- a/src/sourcemap/sourcemap.zig +++ b/src/sourcemap/sourcemap.zig @@ -122,7 +122,7 @@ pub fn parseJSON( comptime result: ParseUrlResultType, err: *Mapping.ParseError, ) !result.ParseType() { - const json_src = bun.logger.Source.initPathString("", source); + const json_src = bun.logger.Source.initPathString("sourcemap.json", source); var log = bun.logger.Log.init(arena); defer log.deinit(); @@ -188,10 +188,6 @@ pub fn parseJSON( if (result != .sources_only) for (sources_paths.items.slice()) |item| { if (item.data != .e_string) { - for (source_paths_slice[0..i]) |slice| alloc.free(slice); - alloc.free(source_paths_slice); - if (result != .mappings_only) alloc.free(source_contents_slice); - return err.fail(error.InvalidSourceMap, .{ .msg = "Expected string in sources", .value = 0, @@ -202,10 +198,6 @@ pub fn parseJSON( const utf16_decode = bun.js_lexer.decodeUTF8(item.data.e_string.string(arena) catch bun.outOfMemory(), arena) catch bun.outOfMemory(); defer arena.free(utf16_decode); source_paths_slice[i] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch { - for (source_paths_slice[0..i]) |slice| alloc.free(slice); - alloc.free(source_paths_slice); - if (result != .mappings_only) alloc.free(source_contents_slice); - return err.fail(error.InvalidSourceMap, .{ .msg = "Expected string in sources", .value = 0, @@ -645,7 +637,7 @@ pub const Mapping = struct { return @ptrFromInt(sc.data); } - pub fn sources(sc: SourceContentPtr) [*]?[]const u8 { + fn sources(sc: SourceContentPtr) [*]?[]const u8 { bun.assert(sc.state == .loaded); return @ptrFromInt(sc.data); } @@ -663,7 +655,8 @@ pub const Mapping = struct { this.mappings.deinit(allocator); if (this.external_source_names.len > 0) { - for (this.external_source_names) |name| allocator.free(name); + for (this.external_source_names) |name| + allocator.free(name); allocator.free(this.external_source_names); } @@ -740,7 +733,7 @@ pub const SourceProviderMap = opaque { if (load_hint != .is_external_map) try_inline: { const source = ZigSourceProvider__getSourceSlice(provider); defer source.deref(); - bun.assert(source.tag == .StaticZigString); + bun.assert(source.tag == .ZigString); const found_url = (if (source.is8Bit()) findSourceMappingURL(u8, source.latin1(), arena.allocator()) diff --git a/src/string.zig b/src/string.zig index c5f2a471fd0a2..d6814ca0f5219 100644 --- a/src/string.zig +++ b/src/string.zig @@ -272,9 +272,9 @@ pub const Tag = enum(u8) { /// String is backed by a WTF::StringImpl from JavaScriptCore. /// Can be in either `latin1` or `utf16le` encodings. WTFStringImpl = 1, - /// Memory is backed by an allocation in Bun's Zig codebase. When converted to JSValue, - /// it has to be cloned into a WTF::String. This string should not be cloned in any - /// other circumstance. + /// Memory has an unknown owner, likely in Bun's Zig codebase. If `isGloballyAllocated` + /// is set, then it is owned by mimalloc. When converted to JSValue it has to be cloned + /// into a WTF::String. /// Can be in either `utf8` or `utf16le` encodings. ZigString = 2, /// Static memory that is guarenteed to never be freed. When converted to WTF::String, diff --git a/test/bundler/bundler_bun.test.ts b/test/bundler/bundler_bun.test.ts index 1aff814d39cf6..38f9f810f80ac 100644 --- a/test/bundler/bundler_bun.test.ts +++ b/test/bundler/bundler_bun.test.ts @@ -2,6 +2,7 @@ import assert from "assert"; import dedent from "dedent"; import { ESBUILD, itBundled, testForFile } from "./expectBundled"; import { Database } from "bun:sqlite"; +import { isWindows } from "harness"; var { describe, test, expect } = testForFile(import.meta.path); describe("bundler", () => { @@ -57,10 +58,10 @@ describe("bundler", () => { `, }, run: { - exitCode: 1, - verify({ stdio }) { - expect(stdio).toInclude("\nnote: missing sourcemaps for "); - expect(stdio).toInclude("\nnote: consider bundling with '--sourcemap' to get an unminified traces\n"); + exitCode: isWindows ? 3 : 1, + validate({ stderr }) { + expect(stderr).toInclude("\nnote: missing sourcemaps for "); + expect(stderr).toInclude("\nnote: consider bundling with '--sourcemap' to get an unminified traces\n"); }, }, }); @@ -80,11 +81,10 @@ describe("bundler", () => { }, sourceMap: "inline", run: { - exitCode: 1, - verify({ stdio }) { + exitCode: isWindows ? 3 : 1, + validate({ stderr }) { assert( - stdio.startsWith( - stdio, + stderr.startsWith( `1 | // this file has comments and weird whitespace, intentionally 2 | // to make it obvious if sourcemaps were generated and mapped properly 3 | if (true) code(); @@ -95,7 +95,7 @@ describe("bundler", () => { error: Hello World`, ), ); - expect(stdio).toInclude("/entry.ts:6:19"); + expect(stderr).toInclude("entry.ts:6:19"); }, }, }); diff --git a/test/bundler/expectBundled.ts b/test/bundler/expectBundled.ts index c926857fa8184..f405addbb39a1 100644 --- a/test/bundler/expectBundled.ts +++ b/test/bundler/expectBundled.ts @@ -1263,13 +1263,15 @@ for (const [key, blob] of build.outputs) { throw new Error(prefix + "run.file is required when there is more than one entrypoint."); } - const { success, stdout, stderr, exitCode } = Bun.spawnSync({ - cmd: [ - ...(compile ? [] : [(run.runtime ?? "bun") === "bun" ? bunExe() : "node"]), - ...(run.bunArgs ?? []), - file, - ...(run.args ?? []), - ] as [string, ...string[]], + const args = [ + ...(compile ? [] : [(run.runtime ?? "bun") === "bun" ? bunExe() : "node"]), + ...(run.bunArgs ?? []), + file, + ...(run.args ?? []), + ] as [string, ...string[]]; + + const { success, stdout, stderr, exitCode, signalCode } = Bun.spawnSync({ + cmd: args, env: { ...bunEnv, FORCE_COLOR: "0", @@ -1279,6 +1281,10 @@ for (const [key, blob] of build.outputs) { cwd: run.setCwd ? root : undefined, }); + if (signalCode === "SIGTRAP") { + throw new Error(prefix + "Runtime failed\n" + stdout!.toUnixString() + "\n" + stderr!.toUnixString()); + } + if (run.error) { if (success) { throw new Error( @@ -1331,14 +1337,14 @@ for (const [key, blob] of build.outputs) { } } else if (!success) { if (run.exitCode) { - expect(exitCode).toBe(run.exitCode); + expect([exitCode, signalCode]).toEqual([run.exitCode, undefined]); } else { throw new Error(prefix + "Runtime failed\n" + stdout!.toUnixString() + "\n" + stderr!.toUnixString()); } } if (run.validate) { - run.validate({ stderr, stdout }); + run.validate({ stderr: stderr.toUnixString(), stdout: stdout.toUnixString() }); } if (run.stdout !== undefined) { diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index efb2916b30267..f19e2f8b76c87 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -321,16 +321,16 @@ it("should hot reload when a file is renamed() into place", async () => { } }); +const comment_spam = ("//" + "A".repeat(2000) + "\n").repeat(1000); it("should work with sourcemap generation", async () => { writeFileSync( hotRunnerRoot, `// source content -// -// +${comment_spam} throw new Error('0');`, ); await using runner = spawn({ - cmd: [bunExe(), "--hot", "run", hotRunnerRoot], + cmd: [bunExe(), "--smol", "--hot", "run", hotRunnerRoot], env: bunEnv, cwd, stdout: "ignore", @@ -342,8 +342,7 @@ throw new Error('0');`, writeFileSync( hotRunnerRoot, `// source content -// etc etc -// etc etc +${comment_spam} ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, ); } @@ -367,13 +366,19 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, expect(line).toContain(`error: ${reloadCounter - 1}`); let next = it.shift()!; - const col = next.match(/\s*at.*?:4:(\d+)$/)![1]; + if (!next) throw new Error(line); + const match = next.match(/\s*at.*?:1003:(\d+)$/); + if (!match) throw new Error("invalid string: " + next); + const col = match[1]; expect(Number(col)).toBe(1 + "throw ".length + (reloadCounter - 1) * 2); any = true; } if (any) await onReload(); } + await runner.exited; + const rss = runner.resourceUsage().maxRSS; + expect(rss).toBeLessThan(100_000_000); expect(reloadCounter).toBe(100); }); @@ -444,3 +449,106 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, expect(reloadCounter).toBe(100); bundler.kill(); }); + +const long_comment = "BBBB".repeat(100000); + +it("should work with sourcemap loading with large files", async () => { + let bundleIn = join(cwd, "bundle_in.ts"); + rmSync(hotRunnerRoot); + writeFileSync( + bundleIn, + `// ${long_comment} +// +console.error("RSS: %s", process.memoryUsage().rss); +throw new Error('0');`, + ); + await using bundler = spawn({ + cmd: [ + // + bunExe(), + "build", + "--watch", + bundleIn, + "--target=bun", + "--sourcemap", + "--outfile", + hotRunnerRoot, + ], + env: bunEnv, + cwd, + stdout: "ignore", + stderr: "ignore", + stdin: "ignore", + }); + await using runner = spawn({ + cmd: [ + // + bunExe(), + "--hot", + "run", + hotRunnerRoot, + ], + env: bunEnv, + cwd, + stdout: "inherit", + stderr: "pipe", + stdin: "ignore", + }); + let reloadCounter = 0; + function onReload() { + writeFileSync( + bundleIn, + `// ${long_comment} +console.error("RSS: %s", process.memoryUsage().rss); +// +${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, + ); + } + let str = ""; + let sampleMemory10: number | undefined; + let sampleMemory100: number | undefined; + outer: for await (const chunk of runner.stderr) { + str += new TextDecoder().decode(chunk); + var any = false; + if (!/error: .*[0-9]\n.*?\n/g.test(str)) continue; + + let it = str.split("\n"); + let line; + while ((line = it.shift())) { + if (!line.includes("error:")) continue; + let rssMatch = str.match(/RSS: (\d+(\.\d+)?)\n/); + let rss; + if (rssMatch) rss = Number(rssMatch[1]); + str = ""; + + if (reloadCounter == 10) { + sampleMemory10 = rss; + } + + if (reloadCounter === 100) { + sampleMemory100 = rss; + runner.kill(); + break; + } + + if (line.includes(`error: ${reloadCounter - 1}`)) { + continue outer; + } + expect(line).toContain(`error: ${reloadCounter}`); + + reloadCounter++; + let next = it.shift()!; + expect(next).toInclude("bundle_in.ts"); + const col = next.match(/\s*at.*?:4:(\d+)$/)![1]; + expect(Number(col)).toBe(1 + "throw ".length + (reloadCounter - 1) * 2); + any = true; + } + + if (any) await onReload(); + } + expect(reloadCounter).toBe(100); + bundler.kill(); + await runner.exited; + // TODO: bun has a memory leak when --hot is used on very large files + // console.log({ sampleMemory10, sampleMemory100 }); +}, 20_000); From e4572cb9e99761396ed7af9b09dc874f3b47d6eb Mon Sep 17 00:00:00 2001 From: dave caruso Date: Wed, 15 May 2024 15:36:38 -0700 Subject: [PATCH 10/18] revert some error handling stuff --- src/bun.js/javascript.zig | 45 ++++--- src/js_ast.zig | 2 +- src/sourcemap/sourcemap.zig | 253 ++++++++++++++++++------------------ 3 files changed, 151 insertions(+), 149 deletions(-) diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 7138c4e0f244c..9f6a56a99b41a 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -143,32 +143,37 @@ pub const SavedSourceMap = struct { default_allocator.free(this.data[0..this.len()]); } - pub fn toMapping(this: SavedMappings, allocator: Allocator, path: string) !ParsedSourceMap { - var err_data: SourceMap.Mapping.ParseError = undefined; - return SourceMap.Mapping.parse( + pub fn toMapping(this: SavedMappings, allocator: Allocator, path: string) anyerror!ParsedSourceMap { + const result = SourceMap.Mapping.parse( allocator, this.data[vlq_offset..this.len()], @as(usize, @bitCast(this.data[8..16].*)), 1, @as(usize, @bitCast(this.data[16..24].*)), - &err_data, - ) catch |err| { - if (Output.enable_ansi_colors_stderr) { - try err_data.toData(path).writeFormat( - Output.errorWriter(), - logger.Kind.warn, - true, - ); - } else { - try err_data.toData(path).writeFormat( - Output.errorWriter(), - logger.Kind.warn, + ); + switch (result) { + .fail => |fail| { + if (Output.enable_ansi_colors_stderr) { + try fail.toData(path).writeFormat( + Output.errorWriter(), + logger.Kind.warn, + true, + ); + } else { + try fail.toData(path).writeFormat( + Output.errorWriter(), + logger.Kind.warn, - false, - ); - } - return err; - }; + false, + ); + } + + return fail.err; + }, + .success => |success| { + return success; + }, + } } }; diff --git a/src/js_ast.zig b/src/js_ast.zig index 6bf4496fceabb..414f17bfc2891 100644 --- a/src/js_ast.zig +++ b/src/js_ast.zig @@ -2429,7 +2429,7 @@ pub const E = struct { { s.resolveRopeIfNeeded(allocator); - const decoded = js_lexer.decodeUTF8(s.slice(allocator), allocator) catch unreachable; + const decoded = js_lexer.decodeStringLiteralEscapeSequencesToUTF16(s.slice(allocator), allocator) catch unreachable; defer allocator.free(decoded); var out, const chars = bun.String.createUninitialized(.utf16, decoded.len); diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig index e74e32ad397cf..1227beab78c9d 100644 --- a/src/sourcemap/sourcemap.zig +++ b/src/sourcemap/sourcemap.zig @@ -71,7 +71,6 @@ pub fn parseUrl( arena: std.mem.Allocator, source: []const u8, comptime result: ParseUrlResultType, - err: *Mapping.ParseError, ) !result.ParseType() { const json_bytes = json_bytes: { const data_prefix = "data:application/json"; @@ -86,11 +85,7 @@ pub fn parseUrl( const bytes = arena.alloc(u8, len) catch bun.outOfMemory(); const decoded = bun.base64.decode(bytes, base64_data); if (decoded.fail) { - return err.fail(error.InvalidBase64, .{ - .msg = "Invalid Base64", - .value = 0, - .loc = .{ .start = 0 }, - }); + return error.InvalidBase64; } break :json_bytes bytes[0..decoded.written]; }, @@ -99,14 +94,10 @@ pub fn parseUrl( } } - return err.fail(error.UnsupportedFormat, .{ - .msg = "Unsupported source map type", - .value = 0, - .loc = .{ .start = 0 }, - }); + return error.UnsupportedFormat; }; - return parseJSON(alloc, arena, json_bytes, result, err); + return parseJSON(alloc, arena, json_bytes, result); } /// Parses a JSON source-map @@ -120,14 +111,13 @@ pub fn parseJSON( arena: std.mem.Allocator, source: []const u8, comptime result: ParseUrlResultType, - err: *Mapping.ParseError, ) !result.ParseType() { const json_src = bun.logger.Source.initPathString("sourcemap.json", source); var log = bun.logger.Log.init(arena); defer log.deinit(); var json = bun.JSON.ParseJSON(&json_src, &log, arena) catch { - return err.fail(error.InvalidJSON, .{}); + return error.InvalidJSON; }; // the allocator given to the JS parser is not respected for all parts @@ -139,32 +129,30 @@ pub fn parseJSON( if (json.get("version")) |version| { if (version.data != .e_number or version.data.e_number.value != 3.0) { - return err.fail(error.UnsupportedVersion, .{}); + return error.UnsupportedVersion; } } const mappings_str = json.get("mappings") orelse { - return err.fail(error.UnsupportedVersion, .{}); + return error.UnsupportedVersion; }; if (mappings_str.data != .e_string) { - return err.fail(error.InvalidSourceMap, .{}); + return error.InvalidSourceMap; } const sources_content = switch ((json.get("sourcesContent") orelse return error.InvalidSourceMap).data) { .e_array => |arr| arr, - else => return err.fail(error.InvalidSourceMap, .{}), + else => return error.InvalidSourceMap, }; const sources_paths = switch ((json.get("sources") orelse return error.InvalidSourceMap).data) { .e_array => |arr| arr, - else => return err.fail(error.InvalidSourceMap, .{}), + else => return error.InvalidSourceMap, }; if (sources_content.items.len != sources_paths.items.len) { - return err.fail(error.InvalidSourceMap, .{ - .msg = "sources.length != sourcesContent.length", - }); + return error.InvalidSourceMap; } var i: usize = 0; @@ -187,23 +175,13 @@ pub fn parseJSON( }; if (result != .sources_only) for (sources_paths.items.slice()) |item| { - if (item.data != .e_string) { - return err.fail(error.InvalidSourceMap, .{ - .msg = "Expected string in sources", - .value = 0, - .loc = .{ .start = 0 }, - }); - } + if (item.data != .e_string) + return error.InvalidSourceMap; - const utf16_decode = bun.js_lexer.decodeUTF8(item.data.e_string.string(arena) catch bun.outOfMemory(), arena) catch bun.outOfMemory(); + const utf16_decode = try bun.js_lexer.decodeStringLiteralEscapeSequencesToUTF16(item.data.e_string.string(arena) catch bun.outOfMemory(), arena); defer arena.free(utf16_decode); - source_paths_slice[i] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch { - return err.fail(error.InvalidSourceMap, .{ - .msg = "Expected string in sources", - .value = 0, - .loc = .{ .start = 0 }, - }); - }; + source_paths_slice[i] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch + return error.InvalidSourceMap; i += 1; }; @@ -224,16 +202,12 @@ pub fn parseJSON( continue; } - const utf16_decode = bun.js_lexer.decodeUTF8(str, arena) catch bun.outOfMemory(); + const utf16_decode = try bun.js_lexer.decodeStringLiteralEscapeSequencesToUTF16(str, arena); defer arena.free(utf16_decode); - source_contents_slice[j] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch { - return err.fail(error.InvalidSourceMap, .{ - .msg = "Expected string in sources", - .value = 0, - .loc = .{ .start = 0 }, - }); - }; + source_contents_slice[j] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch + return error.InvalidSourceMap; + bun.assert(source_contents_slice[j].?.len > 0); } } @@ -242,14 +216,16 @@ pub fn parseJSON( return source_contents_slice; } - var map = try Mapping.parse( + var map = switch (Mapping.parse( alloc, mappings_str.data.e_string.slice(arena), null, std.math.maxInt(i32), std.math.maxInt(i32), - err, - ); + )) { + .success => |x| x, + .fail => |fail| return fail.err, + }; map.external_source_names = source_paths_slice; map.source_contents = if (result != .mappings_only) Mapping.ParsedSourceMap.SourceContentPtr.fromSources(source_contents_slice.ptr) @@ -395,20 +371,17 @@ pub const Mapping = struct { return null; } - /// On error, `err` is populated with error information. pub fn parse( allocator: std.mem.Allocator, bytes: []const u8, estimated_mapping_count: ?usize, sources_count: i32, input_line_count: usize, - err: *ParseError, - ) !ParsedSourceMap { + ) ParseResult { var mapping = Mapping.List{}; if (estimated_mapping_count) |count| { - mapping.ensureTotalCapacity(allocator, count) catch bun.outOfMemory(); + mapping.ensureTotalCapacity(allocator, count) catch unreachable; } - errdefer mapping.deinit(allocator); var generated = LineColumnOffset{ .lines = 0, .columns = 0 }; var original = LineColumnOffset{ .lines = 0, .columns = 0 }; @@ -441,22 +414,28 @@ pub const Mapping = struct { const generated_column_delta = decodeVLQ(remain, 0); if (generated_column_delta.start == 0) { - return err.fail(error.MissingGeneratedColumnValue, .{ - .msg = "Missing generated column value", - .value = generated.columns, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }); + return .{ + .fail = .{ + .msg = "Missing generated column value", + .err = error.MissingGeneratedColumnValue, + .value = generated.columns, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }, + }; } needs_sort = needs_sort or generated_column_delta.value < 0; generated.columns += generated_column_delta.value; if (generated.columns < 0) { - return err.fail(error.InvalidGeneratedColumnValue, .{ - .msg = "Invalid generated column value", - .value = generated.columns, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }); + return .{ + .fail = .{ + .msg = "Invalid generated column value", + .err = error.InvalidGeneratedColumnValue, + .value = generated.columns, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }, + }; } remain = remain[generated_column_delta.start..]; @@ -482,59 +461,81 @@ pub const Mapping = struct { // Read the original source const source_index_delta = decodeVLQ(remain, 0); if (source_index_delta.start == 0) { - return err.fail(error.InvalidSourceIndexDelta, .{ - .msg = "Invalid source index delta", - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }); + return .{ + .fail = .{ + .msg = "Invalid source index delta", + .err = error.InvalidSourceIndexDelta, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }, + }; } source_index += source_index_delta.value; if (source_index < 0 or source_index > sources_count) { - return err.fail(error.InvalidSourceIndexValue, .{ - .msg = "Invalid source index value", - .value = source_index, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }); + return .{ + .fail = .{ + .msg = "Invalid source index value", + .err = error.InvalidSourceIndexValue, + .value = source_index, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }, + }; } remain = remain[source_index_delta.start..]; + // // "AAAA" is extremely common + // if (strings.hasPrefixComptime(remain, "AAAA;")) { + + // } + // Read the original line const original_line_delta = decodeVLQ(remain, 0); if (original_line_delta.start == 0) { - return err.fail(error.MissingOriginalLine, .{ - .msg = "Missing original line", - .err = error.MissingOriginalLine, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }); + return .{ + .fail = .{ + .msg = "Missing original line", + .err = error.MissingOriginalLine, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }, + }; } original.lines += original_line_delta.value; if (original.lines < 0) { - return err.fail(error.InvalidOriginalLineValue, .{ - .msg = "Invalid original line value", - .value = original.lines, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }); + return .{ + .fail = .{ + .msg = "Invalid original line value", + .err = error.InvalidOriginalLineValue, + .value = original.lines, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }, + }; } remain = remain[original_line_delta.start..]; // Read the original column const original_column_delta = decodeVLQ(remain, 0); if (original_column_delta.start == 0) { - return err.fail(error.MissingOriginalColumnValue, .{ - .msg = "Missing original column value", - .value = original.columns, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }); + return .{ + .fail = .{ + .msg = "Missing original column value", + .err = error.MissingOriginalColumnValue, + .value = original.columns, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }, + }; } original.columns += original_column_delta.value; if (original.columns < 0) { - return err.fail(error.InvalidOriginalColumnValue, .{ - .msg = "Invalid original column value", - .value = original.columns, - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }); + return .{ + .fail = .{ + .msg = "Invalid original column value", + .err = error.InvalidOriginalColumnValue, + .value = original.columns, + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }, + }; } remain = remain[original_column_delta.start..]; @@ -545,11 +546,14 @@ pub const Mapping = struct { }, ';' => {}, else => |c| { - return err.fail(error.InvalidSourceMap, .{ - .msg = "Invalid character after mapping", - .value = @as(i32, @intCast(c)), - .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, - }); + return .{ + .fail = .{ + .msg = "Invalid character after mapping", + .err = error.InvalidSourceMap, + .value = @as(i32, @intCast(c)), + .loc = .{ .start = @as(i32, @intCast(bytes.len - remain.len)) }, + }, + }; }, } } @@ -557,38 +561,35 @@ pub const Mapping = struct { .generated = generated, .original = original, .source_index = source_index, - }) catch bun.outOfMemory(); + }) catch unreachable; } - return .{ - .mappings = mapping, - .input_line_count = input_line_count, + return ParseResult{ + .success = .{ + .mappings = mapping, + .input_line_count = input_line_count, + }, }; } - pub const ParseError = struct { - loc: Logger.Loc = Logger.Loc.Empty, - err: anyerror = error.Unexpected, - value: i32 = 0, - msg: []const u8 = "", - - pub fn fail(undefined_ptr: *ParseError, comptime err_tag: anytype, data: ParseError) @Type(.{ - .ErrorSet = &.{.{ .name = @errorName(err_tag) }}, - }) { - undefined_ptr.* = data; - undefined_ptr.err = err_tag; - return err_tag; - } - - pub fn toData(this: @This(), path: []const u8) Logger.Data { - return .{ - .location = .{ - .file = path, - .offset = this.loc.toUsize(), - }, - .text = this.msg, - }; - } + pub const ParseResult = union(enum) { + fail: struct { + loc: Logger.Loc, + err: anyerror, + value: i32 = 0, + msg: []const u8 = "", + + pub fn toData(this: @This(), path: []const u8) Logger.Data { + return Logger.Data{ + .location = Logger.Location{ + .file = path, + .offset = this.loc.toUsize(), + }, + .text = this.msg, + }; + } + }, + success: ParsedSourceMap, }; pub const ParsedSourceMap = struct { @@ -742,13 +743,11 @@ pub const SourceProviderMap = opaque { break :try_inline; defer found_url.deinit(); - var err_data: Mapping.ParseError = undefined; break :parsed parseUrl( bun.default_allocator, arena.allocator(), found_url.slice(), result, - &err_data, ) catch return null; } @@ -771,13 +770,11 @@ pub const SourceProviderMap = opaque { load_from_separate_file = true; - var err_data: Mapping.ParseError = undefined; break :parsed parseJSON( bun.default_allocator, arena.allocator(), data, result, - &err_data, ) catch return null; } From 4a02f816273336b57eec6570ae6590659eb3f6a5 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Wed, 15 May 2024 19:51:28 -0700 Subject: [PATCH 11/18] address feedback --- src/bun.js/javascript.zig | 79 ++++--- src/sourcemap/CodeCoverage.zig | 3 +- src/sourcemap/sourcemap.zig | 375 ++++++++++++++------------------- 3 files changed, 206 insertions(+), 251 deletions(-) diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 9f6a56a99b41a..06ba5d1363732 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -215,19 +215,15 @@ pub const SavedSourceMap = struct { const old_value = Value.from(entry.value_ptr.*); if (old_value.get(SourceProviderMap)) |prov| { if (@intFromPtr(prov) == @intFromPtr(opaque_source_provider)) { + // there is nothing to unref or deinit this.map.removeByPtr(entry.key_ptr); } } else if (old_value.get(ParsedSourceMap)) |map| { - if (map.source_contents.state == .unloaded and - @intFromPtr(map.source_contents.provider()) == @intFromPtr(opaque_source_provider)) - { - map.deinit(default_allocator); - this.map.removeByPtr(entry.key_ptr); - } else { - // not possible to know for sure this is the same map. also not worth - // knowing, because what will happen is one of: - // - a hot reload happens and replaces this entry. - // - the process exits, freeing the memory. + if (map.underlying_provider.provider()) |prov| { + if (@intFromPtr(prov) == @intFromPtr(opaque_source_provider)) { + map.deinit(default_allocator); + this.map.removeByPtr(entry.key_ptr); + } } } } @@ -286,12 +282,16 @@ pub const SavedSourceMap = struct { entry.value_ptr.* = value.ptr(); } - pub fn get(this: *SavedSourceMap, path: string, contents: SourceMap.SourceContentHandling) ?*ParsedSourceMap { + pub fn getWithContent( + this: *SavedSourceMap, + path: string, + hint: SourceMap.ParseUrlResultHint, + ) SourceMap.ParseUrl { const hash = bun.hash(path); - const mapping = this.map.getEntry(hash) orelse return null; + const mapping = this.map.getEntry(hash) orelse return .{}; switch (Value.from(mapping.value_ptr.*).tag()) { Value.Tag.ParsedSourceMap => { - return Value.from(mapping.value_ptr.*).as(ParsedSourceMap); + return .{ .map = Value.from(mapping.value_ptr.*).as(ParsedSourceMap) }; }, Value.Tag.SavedMappings => { var saved = SavedMappings{ .data = @as([*]u8, @ptrCast(Value.from(mapping.value_ptr.*).as(ParsedSourceMap))) }; @@ -299,37 +299,42 @@ pub const SavedSourceMap = struct { const result = default_allocator.create(ParsedSourceMap) catch unreachable; result.* = saved.toMapping(default_allocator, path) catch { _ = this.map.remove(mapping.key_ptr.*); - return null; + return .{}; }; mapping.value_ptr.* = Value.init(result).ptr(); - return result; + return .{ .map = result }; }, Value.Tag.SourceProviderMap => { var ptr = Value.from(mapping.value_ptr.*).as(SourceProviderMap); - if (ptr.toParsedSourceMap(path, contents)) |result| { - mapping.value_ptr.* = Value.init(result).ptr(); - return result; - } else { - // does not have a valid source map. let's not try again - _ = this.map.remove(hash); - // Store path for a user note. - const storage = MissingSourceMapNoteInfo.storage[0..path.len]; - @memcpy(storage, path); - MissingSourceMapNoteInfo.path = storage; + if (ptr.getSourceMap(path, .none, hint)) |parse| + if (parse.map) |map| { + mapping.value_ptr.* = Value.init(map).ptr(); + return parse; + }; - return null; - } + // does not have a valid source map. let's not try again + _ = this.map.remove(hash); + + // Store path for a user note. + const storage = MissingSourceMapNoteInfo.storage[0..path.len]; + @memcpy(storage, path); + MissingSourceMapNoteInfo.path = storage; + return .{}; }, else => { if (Environment.allow_assert) { @panic("Corrupt pointer tag"); } - return null; + return .{}; }, } } + pub fn get(this: *SavedSourceMap, path: string) ?*ParsedSourceMap { + return this.getWithContent(path, .mappings_only).map; + } + pub fn resolveMapping( this: *SavedSourceMap, path: []const u8, @@ -340,14 +345,19 @@ pub const SavedSourceMap = struct { this.mutex.lock(); defer this.mutex.unlock(); - const parsed_mapping = this.get(path, source_handling) orelse - return null; - const mapping = SourceMap.Mapping.find(parsed_mapping.mappings, line, column) orelse + const parse = this.getWithContent(path, switch (source_handling) { + .no_source_contents => .mappings_only, + .source_contents => .{ .all = .{ .line = line, .column = column } }, + }); + const map = parse.map orelse return null; + const mapping = parse.mapping orelse + SourceMap.Mapping.find(map.mappings, line, column) orelse return null; return .{ .mapping = mapping, - .source_map = parsed_mapping, + .source_map = map, + .prefetched_source_code = parse.source_contents, }; } }; @@ -2928,8 +2938,9 @@ pub const VirtualMachine = struct { }, .source_index = 0, }, - // undefined is fine, because this pointer is never read if `top.remapped == true` + // undefined is fine, because these two values are never read if `top.remapped == true` .source_map = undefined, + .prefetched_source_code = undefined, } else this.source_mappings.resolveMapping( @@ -2952,7 +2963,7 @@ pub const VirtualMachine = struct { const code = code: { if (!top.remapped and lookup.source_map.isExternal()) { if (lookup.getSourceCode(top_source_url.slice())) |src| { - break :code ZigString.Slice.fromUTF8NeverFree(src); + break :code src; } } diff --git a/src/sourcemap/CodeCoverage.zig b/src/sourcemap/CodeCoverage.zig index 0576ffa65b2a3..58138c2ff8103 100644 --- a/src/sourcemap/CodeCoverage.zig +++ b/src/sourcemap/CodeCoverage.zig @@ -369,8 +369,7 @@ pub const ByteRangeMapping = struct { var executable_lines: Bitset = Bitset{}; var lines_which_have_executed: Bitset = Bitset{}; - const parsed_mappings_ = bun.JSC.VirtualMachine.get() - .source_mappings.get(source_url.slice(), .no_source_contents); + const parsed_mappings_ = bun.JSC.VirtualMachine.get().source_mappings.get(source_url.slice()); var functions = std.ArrayListUnmanaged(CodeCoverageReport.Block){}; try functions.ensureTotalCapacityPrecise(allocator, function_blocks.len); diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig index 1227beab78c9d..7eb950d76a3b9 100644 --- a/src/sourcemap/sourcemap.zig +++ b/src/sourcemap/sourcemap.zig @@ -39,24 +39,26 @@ sources_content: []string, mapping: Mapping.List = .{}, allocator: std.mem.Allocator, -const ParseUrlResultType = enum { - all, +/// Dictates what parseUrl/parseJSON return. +pub const ParseUrlResultHint = union(enum) { mappings_only, - sources_only, - - pub fn GetSourceMapType(comptime t: ParseUrlResultType) type { - return switch (t) { - .all, .mappings_only => *Mapping.ParsedSourceMap, - .sources_only => []?[]const u8, - }; - } + /// Source Index to fetch + source_only: u32, + /// In order to fetch source contents, you need to know the + /// index, but you cant know the index until the mappings + /// are loaded. So pass in line+col. + all: struct { line: i32, column: i32 }, +}; - pub fn ParseType(comptime t: ParseUrlResultType) type { - return switch (t) { - .all, .mappings_only => Mapping.ParsedSourceMap, - .sources_only => []?[]const u8, - }; - } +pub const ParseUrl = struct { + /// Populated when `mappings_only` or `all`. + map: ?*Mapping.ParsedSourceMap = null, + /// Populated when `all` + /// May be `null` even when requested. + mapping: ?Mapping = null, + /// Populated when `source_only` or `all` + /// May be `null` even when requested, if did not exist in map. + source_contents: ?[]const u8 = null, }; /// Parses an inline source map url like `data:application/json,....` @@ -70,8 +72,8 @@ pub fn parseUrl( alloc: std.mem.Allocator, arena: std.mem.Allocator, source: []const u8, - comptime result: ParseUrlResultType, -) !result.ParseType() { + hint: ParseUrlResultHint, +) !ParseUrl { const json_bytes = json_bytes: { const data_prefix = "data:application/json"; if (bun.strings.hasPrefixComptime(source, data_prefix) and source.len > (data_prefix.len + 1)) try_data_url: { @@ -97,7 +99,7 @@ pub fn parseUrl( return error.UnsupportedFormat; }; - return parseJSON(alloc, arena, json_bytes, result); + return parseJSON(alloc, arena, json_bytes, hint); } /// Parses a JSON source-map @@ -110,8 +112,8 @@ pub fn parseJSON( alloc: std.mem.Allocator, arena: std.mem.Allocator, source: []const u8, - comptime result: ParseUrlResultType, -) !result.ParseType() { + hint: ParseUrlResultHint, +) !ParseUrl { const json_src = bun.logger.Source.initPathString("sourcemap.json", source); var log = bun.logger.Log.init(arena); defer log.deinit(); @@ -157,81 +159,82 @@ pub fn parseJSON( var i: usize = 0; - const source_contents_slice = if (result != .mappings_only) - alloc.alloc(?[]const u8, sources_content.items.len) catch bun.outOfMemory(); - errdefer if (result != .mappings_only) { - for (source_contents_slice[0..i]) |item_nullable| if (item_nullable) |item| { - if (item.len > 0) - alloc.free(item); - }; - alloc.free(source_contents_slice); - }; - - const source_paths_slice = if (result != .sources_only) - alloc.alloc([]const u8, sources_content.items.len) catch bun.outOfMemory(); - errdefer if (result != .sources_only) { - for (source_paths_slice[0..i]) |item| alloc.free(item); - alloc.free(source_paths_slice); + const source_paths_slice = if (hint != .source_only) + alloc.alloc([]const u8, sources_content.items.len) catch bun.outOfMemory() + else + null; + errdefer if (hint != .source_only) { + for (source_paths_slice.?[0..i]) |item| alloc.free(item); + alloc.free(source_paths_slice.?); }; - if (result != .sources_only) for (sources_paths.items.slice()) |item| { + if (hint != .source_only) for (sources_paths.items.slice()) |item| { if (item.data != .e_string) return error.InvalidSourceMap; const utf16_decode = try bun.js_lexer.decodeStringLiteralEscapeSequencesToUTF16(item.data.e_string.string(arena) catch bun.outOfMemory(), arena); defer arena.free(utf16_decode); - source_paths_slice[i] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch + source_paths_slice.?[i] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch return error.InvalidSourceMap; i += 1; }; - var j: usize = 0; - if (result != .mappings_only) { - for (sources_content.items.slice()) |item| { - defer j += 1; - - if (item.data != .e_string) { - source_contents_slice[j] = null; - continue; - } - - const str = item.data.e_string.string(arena) catch bun.outOfMemory(); - if (str.len == 0) { - source_contents_slice[j] = ""; - continue; - } + const map = if (hint != .source_only) map: { + const map_data = switch (Mapping.parse( + alloc, + mappings_str.data.e_string.slice(arena), + null, + std.math.maxInt(i32), + std.math.maxInt(i32), + )) { + .success => |x| x, + .fail => |fail| return fail.err, + }; - const utf16_decode = try bun.js_lexer.decodeStringLiteralEscapeSequencesToUTF16(str, arena); - defer arena.free(utf16_decode); + const ptr = bun.default_allocator.create(Mapping.ParsedSourceMap) catch bun.outOfMemory(); + ptr.* = map_data; + ptr.external_source_names = source_paths_slice.?; + break :map ptr; + } else null; + errdefer if (map) |m| m.deinit(bun.default_allocator); + + const mapping, const source_index = switch (hint) { + .source_only => |index| .{ null, index }, + .all => |loc| brk: { + const mapping = Mapping.find(map.?.mappings, loc.line, loc.column) orelse + break :brk .{ null, null }; + break :brk .{ mapping, std.math.cast(u32, mapping.source_index) }; + }, + .mappings_only => .{ null, null }, + }; - source_contents_slice[j] = bun.strings.toUTF8Alloc(alloc, utf16_decode) catch - return error.InvalidSourceMap; + const content_slice: ?[]const u8 = if (hint != .mappings_only and + source_index != null and + source_index.? < sources_content.items.len) + content: { + const item = sources_content.items.slice()[source_index.?]; + if (item.data != .e_string) { + break :content null; + } - bun.assert(source_contents_slice[j].?.len > 0); + const str = item.data.e_string.string(arena) catch bun.outOfMemory(); + if (str.len == 0) { + break :content null; } - } - if (result == .sources_only) { - return source_contents_slice; - } + const utf16_decode = try bun.js_lexer.decodeStringLiteralEscapeSequencesToUTF16(str, arena); + defer arena.free(utf16_decode); + + break :content bun.strings.toUTF8Alloc(alloc, utf16_decode) catch + return error.InvalidSourceMap; + } else null; - var map = switch (Mapping.parse( - alloc, - mappings_str.data.e_string.slice(arena), - null, - std.math.maxInt(i32), - std.math.maxInt(i32), - )) { - .success => |x| x, - .fail => |fail| return fail.err, + return .{ + .map = map, + .mapping = mapping, + .source_contents = content_slice, }; - map.external_source_names = source_paths_slice; - map.source_contents = if (result != .mappings_only) - Mapping.ParsedSourceMap.SourceContentPtr.fromSources(source_contents_slice.ptr) - else - .{ .data = 0, .state = .unloaded }; - return map; } pub const Mapping = struct { @@ -242,6 +245,9 @@ pub const Mapping = struct { pub const Lookup = struct { mapping: Mapping, source_map: *ParsedSourceMap, + /// Owned by default_allocator always + /// use `getSourceCode` to access this as a Slice + prefetched_source_code: ?[]const u8, /// This creates a bun.String if the source remap *changes* the source url, /// a case that happens only when the source map points to another file. @@ -264,54 +270,49 @@ pub const Mapping = struct { /// Only valid if `lookup.source_map.isExternal()` /// This has the possibility of invoking a call to the filesystem. - pub fn getSourceCode(lookup: Lookup, base_filename: []const u8) ?[]const u8 { - assert(lookup.source_map.isExternal()); - if (lookup.mapping.source_index >= lookup.source_map.external_source_names.len) - return null; + pub fn getSourceCode(lookup: Lookup, base_filename: []const u8) ?bun.JSC.ZigString.Slice { + const bytes = bytes: { + assert(lookup.source_map.isExternal()); + if (lookup.prefetched_source_code) |code| { + break :bytes code; + } - if (lookup.source_map.source_contents.state == .unloaded) { - const map = lookup.source_map.source_contents.provider().getSourceMap( - base_filename, - if (lookup.source_map.source_contents.load_from_separate_file) - .is_external_map - else - .is_inline_map, - .sources_only, - ) orelse return null; - assert(map.len == lookup.source_map.external_source_names.len); - lookup.source_map.source_contents = Mapping.ParsedSourceMap.SourceContentPtr.fromSources(map.ptr); - } + const provider = lookup.source_map.underlying_provider.provider() orelse + return null; - const sources = lookup.source_map.externalSourceContents(); - const i = lookup.mapping.source_index; - if (i >= sources.len or i < 0) return null; // invalid source map - if (sources[@intCast(i)]) |s| { - if (s.len == 0) return null; - return s; - } + const index = lookup.mapping.source_index; - const name = lookup.source_map.external_source_names[@intCast(i)]; - - var buf: bun.PathBuffer = undefined; - const normalized = bun.path.joinAbsStringBufZ( - bun.path.dirname(base_filename, .auto), - &buf, - &.{name}, - .loose, - ); - const bytes = switch (bun.sys.File.readFrom( - std.fs.cwd(), - normalized, - bun.default_allocator, - )) { - .result => |r| r, - .err => { - sources[@intCast(i)] = ""; + if (provider.getSourceMap( + base_filename, + lookup.source_map.underlying_provider.load_hint, + .{ .source_only = @intCast(index) }, + )) |parsed| + if (parsed.source_contents) |contents| + break :bytes contents; + + if (index >= lookup.source_map.external_source_names.len) return null; - }, + + const name = lookup.source_map.external_source_names[@intCast(index)]; + + var buf: bun.PathBuffer = undefined; + const normalized = bun.path.joinAbsStringBufZ( + bun.path.dirname(base_filename, .auto), + &buf, + &.{name}, + .loose, + ); + switch (bun.sys.File.readFrom( + std.fs.cwd(), + normalized, + bun.default_allocator, + )) { + .result => |r| break :bytes r, + .err => return null, + } }; - sources[@intCast(i)] = bytes; - return bytes; + + return bun.JSC.ZigString.Slice.init(bun.default_allocator, bytes); } }; @@ -600,46 +601,25 @@ pub const Mapping = struct { /// loaded without transpilation but with external sources. This array /// maps `source_index` to the correct filename. external_source_names: []const []const u8 = &.{}, - /// This is 64 bits encoding three usizes' worth of data. - /// - /// In .state = .unloaded, we do not have sources loaded in memory. - /// For most operations, having source code is not needed, only the mapping - /// list + filenames. To get source files from this state, `.data` will - /// be a SourceProviderMap - /// - /// When source files are loaded, this stores a pointer to a slice - /// of ?[]const u8, where null means the specified file is 1. not in the - /// sourcemap directly; and 2. not yet loaded. + /// In order to load source contents from a source-map after the fact, + /// a handle to the underying source provider is stored. Within this pointer, + /// a flag is stored if it is known to be an inline or external source map. /// - /// Only valid if `external_source_names` has entries. - /// - /// `.load_from_map_file = true` means the file should load via a `.map` - /// file adacent to the source file, where `false` means there is an - /// inline source map to use. - source_contents: SourceContentPtr = .{ .state = .invalid, .data = 0 }, + /// Source contents are large, we don't preserve them in memory. This has + /// the downside of repeatedly re-decoding sourcemaps if multiple errors + /// are emitted (specifically with Bun.inspect / unhandled; the ones that + /// rely on source contents) + underlying_provider: SourceContentPtr = .{ .data = 0 }, const SourceContentPtr = packed struct(u64) { - load_from_separate_file: bool = false, - state: State, - data: u61, - - const State = enum(u2) { loaded, unloaded, invalid }; + load_hint: SourceMapLoadHint = .none, + data: u62, fn fromProvider(p: *SourceProviderMap) SourceContentPtr { - return .{ .state = .unloaded, .data = @intCast(@intFromPtr(p)) }; + return .{ .data = @intCast(@intFromPtr(p)) }; } - fn fromSources(p: [*]?[]const u8) SourceContentPtr { - return .{ .state = .loaded, .data = @intCast(@intFromPtr(p)) }; - } - - pub fn provider(sc: SourceContentPtr) *SourceProviderMap { - bun.assert(sc.state == .unloaded); - return @ptrFromInt(sc.data); - } - - fn sources(sc: SourceContentPtr) [*]?[]const u8 { - bun.assert(sc.state == .loaded); + pub fn provider(sc: SourceContentPtr) ?*SourceProviderMap { return @ptrFromInt(sc.data); } }; @@ -648,10 +628,6 @@ pub const Mapping = struct { return psm.external_source_names.len != 0; } - pub fn externalSourceContents(psm: *ParsedSourceMap) []?[]const u8 { - return psm.source_contents.sources()[0..psm.external_source_names.len]; - } - pub fn deinit(this: *ParsedSourceMap, allocator: std.mem.Allocator) void { this.mappings.deinit(allocator); @@ -661,19 +637,6 @@ pub const Mapping = struct { allocator.free(this.external_source_names); } - switch (this.source_contents.state) { - .invalid, .unloaded => {}, - .loaded => { - bun.assert(this.external_source_names.len > 0); - const slice = this.externalSourceContents(); - for (slice) |maybe_item| if (maybe_item) |item| { - if (item.len > 0) - allocator.free(item); - }; - allocator.free(slice); - }, - } - allocator.destroy(this); } }; @@ -688,6 +651,8 @@ pub const SourceContentHandling = enum { source_contents, }; +/// For some sourcemap loading code, this enum is used as a hint if we already +/// know if the sourcemap is located on disk or inline in the source code. pub const SourceMapLoadHint = enum { none, is_inline_map, @@ -721,15 +686,13 @@ pub const SourceProviderMap = opaque { provider: *SourceProviderMap, source_filename: []const u8, load_hint: SourceMapLoadHint, - comptime result: ParseUrlResultType, - ) ?result.GetSourceMapType() { + result: ParseUrlResultHint, + ) ?SourceMap.ParseUrl { var sfb = std.heap.stackFallback(65536, bun.default_allocator); var arena = bun.ArenaAllocator.init(sfb.get()); defer arena.deinit(); - var load_from_separate_file = false; - - const parsed = parsed: { + const new_load_hint: SourceMapLoadHint, const parsed = parsed: { // try to get an inline source map if (load_hint != .is_external_map) try_inline: { const source = ZigSourceProvider__getSourceSlice(provider); @@ -743,12 +706,15 @@ pub const SourceProviderMap = opaque { break :try_inline; defer found_url.deinit(); - break :parsed parseUrl( - bun.default_allocator, - arena.allocator(), - found_url.slice(), - result, - ) catch return null; + break :parsed .{ + .is_inline_map, + parseUrl( + bun.default_allocator, + arena.allocator(), + found_url.slice(), + result, + ) catch return null, + }; } // try to load a .map file @@ -768,45 +734,24 @@ pub const SourceProviderMap = opaque { .result => |data| data, }; - load_from_separate_file = true; - - break :parsed parseJSON( - bun.default_allocator, - arena.allocator(), - data, - result, - ) catch return null; + break :parsed .{ + .is_external_map, + parseJSON( + bun.default_allocator, + arena.allocator(), + data, + result, + ) catch return null, + }; } return null; }; - - if (result == .sources_only) return parsed; - - const ptr = bun.default_allocator.create(Mapping.ParsedSourceMap) catch bun.outOfMemory(); - ptr.* = parsed; - if (ptr.source_contents.state == .unloaded) { - ptr.source_contents = Mapping.ParsedSourceMap.SourceContentPtr.fromProvider(provider); - ptr.source_contents.load_from_separate_file = load_from_separate_file; + if (parsed.map) |ptr| { + ptr.underlying_provider = Mapping.ParsedSourceMap.SourceContentPtr.fromProvider(provider); + ptr.underlying_provider.load_hint = new_load_hint; } - return ptr; - } - - pub fn toParsedSourceMap( - provider: *SourceProviderMap, - source_filename: []const u8, - source_handling: SourceContentHandling, - ) ?*Mapping.ParsedSourceMap { - return switch (source_handling) { - inline else => |tag| provider.getSourceMap( - source_filename, - .none, - switch (tag) { - .no_source_contents => .mappings_only, - .source_contents => .all, - }, - ), - }; + return parsed; } }; From 8bea8e51566dee6571dc4b3a5e152114599c7399 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Wed, 15 May 2024 20:35:18 -0700 Subject: [PATCH 12/18] fix memory issue --- src/bun.js/ConsoleObject.zig | 11 ++++++++++- src/bun.js/javascript.zig | 15 +++++++++++++-- test/bundler/bundler_bun.test.ts | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/bun.js/ConsoleObject.zig b/src/bun.js/ConsoleObject.zig index 2a702d5c73009..0f4b9a6db13c1 100644 --- a/src/bun.js/ConsoleObject.zig +++ b/src/bun.js/ConsoleObject.zig @@ -630,9 +630,18 @@ pub fn writeTrace(comptime Writer: type, writer: Writer, global: *JSGlobalObject defer holder.deinit(vm); const exception = holder.zigException(); + var source_code_slice: ?ZigString.Slice = null; + defer if (source_code_slice) |slice| slice.deinit(); + var err = ZigString.init("trace output").toErrorInstance(global); err.toZigException(global, exception); - vm.remapZigException(exception, err, null, &holder.need_to_clear_parser_arena_on_deinit); + vm.remapZigException( + exception, + err, + null, + &holder.need_to_clear_parser_arena_on_deinit, + &source_code_slice, + ); if (Output.enable_ansi_colors_stderr) VirtualMachine.printStackTrace( diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 06ba5d1363732..79f4b30d558c9 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -2849,6 +2849,7 @@ pub const VirtualMachine = struct { error_instance: JSValue, exception_list: ?*ExceptionList, must_reset_parser_arena_later: *bool, + source_code_slice: *?ZigString.Slice, ) void { error_instance.toZigException(this.global, exception); @@ -2972,7 +2973,7 @@ pub const VirtualMachine = struct { must_reset_parser_arena_later.* = true; break :code original_source.source_code.toUTF8(bun.default_allocator); }; - defer code.deinit(); + source_code_slice.* = code; top.position.line = mapping.original.lines; top.position.line_start = mapping.original.lines; @@ -3046,7 +3047,17 @@ pub const VirtualMachine = struct { var exception_holder = ZigException.Holder.init(); var exception = exception_holder.zigException(); defer exception_holder.deinit(this); - this.remapZigException(exception, error_instance, exception_list, &exception_holder.need_to_clear_parser_arena_on_deinit); + + var source_code_slice: ?ZigString.Slice = null; + defer if (source_code_slice) |slice| slice.deinit(); + + this.remapZigException( + exception, + error_instance, + exception_list, + &exception_holder.need_to_clear_parser_arena_on_deinit, + &source_code_slice, + ); const prev_had_errors = this.had_errors; this.had_errors = true; defer this.had_errors = prev_had_errors; diff --git a/test/bundler/bundler_bun.test.ts b/test/bundler/bundler_bun.test.ts index 38f9f810f80ac..0891a6deeac82 100644 --- a/test/bundler/bundler_bun.test.ts +++ b/test/bundler/bundler_bun.test.ts @@ -93,7 +93,7 @@ describe("bundler", () => { 6 | throw new ^ error: Hello World`, - ), + ) || void console.error(stderr), ); expect(stderr).toInclude("entry.ts:6:19"); }, From e92b4758fafd55fdc9005b76a75336281bd4e72c Mon Sep 17 00:00:00 2001 From: dave caruso Date: Wed, 15 May 2024 22:37:43 -0700 Subject: [PATCH 13/18] make test more forgiving --- src/deps/zig | 2 +- test/cli/hot/hot.test.ts | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/deps/zig b/src/deps/zig index d5b5fb3182864..1dcf931391371 160000 --- a/src/deps/zig +++ b/src/deps/zig @@ -1 +1 @@ -Subproject commit d5b5fb31828640ce3cd552f18ca74c2eb05993ad +Subproject commit 1dcf9313913712999cda6901d1d8bee77d9e08e9 diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index 5ffa29e2dfe47..71851938c0d82 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -346,7 +346,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, ); } let str = ""; - for await (const chunk of runner.stderr) { + outer: for await (const chunk of runner.stderr) { str += new TextDecoder().decode(chunk); var any = false; if (!/error: .*[0-9]\n.*?\n/g.test(str)) continue; @@ -363,7 +363,12 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, break; } - expect(line).toContain(`error: ${reloadCounter - 1}`); + if (line.includes(`error: ${reloadCounter - 1}`)) { + continue outer; + } + expect(line).toContain(`error: ${reloadCounter}`); + reloadCounter++; + let next = it.shift()!; if (!next) throw new Error(line); const match = next.match(/\s*at.*?:1003:(\d+)$/); @@ -418,7 +423,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, ); } let str = ""; - for await (const chunk of runner.stderr) { + outer: for await (const chunk of runner.stderr) { str += new TextDecoder().decode(chunk); var any = false; if (!/error: .*[0-9]\n.*?\n/g.test(str)) continue; @@ -427,7 +432,6 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, let line; while ((line = it.shift())) { if (!line.includes("error")) continue; - reloadCounter++; str = ""; if (reloadCounter === 100) { @@ -435,7 +439,12 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, break; } - expect(line).toContain(`error: ${reloadCounter - 1}`); + if (line.includes(`error: ${reloadCounter - 1}`)) { + continue outer; + } + expect(line).toContain(`error: ${reloadCounter}`); + reloadCounter++; + let next = it.shift()!; expect(next).toInclude("bundle_in.ts"); const col = next.match(/\s*at.*?:4:(\d+)$/)![1]; From ef3e0b97bdb6e2a1001ea8e546d77344e81e5e19 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Thu, 16 May 2024 17:23:42 -0700 Subject: [PATCH 14/18] fix the build --- CMakeLists.txt | 16 +++++++++++----- src/bun.js/WebKit | 2 +- src/deps/zig | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2397d442ef63a..8a21c4af9c1f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -307,6 +307,7 @@ option(USE_CUSTOM_TINYCC "Use Bun's recommended version of tinycc" ON) option(USE_CUSTOM_LIBUV "Use Bun's recommended version of libuv (Windows only)" ON) option(USE_CUSTOM_LSHPACK "Use Bun's recommended version of ls-hpack" ON) option(USE_BASELINE_BUILD "Build Bun for baseline (older) CPUs" OFF) +option(USE_SYSTEM_ICU "Use the system-provided libicu. May fix startup crashes when building WebKit yourself." OFF) option(USE_VALGRIND "Build Bun with Valgrind support (Linux only)" OFF) @@ -320,7 +321,6 @@ option(USE_LTO "Enable Link-Time Optimization" ${DEFAULT_LTO}) option(BUN_TIDY_ONLY "Only run clang-tidy" OFF) option(BUN_TIDY_ONLY_EXTRA " Only run clang-tidy, with extra checks for local development" OFF) - if(NOT ZIG_LIB_DIR) cmake_path(SET ZIG_LIB_DIR NORMALIZE "${CMAKE_CURRENT_SOURCE_DIR}/src/deps/zig/lib") endif() @@ -1179,9 +1179,15 @@ if(UNIX AND NOT APPLE) target_link_libraries(${bun} PRIVATE "libatomic.a") endif() - target_link_libraries(${bun} PRIVATE "${WEBKIT_LIB_DIR}/libicudata.a") - target_link_libraries(${bun} PRIVATE "${WEBKIT_LIB_DIR}/libicui18n.a") - target_link_libraries(${bun} PRIVATE "${WEBKIT_LIB_DIR}/libicuuc.a") + if(USE_SYSTEM_ICU) + target_link_libraries(${bun} PRIVATE "libicudata.a") + target_link_libraries(${bun} PRIVATE "libicui18n.a") + target_link_libraries(${bun} PRIVATE "libicuuc.a") + else() + target_link_libraries(${bun} PRIVATE "${WEBKIT_LIB_DIR}/libicudata.a") + target_link_libraries(${bun} PRIVATE "${WEBKIT_LIB_DIR}/libicui18n.a") + target_link_libraries(${bun} PRIVATE "${WEBKIT_LIB_DIR}/libicuuc.a") + endif() set_target_properties(${bun} PROPERTIES LINK_DEPENDS "${BUN_SRC}/linker.lds") set_target_properties(${bun} PROPERTIES LINK_DEPENDS "${BUN_SRC}/symbols.dyn") @@ -1446,7 +1452,7 @@ if(BUN_TIDY_ONLY) set_target_properties(${bun} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_COMMAND}") endif() -if (BUN_TIDY_ONLY_EXTRA) +if(BUN_TIDY_ONLY_EXTRA) find_program(CLANG_TIDY_EXE NAMES "clang-tidy") set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "-checks=-*,clang-analyzer-*,performance-*,-clang-analyzer-webkit.UncountedLambdaCapturesChecker" "--fix" "--fix-errors" "--format-style=webkit" "--warnings-as-errors=*") set_target_properties(${bun} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_COMMAND}") diff --git a/src/bun.js/WebKit b/src/bun.js/WebKit index ddc47cfa03b87..413f6fc119cfc 160000 --- a/src/bun.js/WebKit +++ b/src/bun.js/WebKit @@ -1 +1 @@ -Subproject commit ddc47cfa03b87d48217079b7aaa64d23ebd3c3a2 +Subproject commit 413f6fc119cfca98d5063049ce444cc4eb56f0a4 diff --git a/src/deps/zig b/src/deps/zig index 1dcf931391371..d5b5fb3182864 160000 --- a/src/deps/zig +++ b/src/deps/zig @@ -1 +1 @@ -Subproject commit 1dcf9313913712999cda6901d1d8bee77d9e08e9 +Subproject commit d5b5fb31828640ce3cd552f18ca74c2eb05993ad From 1e3ec457e0159bb252cb0365be6bf64cb1d6a8b9 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Thu, 16 May 2024 18:32:56 -0700 Subject: [PATCH 15/18] make tests more chillax --- test/bundler/bundler_bun.test.ts | 4 +- test/cli/hot/hot.test.ts | 296 ++++++++++++++++--------------- test/harness.ts | 1 + 3 files changed, 155 insertions(+), 146 deletions(-) diff --git a/test/bundler/bundler_bun.test.ts b/test/bundler/bundler_bun.test.ts index 0891a6deeac82..663186068445d 100644 --- a/test/bundler/bundler_bun.test.ts +++ b/test/bundler/bundler_bun.test.ts @@ -58,7 +58,7 @@ describe("bundler", () => { `, }, run: { - exitCode: isWindows ? 3 : 1, + exitCode: 1, validate({ stderr }) { expect(stderr).toInclude("\nnote: missing sourcemaps for "); expect(stderr).toInclude("\nnote: consider bundling with '--sourcemap' to get an unminified traces\n"); @@ -81,7 +81,7 @@ describe("bundler", () => { }, sourceMap: "inline", run: { - exitCode: isWindows ? 3 : 1, + exitCode: 1, validate({ stderr }) { assert( stderr.startsWith( diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index 71851938c0d82..3090415d64b2e 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -1,6 +1,6 @@ import { spawn } from "bun"; import { beforeAll, beforeEach, expect, it } from "bun:test"; -import { bunExe, bunEnv, tmpdirSync } from "harness"; +import { bunExe, bunEnv, tmpdirSync, isDebug } from "harness"; import { cpSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync, copyFileSync } from "fs"; import { join } from "path"; @@ -320,71 +320,73 @@ it("should hot reload when a file is renamed() into place", async () => { } }); -const comment_spam = ("//" + "A".repeat(2000) + "\n").repeat(1000); -it("should work with sourcemap generation", async () => { - writeFileSync( - hotRunnerRoot, - `// source content -${comment_spam} -throw new Error('0');`, - ); - await using runner = spawn({ - cmd: [bunExe(), "--smol", "--hot", "run", hotRunnerRoot], - env: bunEnv, - cwd, - stdout: "ignore", - stderr: "pipe", - stdin: "ignore", - }); - let reloadCounter = 0; - function onReload() { +const comment_spam = ("//" + "B".repeat(2000) + "\n").repeat(1000); +it( + "should work with sourcemap generation", + async () => { writeFileSync( hotRunnerRoot, `// source content ${comment_spam} -${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, +throw new Error('0');`, ); - } - let str = ""; - outer: for await (const chunk of runner.stderr) { - str += new TextDecoder().decode(chunk); - var any = false; - if (!/error: .*[0-9]\n.*?\n/g.test(str)) continue; + await using runner = spawn({ + cmd: [bunExe(), "--smol", "--hot", "run", hotRunnerRoot], + env: bunEnv, + cwd, + stdout: "ignore", + stderr: "pipe", + stdin: "ignore", + }); + let reloadCounter = 0; + function onReload() { + writeFileSync( + hotRunnerRoot, + `// source content +${comment_spam} +${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, + ); + } + let str = ""; + outer: for await (const chunk of runner.stderr) { + str += new TextDecoder().decode(chunk); + var any = false; + if (!/error: .*[0-9]\n.*?\n/g.test(str)) continue; - let it = str.split("\n"); - let line; - while ((line = it.shift())) { - if (!line.includes("error")) continue; - reloadCounter++; - str = ""; + let it = str.split("\n"); + let line; + while ((line = it.shift())) { + if (!line.includes("error")) continue; + str = ""; - if (reloadCounter === 100) { - runner.kill(); - break; - } + if (reloadCounter === 50) { + runner.kill(); + break; + } - if (line.includes(`error: ${reloadCounter - 1}`)) { - continue outer; + if (line.includes(`error: ${reloadCounter - 1}`)) { + onReload(); // re-save file to prevent deadlock + continue outer; + } + expect(line).toContain(`error: ${reloadCounter}`); + reloadCounter++; + + let next = it.shift()!; + if (!next) throw new Error(line); + const match = next.match(/\s*at.*?:1003:(\d+)$/); + if (!match) throw new Error("invalid string: " + next); + const col = match[1]; + expect(Number(col)).toBe(1 + "throw ".length + (reloadCounter - 1) * 2); + any = true; } - expect(line).toContain(`error: ${reloadCounter}`); - reloadCounter++; - let next = it.shift()!; - if (!next) throw new Error(line); - const match = next.match(/\s*at.*?:1003:(\d+)$/); - if (!match) throw new Error("invalid string: " + next); - const col = match[1]; - expect(Number(col)).toBe(1 + "throw ".length + (reloadCounter - 1) * 2); - any = true; + if (any) await onReload(); } - - if (any) await onReload(); - } - await runner.exited; - const rss = runner.resourceUsage().maxRSS; - expect(rss).toBeLessThan(100_000_000); - expect(reloadCounter).toBe(100); -}); + await runner.exited; + expect(reloadCounter).toBe(50); + }, + isDebug ? Infinity : 10_000, +); it("should work with sourcemap loading", async () => { let bundleIn = join(cwd, "bundle_in.ts"); @@ -434,12 +436,13 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, if (!line.includes("error")) continue; str = ""; - if (reloadCounter === 100) { + if (reloadCounter === 50) { runner.kill(); break; } if (line.includes(`error: ${reloadCounter - 1}`)) { + onReload(); // re-save file to prevent deadlock continue outer; } expect(line).toContain(`error: ${reloadCounter}`); @@ -454,109 +457,114 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, if (any) await onReload(); } - expect(reloadCounter).toBe(100); + expect(reloadCounter).toBe(50); bundler.kill(); }); const long_comment = "BBBB".repeat(100000); -it("should work with sourcemap loading with large files", async () => { - let bundleIn = join(cwd, "bundle_in.ts"); - rmSync(hotRunnerRoot); - writeFileSync( - bundleIn, - `// ${long_comment} -// -console.error("RSS: %s", process.memoryUsage().rss); -throw new Error('0');`, - ); - await using bundler = spawn({ - cmd: [ - // - bunExe(), - "build", - "--watch", - bundleIn, - "--target=bun", - "--sourcemap", - "--outfile", - hotRunnerRoot, - ], - env: bunEnv, - cwd, - stdout: "ignore", - stderr: "ignore", - stdin: "ignore", - }); - await using runner = spawn({ - cmd: [ - // - bunExe(), - "--hot", - "run", - hotRunnerRoot, - ], - env: bunEnv, - cwd, - stdout: "inherit", - stderr: "pipe", - stdin: "ignore", - }); - let reloadCounter = 0; - function onReload() { +it( + "should work with sourcemap loading with large files", + async () => { + let bundleIn = join(cwd, "bundle_in.ts"); + rmSync(hotRunnerRoot); writeFileSync( bundleIn, `// ${long_comment} +// +console.error("RSS: %s", process.memoryUsage().rss); +throw new Error('0');`, + ); + await using bundler = spawn({ + cmd: [ + // + bunExe(), + "build", + "--watch", + bundleIn, + "--target=bun", + "--sourcemap", + "--outfile", + hotRunnerRoot, + ], + env: bunEnv, + cwd, + stdout: "ignore", + stderr: "ignore", + stdin: "ignore", + }); + await using runner = spawn({ + cmd: [ + // + bunExe(), + "--hot", + "run", + hotRunnerRoot, + ], + env: bunEnv, + cwd, + stdout: "inherit", + stderr: "pipe", + stdin: "ignore", + }); + let reloadCounter = 0; + function onReload() { + writeFileSync( + bundleIn, + `// ${long_comment} console.error("RSS: %s", process.memoryUsage().rss); // ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, - ); - } - let str = ""; - let sampleMemory10: number | undefined; - let sampleMemory100: number | undefined; - outer: for await (const chunk of runner.stderr) { - str += new TextDecoder().decode(chunk); - var any = false; - if (!/error: .*[0-9]\n.*?\n/g.test(str)) continue; + ); + } + let str = ""; + let sampleMemory10: number | undefined; + let sampleMemory100: number | undefined; + outer: for await (const chunk of runner.stderr) { + str += new TextDecoder().decode(chunk); + var any = false; + if (!/error: .*[0-9]\n.*?\n/g.test(str)) continue; + + let it = str.split("\n"); + let line; + while ((line = it.shift())) { + if (!line.includes("error:")) continue; + let rssMatch = str.match(/RSS: (\d+(\.\d+)?)\n/); + let rss; + if (rssMatch) rss = Number(rssMatch[1]); + str = ""; - let it = str.split("\n"); - let line; - while ((line = it.shift())) { - if (!line.includes("error:")) continue; - let rssMatch = str.match(/RSS: (\d+(\.\d+)?)\n/); - let rss; - if (rssMatch) rss = Number(rssMatch[1]); - str = ""; + if (reloadCounter == 10) { + sampleMemory10 = rss; + } - if (reloadCounter == 10) { - sampleMemory10 = rss; - } + if (reloadCounter === 50) { + sampleMemory100 = rss; + runner.kill(); + break; + } - if (reloadCounter === 100) { - sampleMemory100 = rss; - runner.kill(); - break; - } + if (line.includes(`error: ${reloadCounter - 1}`)) { + onReload(); // re-save file to prevent deadlock + continue outer; + } + expect(line).toContain(`error: ${reloadCounter}`); - if (line.includes(`error: ${reloadCounter - 1}`)) { - continue outer; + reloadCounter++; + let next = it.shift()!; + expect(next).toInclude("bundle_in.ts"); + const col = next.match(/\s*at.*?:4:(\d+)$/)![1]; + expect(Number(col)).toBe(1 + "throw ".length + (reloadCounter - 1) * 2); + any = true; } - expect(line).toContain(`error: ${reloadCounter}`); - reloadCounter++; - let next = it.shift()!; - expect(next).toInclude("bundle_in.ts"); - const col = next.match(/\s*at.*?:4:(\d+)$/)![1]; - expect(Number(col)).toBe(1 + "throw ".length + (reloadCounter - 1) * 2); - any = true; + if (any) await onReload(); } - - if (any) await onReload(); - } - expect(reloadCounter).toBe(100); - bundler.kill(); - await runner.exited; - // TODO: bun has a memory leak when --hot is used on very large files - // console.log({ sampleMemory10, sampleMemory100 }); -}, 20_000); + expect(reloadCounter).toBe(50); + bundler.kill(); + await runner.exited; + // TODO: bun has a memory leak when --hot is used on very large files + // console.log({ sampleMemory10, sampleMemory100 }); + }, + isDebug ? Infinity : 20_000, +); diff --git a/test/harness.ts b/test/harness.ts index bcca18648b5d3..6ca1dac819fe1 100644 --- a/test/harness.ts +++ b/test/harness.ts @@ -13,6 +13,7 @@ export const isLinux = process.platform === "linux"; export const isPosix = isMacOS || isLinux; export const isWindows = process.platform === "win32"; export const isIntelMacOS = isMacOS && process.arch === "x64"; +export const isDebug = Bun.version.includes("debug"); export const bunEnv: NodeJS.ProcessEnv = { ...process.env, From f7cb9b3afffcdf173d1954a3abfdbc7cc1ed0dd1 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Tue, 21 May 2024 11:04:09 -0700 Subject: [PATCH 16/18] make tests on windows hopefully not timeout --- packages/bun-internal-test/src/runner.node.mjs | 2 +- test/cli/hot/hot.test.ts | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/bun-internal-test/src/runner.node.mjs b/packages/bun-internal-test/src/runner.node.mjs index 9eba40bb8eddb..70b7b372f2150 100644 --- a/packages/bun-internal-test/src/runner.node.mjs +++ b/packages/bun-internal-test/src/runner.node.mjs @@ -186,7 +186,7 @@ function checkSlowTests() { ); proc?.stdout?.destroy?.(); proc?.stderr?.destroy?.(); - proc?.kill?.(); + proc?.kill?.(9); } else if (now - start > SHORT_TIMEOUT_DURATION) { console.error( `\x1b[33mwarning\x1b[0;2m:\x1b[0m Test ${JSON.stringify(path)} has been running for ${Math.ceil( diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index 3090415d64b2e..bd17faff92dd5 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -1,6 +1,6 @@ import { spawn } from "bun"; -import { beforeAll, beforeEach, expect, it } from "bun:test"; -import { bunExe, bunEnv, tmpdirSync, isDebug } from "harness"; +import { beforeEach, expect, it } from "bun:test"; +import { bunExe, bunEnv, tmpdirSync, isDebug, isWindows } from "harness"; import { cpSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync, copyFileSync } from "fs"; import { join } from "path"; @@ -359,7 +359,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, if (!line.includes("error")) continue; str = ""; - if (reloadCounter === 50) { + if (reloadCounter === (isWindows ? 10 : 50)) { runner.kill(); break; } @@ -383,7 +383,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, if (any) await onReload(); } await runner.exited; - expect(reloadCounter).toBe(50); + expect(reloadCounter).toBe(isWindows ? 10 : 50); }, isDebug ? Infinity : 10_000, ); @@ -436,7 +436,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, if (!line.includes("error")) continue; str = ""; - if (reloadCounter === 50) { + if (reloadCounter === (isWindows ? 10 : 50)) { runner.kill(); break; } @@ -457,7 +457,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, if (any) await onReload(); } - expect(reloadCounter).toBe(50); + expect(reloadCounter).toBe(isWindows ? 10 : 50); bundler.kill(); }); @@ -538,7 +538,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, sampleMemory10 = rss; } - if (reloadCounter === 50) { + if (reloadCounter >= (isWindows ? 10 : 50)) { sampleMemory100 = rss; runner.kill(); break; From e877090858abcd8d5a4b582652b82c7e347396b4 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Tue, 21 May 2024 12:47:13 -0700 Subject: [PATCH 17/18] a --- test/cli/hot/hot.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index bd17faff92dd5..db9045cee2ed7 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -402,8 +402,8 @@ throw new Error('0');`, cmd: [bunExe(), "build", "--watch", bundleIn, "--target=bun", "--sourcemap", "--outfile", hotRunnerRoot], env: bunEnv, cwd, - stdout: "inherit", - stderr: "inherit", + stdout: "pipe", + stderr: "pipe", stdin: "ignore", }); await using runner = spawn({ From 90decee141b6a4d2addfdc46254f40f7da854dbc Mon Sep 17 00:00:00 2001 From: dave caruso Date: Tue, 21 May 2024 14:32:07 -0700 Subject: [PATCH 18/18] t --- test/cli/hot/hot.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index db9045cee2ed7..c699638fb33ec 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -359,7 +359,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, if (!line.includes("error")) continue; str = ""; - if (reloadCounter === (isWindows ? 10 : 50)) { + if (reloadCounter === 50) { runner.kill(); break; } @@ -383,7 +383,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, if (any) await onReload(); } await runner.exited; - expect(reloadCounter).toBe(isWindows ? 10 : 50); + expect(reloadCounter).toBe(50); }, isDebug ? Infinity : 10_000, ); @@ -436,7 +436,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, if (!line.includes("error")) continue; str = ""; - if (reloadCounter === (isWindows ? 10 : 50)) { + if (reloadCounter === 50) { runner.kill(); break; } @@ -457,7 +457,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, if (any) await onReload(); } - expect(reloadCounter).toBe(isWindows ? 10 : 50); + expect(reloadCounter).toBe(50); bundler.kill(); }); @@ -538,7 +538,7 @@ ${" ".repeat(reloadCounter * 2)}throw new Error(${reloadCounter});`, sampleMemory10 = rss; } - if (reloadCounter >= (isWindows ? 10 : 50)) { + if (reloadCounter >= 50) { sampleMemory100 = rss; runner.kill(); break;