Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow generating sourcemaps for Lua in the same format as JS #11454

Merged
merged 1 commit into from
Dec 29, 2023
Merged

Allow generating sourcemaps for Lua in the same format as JS #11454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
151 changes: 9 additions & 142 deletions src/generators/genjs.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,7 @@ open Globals
open Ast
open Type
open Common

type sourcemap = {
sources : (string) DynArray.t;
sources_hash : (string, int) Hashtbl.t;
mappings : Rbuffer.t;

mutable source_last_pos : sourcemap_pos;
mutable print_comma : bool;
mutable output_last_col : int;
mutable output_current_col : int;
mutable current_expr : sourcemap_pos option;
}

and sourcemap_pos = {
file : int;
line : int;
col : int;
}
open JsSourcemap

type ctx = {
com : Common.context;
Expand Down Expand Up @@ -169,98 +152,6 @@ let add_feature ctx = Common.add_feature ctx.com

let unsupported p = abort "This expression cannot be compiled to Javascript" p

let encode_mapping smap pos =
if smap.print_comma then
Rbuffer.add_char smap.mappings ','
else
smap.print_comma <- true;

let base64_vlq number =
let encode_digit digit =
let chars = [|
'A';'B';'C';'D';'E';'F';'G';'H';'I';'J';'K';'L';'M';'N';'O';'P';
'Q';'R';'S';'T';'U';'V';'W';'X';'Y';'Z';'a';'b';'c';'d';'e';'f';
'g';'h';'i';'j';'k';'l';'m';'n';'o';'p';'q';'r';'s';'t';'u';'v';
'w';'x';'y';'z';'0';'1';'2';'3';'4';'5';'6';'7';'8';'9';'+';'/'
|] in
Array.unsafe_get chars digit
in
let to_vlq number =
if number < 0 then
((-number) lsl 1) + 1
else
number lsl 1
in
let rec loop vlq =
let shift = 5 in
let base = 1 lsl shift in
let mask = base - 1 in
let continuation_bit = base in
let digit = vlq land mask in
let next = vlq asr shift in
Rbuffer.add_char smap.mappings (encode_digit (
if next > 0 then digit lor continuation_bit else digit));
if next > 0 then loop next else ()
in
loop (to_vlq number)
in

base64_vlq (smap.output_current_col - smap.output_last_col);
base64_vlq (pos.file - smap.source_last_pos.file);
base64_vlq (pos.line - smap.source_last_pos.line);
base64_vlq (pos.col - smap.source_last_pos.col);

smap.source_last_pos <- pos;
smap.output_last_col <- smap.output_current_col

let noop () = ()

let add_mapping smap pos =
if pos.pmin < 0 then noop else

let file = try
Hashtbl.find smap.sources_hash pos.pfile
with Not_found ->
let length = DynArray.length smap.sources in
Hashtbl.replace smap.sources_hash pos.pfile length;
DynArray.add smap.sources pos.pfile;
length
in

let pos =
let line, col = Lexer.find_pos pos in
let line = line - 1 in
{ file = file; line = line; col = col }
in

if smap.source_last_pos <> pos then begin
let old_current_expr = smap.current_expr in
smap.current_expr <- Some pos;
encode_mapping smap pos;
(fun () -> smap.current_expr <- old_current_expr)
end else
noop

let add_mapping ctx e =
Option.map_default (fun smap -> add_mapping smap e.epos) noop ctx.smap

let handle_newlines ctx str =
Option.may (fun smap ->
let rec loop from =
try begin
let next = String.index_from str from '\n' + 1 in
Rbuffer.add_char smap.mappings ';';
smap.output_last_col <- 0;
smap.output_current_col <- 0;
smap.print_comma <- false;
Option.may (encode_mapping smap) smap.current_expr;
loop next
end with Not_found ->
smap.output_current_col <- smap.output_current_col + (String.length str - from);
in
loop 0
) ctx.smap

let flush ctx =
let chan =
match ctx.chan with
Expand All @@ -275,43 +166,16 @@ let flush ctx =

let spr ctx s =
ctx.separator <- false;
handle_newlines ctx s;
handle_newlines ctx.smap s;
Rbuffer.add_string ctx.buf s

let print ctx =
ctx.separator <- false;
Printf.kprintf (fun s -> begin
handle_newlines ctx s;
handle_newlines ctx.smap s;
Rbuffer.add_string ctx.buf s
end)

let write_mappings ctx smap =
let basefile = Filename.basename ctx.com.file in
print ctx "\n//# sourceMappingURL=%s.map" (url_encode_s basefile);
let channel = open_out_bin (ctx.com.file ^ ".map") in
let sources = DynArray.to_list smap.sources in
let to_url file =
ExtString.String.map (fun c -> if c == '\\' then '/' else c) (Path.get_full_path file)
in
output_string channel "{\n";
output_string channel "\"version\":3,\n";
output_string channel ("\"file\":\"" ^ (String.concat "\\\\" (ExtString.String.nsplit basefile "\\")) ^ "\",\n");
output_string channel ("\"sourceRoot\":\"\",\n");
output_string channel ("\"sources\":[" ^
(String.concat "," (List.map (fun s -> "\"file:///" ^ to_url s ^ "\"") sources)) ^
"],\n");
if Common.defined ctx.com Define.SourceMapContent then begin
output_string channel ("\"sourcesContent\":[" ^
(String.concat "," (List.map (fun s -> try "\"" ^ StringHelper.s_escape (Std.input_file ~bin:true s) ^ "\"" with _ -> "null") sources)) ^
"],\n");
end;
output_string channel "\"names\":[],\n";
output_string channel "\"mappings\":\"";
Rbuffer.output_buffer channel smap.mappings;
output_string channel "\"\n";
output_string channel "}";
close_out channel

let newline ctx =
match Rbuffer.nth ctx.buf (Rbuffer.length ctx.buf - 1) with
| '}' | '{' | ':' | ';' when not ctx.separator -> print ctx "\n%s" ctx.tabs
Expand Down Expand Up @@ -613,7 +477,7 @@ and add_objectdecl_parens e =
loop e

and gen_expr ctx e =
let clear_mapping = add_mapping ctx e in
let clear_mapping = add_mapping ctx.smap e in
(match e.eexpr with
| TConst c -> gen_constant ctx e.epos c
| TLocal v -> spr ctx (ident v.v_name)
Expand Down Expand Up @@ -978,7 +842,7 @@ and gen_block_element ?(newline_after=false) ?(keep_blocks=false) ctx e =
if newline_after then newline ctx

and gen_value ctx e =
let clear_mapping = add_mapping ctx e in
let clear_mapping = add_mapping ctx.smap e in
let assign e =
mk (TBinop (Ast.OpAssign,
mk (TLocal (match ctx.in_value with None -> die "" __LOC__ | Some v -> v)) t_dynamic e.epos,
Expand Down Expand Up @@ -2138,7 +2002,10 @@ let generate com =
);

(match ctx.smap with
| Some smap -> write_mappings ctx smap
| Some smap ->
write_mappings ctx.com smap "file:///";
let basefile = Filename.basename com.file in
print ctx "\n//# sourceMappingURL=%s.map" (url_encode_s basefile);
| None -> try Sys.remove (com.file ^ ".map") with _ -> ());
flush ctx;
Option.may (fun chan -> close_out chan) ctx.chan
Expand Down
40 changes: 37 additions & 3 deletions src/generators/genlua.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,21 @@
* DEALINGS IN THE SOFTWARE.
*)

open Extlib_leftovers
open Ast
open Type
open Common
open ExtList
open Error
open JsSourcemap

type pos = Globals.pos

type ctx = {
com : Common.context;
buf : Buffer.t;
packages : (string list,unit) Hashtbl.t;
smap : sourcemap option;
mutable current : tclass;
mutable statics : (tclass * tclass_field * texpr) list;
mutable inits : texpr list;
Expand Down Expand Up @@ -143,11 +146,15 @@ let temp ctx =

let spr ctx s =
ctx.separator <- false;
handle_newlines ctx.smap s;
Buffer.add_string ctx.buf s

let print ctx =
ctx.separator <- false;
Printf.kprintf (fun s -> Buffer.add_string ctx.buf s)
Printf.kprintf (fun s -> begin
handle_newlines ctx.smap s;
Buffer.add_string ctx.buf s
end)

let newline ctx = print ctx "\n%s" ctx.tabs

Expand Down Expand Up @@ -662,6 +669,7 @@ and lua_arg_name(a,_) =
| _, _, _ -> ident a.v_name;

and gen_expr ?(local=true) ctx e = begin
let clear_mapping = add_mapping ctx.smap e in
match e.eexpr with
TConst c ->
gen_constant ctx e.epos c;
Expand Down Expand Up @@ -1043,13 +1051,16 @@ and gen_expr ?(local=true) ctx e = begin
| TCast (e1,None) ->
gen_value ctx e1;
| TIdent s ->
spr ctx s
spr ctx s;

clear_mapping ()
end;

(* gen_block_element handles expressions that map to "statements" in lua. *)
(* It handles no-op situations, and ensures that expressions are formatted with newlines *)
and gen_block_element ctx e =
ctx.iife_assign <- false;
let clear_mapping = add_mapping ctx.smap e in
begin match e.eexpr with
| TTypeExpr _ | TConst _ | TLocal _ | TFunction _ ->
()
Expand Down Expand Up @@ -1109,6 +1120,7 @@ and gen_block_element ctx e =
gen_expr ctx e;
semicolon ctx;
end;
clear_mapping ()

and is_const_null e =
match e.eexpr with
Expand Down Expand Up @@ -1147,6 +1159,7 @@ and gen_anon_value ctx e =
gen_value ctx e

and gen_value ctx e =
let clear_mapping = add_mapping ctx.smap e in
let assign e =
mk (TBinop (Ast.OpAssign,
mk (TLocal (match ctx.in_value with None -> Globals.die "" __LOC__ | Some v -> v)) t_dynamic e.epos,
Expand Down Expand Up @@ -1280,7 +1293,8 @@ and gen_value ctx e =
gen_block_element ctx (mk (TTry (block (assign b),
List.map (fun (v,e) -> v, block (assign e)) catchs
)) e.etype e.epos);
v()
v();
clear_mapping ()

and gen_tbinop ctx op e1 e2 =
(match op, e1.eexpr, e2.eexpr with
Expand Down Expand Up @@ -1866,10 +1880,26 @@ let generate_type_forward ctx = function
| TTypeDecl _ | TAbstractDecl _ -> ()

let alloc_ctx com =
let smap =
if com.debug || Common.defined com Define.SourceMap then
Some {
source_last_pos = { file = 0; line = 0; col = 0};
print_comma = false;
output_last_col = 0;
output_current_col = 0;
sources = DynArray.create();
sources_hash = Hashtbl.create 0;
mappings = Rbuffer.create 16;
current_expr = None;
}
else
None
in
let ctx = {
com = com;
buf = Buffer.create 16000;
packages = Hashtbl.create 0;
smap = smap;
statics = [];
inits = [];
current = null_class;
Expand Down Expand Up @@ -2175,6 +2205,10 @@ let generate com =
if anyExposed then
println ctx "return _hx_exports";

(match ctx.smap with
| Some smap -> write_mappings ctx.com smap ""
| None -> try Sys.remove (com.file ^ ".map") with _ -> ());

let ch = open_out_bin com.file in
output_string ch (Buffer.contents ctx.buf);
close_out ch