From 2a71493447b91731cfbcbd3cce18bbdb8949c96a Mon Sep 17 00:00:00 2001 From: John Resig Date: Sun, 17 Apr 2011 13:28:36 -0700 Subject: [PATCH] Updating UglifyJS. --- build/lib/parse-js.js | 278 ++++++++++------ build/lib/process.js | 730 +++++++++++++++++++++++------------------- 2 files changed, 571 insertions(+), 437 deletions(-) diff --git a/build/lib/parse-js.js b/build/lib/parse-js.js index 7e4fd0e272..9f90dfeece 100644 --- a/build/lib/parse-js.js +++ b/build/lib/parse-js.js @@ -182,8 +182,6 @@ var OPERATORS = array_to_hash([ ">>=", "<<=", ">>>=", - "~=", - "%=", "|=", "^=", "&=", @@ -191,7 +189,7 @@ var OPERATORS = array_to_hash([ "||" ]); -var WHITESPACE_CHARS = array_to_hash(characters(" \n\r\t")); +var WHITESPACE_CHARS = array_to_hash(characters(" \n\r\t\u200b")); var PUNC_BEFORE_EXPRESSION = array_to_hash(characters("[{}(,.;:")); @@ -201,20 +199,47 @@ var REGEXP_MODIFIERS = array_to_hash(characters("gmsiy")); /* -----[ Tokenizer ]----- */ -function is_alphanumeric_char(ch) { - ch = ch.charCodeAt(0); - return (ch >= 48 && ch <= 57) || - (ch >= 65 && ch <= 90) || - (ch >= 97 && ch <= 122); +// regexps adapted from http://xregexp.com/plugins/#unicode +var UNICODE = { + letter: new RegExp("[\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376\\u0377\\u037A-\\u037D\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u0523\\u0531-\\u0556\\u0559\\u0561-\\u0587\\u05D0-\\u05EA\\u05F0-\\u05F2\\u0621-\\u064A\\u066E\\u066F\\u0671-\\u06D3\\u06D5\\u06E5\\u06E6\\u06EE\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4\\u07F5\\u07FA\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971\\u0972\\u097B-\\u097F\\u0985-\\u098C\\u098F\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC\\u09DD\\u09DF-\\u09E1\\u09F0\\u09F1\\u0A05-\\u0A0A\\u0A0F\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32\\u0A33\\u0A35\\u0A36\\u0A38\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0\\u0AE1\\u0B05-\\u0B0C\\u0B0F\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99\\u0B9A\\u0B9C\\u0B9E\\u0B9F\\u0BA3\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C33\\u0C35-\\u0C39\\u0C3D\\u0C58\\u0C59\\u0C60\\u0C61\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDE\\u0CE0\\u0CE1\\u0D05-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D28\\u0D2A-\\u0D39\\u0D3D\\u0D60\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32\\u0E33\\u0E40-\\u0E46\\u0E81\\u0E82\\u0E84\\u0E87\\u0E88\\u0E8A\\u0E8D\\u0E94-\\u0E97\\u0E99-\\u0E9F\\u0EA1-\\u0EA3\\u0EA5\\u0EA7\\u0EAA\\u0EAB\\u0EAD-\\u0EB0\\u0EB2\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC\\u0EDD\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8B\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10D0-\\u10FA\\u10FC\\u1100-\\u1159\\u115F-\\u11A2\\u11A8-\\u11F9\\u1200-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F4\\u1401-\\u166C\\u166F-\\u1676\\u1681-\\u169A\\u16A0-\\u16EA\\u1700-\\u170C\\u170E-\\u1711\\u1720-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1877\\u1880-\\u18A8\\u18AA\\u1900-\\u191C\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19A9\\u19C1-\\u19C7\\u1A00-\\u1A16\\u1B05-\\u1B33\\u1B45-\\u1B4B\\u1B83-\\u1BA0\\u1BAE\\u1BAF\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u2094\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183\\u2184\\u2C00-\\u2C2E\\u2C30-\\u2C5E\\u2C60-\\u2C6F\\u2C71-\\u2C7D\\u2C80-\\u2CE4\\u2D00-\\u2D25\\u2D30-\\u2D65\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005\\u3006\\u3031-\\u3035\\u303B\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312D\\u3131-\\u318E\\u31A0-\\u31B7\\u31F0-\\u31FF\\u3400\\u4DB5\\u4E00\\u9FC3\\uA000-\\uA48C\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A\\uA62B\\uA640-\\uA65F\\uA662-\\uA66E\\uA67F-\\uA697\\uA717-\\uA71F\\uA722-\\uA788\\uA78B\\uA78C\\uA7FB-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA90A-\\uA925\\uA930-\\uA946\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAC00\\uD7A3\\uF900-\\uFA2D\\uFA30-\\uFA6A\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40\\uFB41\\uFB43\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC]"), + non_spacing_mark: new RegExp("[\\u0300-\\u036F\\u0483-\\u0487\\u0591-\\u05BD\\u05BF\\u05C1\\u05C2\\u05C4\\u05C5\\u05C7\\u0610-\\u061A\\u064B-\\u065E\\u0670\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7\\u06E8\\u06EA-\\u06ED\\u0711\\u0730-\\u074A\\u07A6-\\u07B0\\u07EB-\\u07F3\\u0816-\\u0819\\u081B-\\u0823\\u0825-\\u0827\\u0829-\\u082D\\u0900-\\u0902\\u093C\\u0941-\\u0948\\u094D\\u0951-\\u0955\\u0962\\u0963\\u0981\\u09BC\\u09C1-\\u09C4\\u09CD\\u09E2\\u09E3\\u0A01\\u0A02\\u0A3C\\u0A41\\u0A42\\u0A47\\u0A48\\u0A4B-\\u0A4D\\u0A51\\u0A70\\u0A71\\u0A75\\u0A81\\u0A82\\u0ABC\\u0AC1-\\u0AC5\\u0AC7\\u0AC8\\u0ACD\\u0AE2\\u0AE3\\u0B01\\u0B3C\\u0B3F\\u0B41-\\u0B44\\u0B4D\\u0B56\\u0B62\\u0B63\\u0B82\\u0BC0\\u0BCD\\u0C3E-\\u0C40\\u0C46-\\u0C48\\u0C4A-\\u0C4D\\u0C55\\u0C56\\u0C62\\u0C63\\u0CBC\\u0CBF\\u0CC6\\u0CCC\\u0CCD\\u0CE2\\u0CE3\\u0D41-\\u0D44\\u0D4D\\u0D62\\u0D63\\u0DCA\\u0DD2-\\u0DD4\\u0DD6\\u0E31\\u0E34-\\u0E3A\\u0E47-\\u0E4E\\u0EB1\\u0EB4-\\u0EB9\\u0EBB\\u0EBC\\u0EC8-\\u0ECD\\u0F18\\u0F19\\u0F35\\u0F37\\u0F39\\u0F71-\\u0F7E\\u0F80-\\u0F84\\u0F86\\u0F87\\u0F90-\\u0F97\\u0F99-\\u0FBC\\u0FC6\\u102D-\\u1030\\u1032-\\u1037\\u1039\\u103A\\u103D\\u103E\\u1058\\u1059\\u105E-\\u1060\\u1071-\\u1074\\u1082\\u1085\\u1086\\u108D\\u109D\\u135F\\u1712-\\u1714\\u1732-\\u1734\\u1752\\u1753\\u1772\\u1773\\u17B7-\\u17BD\\u17C6\\u17C9-\\u17D3\\u17DD\\u180B-\\u180D\\u18A9\\u1920-\\u1922\\u1927\\u1928\\u1932\\u1939-\\u193B\\u1A17\\u1A18\\u1A56\\u1A58-\\u1A5E\\u1A60\\u1A62\\u1A65-\\u1A6C\\u1A73-\\u1A7C\\u1A7F\\u1B00-\\u1B03\\u1B34\\u1B36-\\u1B3A\\u1B3C\\u1B42\\u1B6B-\\u1B73\\u1B80\\u1B81\\u1BA2-\\u1BA5\\u1BA8\\u1BA9\\u1C2C-\\u1C33\\u1C36\\u1C37\\u1CD0-\\u1CD2\\u1CD4-\\u1CE0\\u1CE2-\\u1CE8\\u1CED\\u1DC0-\\u1DE6\\u1DFD-\\u1DFF\\u20D0-\\u20DC\\u20E1\\u20E5-\\u20F0\\u2CEF-\\u2CF1\\u2DE0-\\u2DFF\\u302A-\\u302F\\u3099\\u309A\\uA66F\\uA67C\\uA67D\\uA6F0\\uA6F1\\uA802\\uA806\\uA80B\\uA825\\uA826\\uA8C4\\uA8E0-\\uA8F1\\uA926-\\uA92D\\uA947-\\uA951\\uA980-\\uA982\\uA9B3\\uA9B6-\\uA9B9\\uA9BC\\uAA29-\\uAA2E\\uAA31\\uAA32\\uAA35\\uAA36\\uAA43\\uAA4C\\uAAB0\\uAAB2-\\uAAB4\\uAAB7\\uAAB8\\uAABE\\uAABF\\uAAC1\\uABE5\\uABE8\\uABED\\uFB1E\\uFE00-\\uFE0F\\uFE20-\\uFE26]"), + space_combining_mark: new RegExp("[\\u0903\\u093E-\\u0940\\u0949-\\u094C\\u094E\\u0982\\u0983\\u09BE-\\u09C0\\u09C7\\u09C8\\u09CB\\u09CC\\u09D7\\u0A03\\u0A3E-\\u0A40\\u0A83\\u0ABE-\\u0AC0\\u0AC9\\u0ACB\\u0ACC\\u0B02\\u0B03\\u0B3E\\u0B40\\u0B47\\u0B48\\u0B4B\\u0B4C\\u0B57\\u0BBE\\u0BBF\\u0BC1\\u0BC2\\u0BC6-\\u0BC8\\u0BCA-\\u0BCC\\u0BD7\\u0C01-\\u0C03\\u0C41-\\u0C44\\u0C82\\u0C83\\u0CBE\\u0CC0-\\u0CC4\\u0CC7\\u0CC8\\u0CCA\\u0CCB\\u0CD5\\u0CD6\\u0D02\\u0D03\\u0D3E-\\u0D40\\u0D46-\\u0D48\\u0D4A-\\u0D4C\\u0D57\\u0D82\\u0D83\\u0DCF-\\u0DD1\\u0DD8-\\u0DDF\\u0DF2\\u0DF3\\u0F3E\\u0F3F\\u0F7F\\u102B\\u102C\\u1031\\u1038\\u103B\\u103C\\u1056\\u1057\\u1062-\\u1064\\u1067-\\u106D\\u1083\\u1084\\u1087-\\u108C\\u108F\\u109A-\\u109C\\u17B6\\u17BE-\\u17C5\\u17C7\\u17C8\\u1923-\\u1926\\u1929-\\u192B\\u1930\\u1931\\u1933-\\u1938\\u19B0-\\u19C0\\u19C8\\u19C9\\u1A19-\\u1A1B\\u1A55\\u1A57\\u1A61\\u1A63\\u1A64\\u1A6D-\\u1A72\\u1B04\\u1B35\\u1B3B\\u1B3D-\\u1B41\\u1B43\\u1B44\\u1B82\\u1BA1\\u1BA6\\u1BA7\\u1BAA\\u1C24-\\u1C2B\\u1C34\\u1C35\\u1CE1\\u1CF2\\uA823\\uA824\\uA827\\uA880\\uA881\\uA8B4-\\uA8C3\\uA952\\uA953\\uA983\\uA9B4\\uA9B5\\uA9BA\\uA9BB\\uA9BD-\\uA9C0\\uAA2F\\uAA30\\uAA33\\uAA34\\uAA4D\\uAA7B\\uABE3\\uABE4\\uABE6\\uABE7\\uABE9\\uABEA\\uABEC]"), + connector_punctuation: new RegExp("[\\u005F\\u203F\\u2040\\u2054\\uFE33\\uFE34\\uFE4D-\\uFE4F\\uFF3F]") }; -function is_identifier_char(ch) { - return is_alphanumeric_char(ch) || ch == "$" || ch == "_"; +function is_letter(ch) { + return UNICODE.letter.test(ch); }; function is_digit(ch) { ch = ch.charCodeAt(0); - return ch >= 48 && ch <= 57; + return ch >= 48 && ch <= 57; //XXX: find out if "UnicodeDigit" means something else than 0..9 +}; + +function is_alphanumeric_char(ch) { + return is_digit(ch) || is_letter(ch); +}; + +function is_unicode_combining_mark(ch) { + return UNICODE.non_spacing_mark.test(ch) || UNICODE.space_combining_mark.test(ch); +}; + +function is_unicode_connector_punctuation(ch) { + return UNICODE.connector_punctuation.test(ch); +}; + +function is_identifier_start(ch) { + return ch == "$" || ch == "_" || is_letter(ch); +}; + +function is_identifier_char(ch) { + return is_identifier_start(ch) + || is_unicode_combining_mark(ch) + || is_digit(ch) + || is_unicode_connector_punctuation(ch) + || ch == "\u200c" // zero-width non-joiner + || ch == "\u200d" // zero-width joiner (in my ECMA-262 PDF, this is also 200c) + ; }; function parse_js_number(num) { @@ -253,18 +278,19 @@ function is_token(token, type, val) { var EX_EOF = {}; -function tokenizer($TEXT, skip_comments) { +function tokenizer($TEXT) { var S = { - text : $TEXT.replace(/\r\n?|[\n\u2028\u2029]/g, "\n").replace(/^\uFEFF/, ''), - pos : 0, - tokpos : 0, - line : 0, - tokline : 0, - col : 0, - tokcol : 0, - newline_before : false, - regex_allowed : false + text : $TEXT.replace(/\r\n?|[\n\u2028\u2029]/g, "\n").replace(/^\uFEFF/, ''), + pos : 0, + tokpos : 0, + line : 0, + tokline : 0, + col : 0, + tokcol : 0, + newline_before : false, + regex_allowed : false, + comments_before : [] }; function peek() { return S.text.charAt(S.pos); }; @@ -299,7 +325,7 @@ function tokenizer($TEXT, skip_comments) { S.tokpos = S.pos; }; - function token(type, value) { + function token(type, value, is_comment) { S.regex_allowed = ((type == "operator" && !HOP(UNARY_POSTFIX, value)) || (type == "keyword" && HOP(KEYWORDS_BEFORE_EXPRESSION, value)) || (type == "punc" && HOP(PUNC_BEFORE_EXPRESSION, value))); @@ -311,6 +337,10 @@ function tokenizer($TEXT, skip_comments) { pos : S.tokpos, nlb : S.newline_before }; + if (!is_comment) { + ret.comments_before = S.comments_before; + S.comments_before = []; + } S.newline_before = false; return ret; }; @@ -351,7 +381,7 @@ function tokenizer($TEXT, skip_comments) { if (ch == "+") return after_e; after_e = false; if (ch == ".") { - if (!has_dot) + if (!has_dot && !has_x) return has_dot = true; return false; } @@ -417,7 +447,7 @@ function tokenizer($TEXT, skip_comments) { ret = S.text.substring(S.pos, i); S.pos = i; } - return token("comment1", ret); + return token("comment1", ret, true); }; function read_multiline_comment() { @@ -425,14 +455,41 @@ function tokenizer($TEXT, skip_comments) { return with_eof_error("Unterminated multiline comment", function(){ var i = find("*/", true), text = S.text.substring(S.pos, i), - tok = token("comment2", text); + tok = token("comment2", text, true); S.pos = i + 2; S.line += text.split("\n").length - 1; S.newline_before = text.indexOf("\n") >= 0; + + // https://github.com/mishoo/UglifyJS/issues/#issue/100 + if (/^@cc_on/i.test(text)) { + warn("WARNING: at line " + S.line); + warn("*** Found \"conditional comment\": " + text); + warn("*** UglifyJS DISCARDS ALL COMMENTS. This means your code might no longer work properly in Internet Explorer."); + } + return tok; }); }; + function read_name() { + var backslash = false, name = "", ch; + while ((ch = peek()) != null) { + if (!backslash) { + if (ch == "\\") backslash = true, next(); + else if (is_identifier_char(ch)) name += next(); + else break; + } + else { + if (ch != "u") parse_error("Expecting UnicodeEscapeSequence -- uXXXX"); + ch = read_escaped_char(); + if (!is_identifier_char(ch)) parse_error("Unicode char: " + ch.charCodeAt(0) + " is not valid in identifier"); + name += ch; + backslash = false; + } + } + return name; + }; + function read_regexp() { return with_eof_error("Unterminated regular expression", function(){ var prev_backslash = false, regexp = "", ch, in_class = false; @@ -452,15 +509,14 @@ function tokenizer($TEXT, skip_comments) { } else { regexp += ch; } - var mods = read_while(function(ch){ - return HOP(REGEXP_MODIFIERS, ch); - }); + var mods = read_name(); return token("regexp", [ regexp, mods ]); }); }; function read_operator(prefix) { function grow(op) { + if (!peek()) return op; var bigger = op + peek(); if (HOP(OPERATORS, bigger)) { next(); @@ -472,19 +528,18 @@ function tokenizer($TEXT, skip_comments) { return token("operator", grow(prefix || next())); }; - var handle_slash = skip_comments ? function() { + function handle_slash() { next(); var regex_allowed = S.regex_allowed; switch (peek()) { - case "/": read_line_comment(); S.regex_allowed = regex_allowed; return next_token(); - case "*": read_multiline_comment(); S.regex_allowed = regex_allowed; return next_token(); - } - return S.regex_allowed ? read_regexp() : read_operator("/"); - } : function() { - next(); - switch (peek()) { - case "/": return read_line_comment(); - case "*": return read_multiline_comment(); + case "/": + S.comments_before.push(read_line_comment()); + S.regex_allowed = regex_allowed; + return next_token(); + case "*": + S.comments_before.push(read_multiline_comment()); + S.regex_allowed = regex_allowed; + return next_token(); } return S.regex_allowed ? read_regexp() : read_operator("/"); }; @@ -497,7 +552,7 @@ function tokenizer($TEXT, skip_comments) { }; function read_word() { - var word = read_while(is_identifier_char); + var word = read_name(); return !HOP(KEYWORDS, word) ? token("name", word) : HOP(OPERATORS, word) @@ -529,7 +584,7 @@ function tokenizer($TEXT, skip_comments) { if (ch == ".") return handle_dot(); if (ch == "/") return handle_slash(); if (HOP(OPERATOR_CHARS, ch)) return read_operator(); - if (is_identifier_char(ch)) return read_word(); + if (ch == "\\" || is_identifier_start(ch)) return read_word(); parse_error("Unexpected character '" + ch + "'"); }; @@ -565,7 +620,7 @@ var ASSIGNMENT = (function(a, ret, i){ } return ret; })( - ["+=", "-=", "/=", "*=", "%=", ">>=", "<<=", ">>>=", "~=", "%=", "|=", "^=", "&="], + ["+=", "-=", "/=", "*=", "%=", ">>=", "<<=", ">>>=", "|=", "^=", "&="], { "=": true }, 0 ); @@ -608,16 +663,16 @@ function NodeWithToken(str, start, end) { NodeWithToken.prototype.toString = function() { return this.name; }; -function parse($TEXT, strict_mode, embed_tokens) { +function parse($TEXT, exigent_mode, embed_tokens) { var S = { - input: tokenizer($TEXT, true), - token: null, - prev: null, - peeked: null, - in_function: 0, - in_loop: 0, - labels: [] + input : typeof $TEXT == "string" ? tokenizer($TEXT, true) : $TEXT, + token : null, + prev : null, + peeked : null, + in_function : 0, + in_loop : 0, + labels : [] }; S.token = next(); @@ -671,7 +726,7 @@ function parse($TEXT, strict_mode, embed_tokens) { function expect(punc) { return expect_token("punc", punc); }; function can_insert_semicolon() { - return !strict_mode && ( + return !exigent_mode && ( S.token.nlb || is("eof") || is("punc", "}") ); }; @@ -693,14 +748,14 @@ function parse($TEXT, strict_mode, embed_tokens) { }; function add_tokens(str, start, end) { - return new NodeWithToken(str, start, end); + return str instanceof NodeWithToken ? str : new NodeWithToken(str, start, end); }; var statement = embed_tokens ? function() { var start = S.token; - var stmt = $statement(); - stmt[0] = add_tokens(stmt[0], start, prev()); - return stmt; + var ast = $statement.apply(this, arguments); + ast[0] = add_tokens(ast[0], start, prev()); + return ast; } : $statement; function $statement() { @@ -802,7 +857,7 @@ function parse($TEXT, strict_mode, embed_tokens) { function labeled_statement(label) { S.labels.push(label); var start = S.token, stat = statement(); - if (strict_mode && !HOP(STATEMENTS_WITH_LABELS, stat[0])) + if (exigent_mode && !HOP(STATEMENTS_WITH_LABELS, stat[0])) unexpected(start); S.labels.pop(); return as("label", label, stat); @@ -827,29 +882,42 @@ function parse($TEXT, strict_mode, embed_tokens) { function for_() { expect("("); - var has_var = is("keyword", "var"); - if (has_var) - next(); - if (is("name") && is_token(peek(), "operator", "in")) { - // for (i in foo) - var name = S.token.value; - next(); next(); - var obj = expression(); - expect(")"); - return as("for-in", has_var, name, obj, in_loop(statement)); - } else { - // classic for - var init = is("punc", ";") ? null : has_var ? var_() : expression(); - expect(";"); - var test = is("punc", ";") ? null : expression(); - expect(";"); - var step = is("punc", ")") ? null : expression(); - expect(")"); - return as("for", init, test, step, in_loop(statement)); + var init = null; + if (!is("punc", ";")) { + init = is("keyword", "var") + ? (next(), var_(true)) + : expression(true, true); + if (is("operator", "in")) + return for_in(init); } + return regular_for(init); + }; + + function regular_for(init) { + expect(";"); + var test = is("punc", ";") ? null : expression(); + expect(";"); + var step = is("punc", ")") ? null : expression(); + expect(")"); + return as("for", init, test, step, in_loop(statement)); + }; + + function for_in(init) { + var lhs = init[0] == "var" ? as("name", init[1][0]) : init; + next(); + var obj = expression(); + expect(")"); + return as("for-in", init, lhs, obj, in_loop(statement)); }; - function function_(in_statement) { + var function_ = embed_tokens ? function() { + var start = prev(); + var ast = $function_.apply(this, arguments); + ast[0] = add_tokens(ast[0], start, prev()); + return ast; + } : $function_; + + function $function_(in_statement) { var name = is("name") ? prog1(S.token.value, next) : null; if (in_statement && !name) unexpected(); @@ -946,7 +1014,7 @@ function parse($TEXT, strict_mode, embed_tokens) { return as("try", body, bcatch, bfinally); }; - function vardefs() { + function vardefs(no_in) { var a = []; for (;;) { if (!is("name")) @@ -955,7 +1023,7 @@ function parse($TEXT, strict_mode, embed_tokens) { next(); if (is("operator", "=")) { next(); - a.push([ name, expression(false) ]); + a.push([ name, expression(false, no_in) ]); } else { a.push([ name ]); } @@ -966,8 +1034,8 @@ function parse($TEXT, strict_mode, embed_tokens) { return a; }; - function var_() { - return as("var", vardefs()); + function var_(no_in) { + return as("var", vardefs(no_in)); }; function const_() { @@ -1022,27 +1090,30 @@ function parse($TEXT, strict_mode, embed_tokens) { unexpected(); }; - function expr_list(closing, allow_trailing_comma) { + function expr_list(closing, allow_trailing_comma, allow_empty) { var first = true, a = []; while (!is("punc", closing)) { if (first) first = false; else expect(","); - if (allow_trailing_comma && is("punc", closing)) - break; - a.push(expression(false)); + if (allow_trailing_comma && is("punc", closing)) break; + if (is("punc", ",") && allow_empty) { + a.push([ "atom", "undefined" ]); + } else { + a.push(expression(false)); + } } next(); return a; }; function array_() { - return as("array", expr_list("]", !strict_mode)); + return as("array", expr_list("]", !exigent_mode, true)); }; function object_() { var first = true, a = []; while (!is("punc", "}")) { if (first) first = false; else expect(","); - if (!strict_mode && is("punc", "}")) + if (!exigent_mode && is("punc", "}")) // allow trailing comma break; var type = S.token.type; @@ -1105,61 +1176,65 @@ function parse($TEXT, strict_mode, embed_tokens) { return as(tag, op, expr); }; - function expr_op(left, min_prec) { + function expr_op(left, min_prec, no_in) { var op = is("operator") ? S.token.value : null; + if (op && op == "in" && no_in) op = null; var prec = op != null ? PRECEDENCE[op] : null; if (prec != null && prec > min_prec) { next(); - var right = expr_op(expr_atom(true), prec); - return expr_op(as("binary", op, left, right), min_prec); + var right = expr_op(expr_atom(true), prec, no_in); + return expr_op(as("binary", op, left, right), min_prec, no_in); } return left; }; - function expr_ops() { - return expr_op(expr_atom(true), 0); + function expr_ops(no_in) { + return expr_op(expr_atom(true), 0, no_in); }; - function maybe_conditional() { - var expr = expr_ops(); + function maybe_conditional(no_in) { + var expr = expr_ops(no_in); if (is("operator", "?")) { next(); var yes = expression(false); expect(":"); - return as("conditional", expr, yes, expression(false)); + return as("conditional", expr, yes, expression(false, no_in)); } return expr; }; function is_assignable(expr) { + if (!exigent_mode) return true; switch (expr[0]) { case "dot": case "sub": + case "new": + case "call": return true; case "name": return expr[1] != "this"; } }; - function maybe_assign() { - var left = maybe_conditional(), val = S.token.value; + function maybe_assign(no_in) { + var left = maybe_conditional(no_in), val = S.token.value; if (is("operator") && HOP(ASSIGNMENT, val)) { if (is_assignable(left)) { next(); - return as("assign", ASSIGNMENT[val], left, maybe_assign()); + return as("assign", ASSIGNMENT[val], left, maybe_assign(no_in)); } croak("Invalid assignment"); } return left; }; - function expression(commas) { + function expression(commas, no_in) { if (arguments.length == 0) commas = true; - var expr = maybe_assign(); + var expr = maybe_assign(no_in); if (commas && is("punc", ",")) { next(); - return as("seq", expr, expression()); + return as("seq", expr, expression(true, no_in)); } return expr; }; @@ -1222,6 +1297,8 @@ function HOP(obj, prop) { return Object.prototype.hasOwnProperty.call(obj, prop); }; +var warn = function() {}; + /* -----[ Exports ]----- */ exports.tokenizer = tokenizer; @@ -1237,3 +1314,6 @@ exports.KEYWORDS = KEYWORDS; exports.ATOMIC_START_TOKEN = ATOMIC_START_TOKEN; exports.OPERATORS = OPERATORS; exports.is_alphanumeric_char = is_alphanumeric_char; +exports.set_logger = function(logger) { + warn = logger; +}; diff --git a/build/lib/process.js b/build/lib/process.js index edcf599dc8..09cbc2ad8e 100644 --- a/build/lib/process.js +++ b/build/lib/process.js @@ -10,14 +10,13 @@ Exported functions: - - ast_mangle(ast, include_toplevel) -- mangles the - variable/function names in the AST. Returns an AST. Pass true - as second argument to mangle toplevel names too. + - ast_mangle(ast, options) -- mangles the variable/function names + in the AST. Returns an AST. - ast_squeeze(ast) -- employs various optimizations to make the final generated code even smaller. Returns an AST. - - gen_code(ast, beautify) -- generates JS code from the AST. Pass + - gen_code(ast, options) -- generates JS code from the AST. Pass true (or an object, see the code for some options) as second argument to get "pretty" (indented) code. @@ -69,139 +68,135 @@ var jsp = require("./parse-js"), function ast_walker(ast) { function _vardefs(defs) { - return MAP(defs, function(def){ + return [ this[0], MAP(defs, function(def){ var a = [ def[0] ]; if (def.length > 1) a[1] = walk(def[1]); return a; - }); + }) ]; }; var walkers = { "string": function(str) { - return [ "string", str ]; + return [ this[0], str ]; }, "num": function(num) { - return [ "num", num ]; + return [ this[0], num ]; }, "name": function(name) { - return [ "name", name ]; + return [ this[0], name ]; }, "toplevel": function(statements) { - return [ "toplevel", MAP(statements, walk) ]; + return [ this[0], MAP(statements, walk) ]; }, "block": function(statements) { - var out = [ "block" ]; + var out = [ this[0] ]; if (statements != null) out.push(MAP(statements, walk)); return out; }, - "var": function(defs) { - return [ "var", _vardefs(defs) ]; - }, - "const": function(defs) { - return [ "const", _vardefs(defs) ]; - }, + "var": _vardefs, + "const": _vardefs, "try": function(t, c, f) { return [ - "try", + this[0], MAP(t, walk), c != null ? [ c[0], MAP(c[1], walk) ] : null, f != null ? MAP(f, walk) : null ]; }, "throw": function(expr) { - return [ "throw", walk(expr) ]; + return [ this[0], walk(expr) ]; }, "new": function(ctor, args) { - return [ "new", walk(ctor), MAP(args, walk) ]; + return [ this[0], walk(ctor), MAP(args, walk) ]; }, "switch": function(expr, body) { - return [ "switch", walk(expr), MAP(body, function(branch){ + return [ this[0], walk(expr), MAP(body, function(branch){ return [ branch[0] ? walk(branch[0]) : null, MAP(branch[1], walk) ]; }) ]; }, "break": function(label) { - return [ "break", label ]; + return [ this[0], label ]; }, "continue": function(label) { - return [ "continue", label ]; + return [ this[0], label ]; }, "conditional": function(cond, t, e) { - return [ "conditional", walk(cond), walk(t), walk(e) ]; + return [ this[0], walk(cond), walk(t), walk(e) ]; }, "assign": function(op, lvalue, rvalue) { - return [ "assign", op, walk(lvalue), walk(rvalue) ]; + return [ this[0], op, walk(lvalue), walk(rvalue) ]; }, "dot": function(expr) { - return [ "dot", walk(expr) ].concat(slice(arguments, 1)); + return [ this[0], walk(expr) ].concat(slice(arguments, 1)); }, "call": function(expr, args) { - return [ "call", walk(expr), MAP(args, walk) ]; + return [ this[0], walk(expr), MAP(args, walk) ]; }, "function": function(name, args, body) { - return [ "function", name, args.slice(), MAP(body, walk) ]; + return [ this[0], name, args.slice(), MAP(body, walk) ]; }, "defun": function(name, args, body) { - return [ "defun", name, args.slice(), MAP(body, walk) ]; + return [ this[0], name, args.slice(), MAP(body, walk) ]; }, "if": function(conditional, t, e) { - return [ "if", walk(conditional), walk(t), walk(e) ]; + return [ this[0], walk(conditional), walk(t), walk(e) ]; }, "for": function(init, cond, step, block) { - return [ "for", walk(init), walk(cond), walk(step), walk(block) ]; + return [ this[0], walk(init), walk(cond), walk(step), walk(block) ]; }, - "for-in": function(has_var, key, hash, block) { - return [ "for-in", has_var, key, walk(hash), walk(block) ]; + "for-in": function(vvar, key, hash, block) { + return [ this[0], walk(vvar), walk(key), walk(hash), walk(block) ]; }, "while": function(cond, block) { - return [ "while", walk(cond), walk(block) ]; + return [ this[0], walk(cond), walk(block) ]; }, "do": function(cond, block) { - return [ "do", walk(cond), walk(block) ]; + return [ this[0], walk(cond), walk(block) ]; }, "return": function(expr) { - return [ "return", walk(expr) ]; + return [ this[0], walk(expr) ]; }, "binary": function(op, left, right) { - return [ "binary", op, walk(left), walk(right) ]; + return [ this[0], op, walk(left), walk(right) ]; }, "unary-prefix": function(op, expr) { - return [ "unary-prefix", op, walk(expr) ]; + return [ this[0], op, walk(expr) ]; }, "unary-postfix": function(op, expr) { - return [ "unary-postfix", op, walk(expr) ]; + return [ this[0], op, walk(expr) ]; }, "sub": function(expr, subscript) { - return [ "sub", walk(expr), walk(subscript) ]; + return [ this[0], walk(expr), walk(subscript) ]; }, "object": function(props) { - return [ "object", MAP(props, function(p){ + return [ this[0], MAP(props, function(p){ return p.length == 2 ? [ p[0], walk(p[1]) ] : [ p[0], walk(p[1]), p[2] ]; // get/set-ter }) ]; }, "regexp": function(rx, mods) { - return [ "regexp", rx, mods ]; + return [ this[0], rx, mods ]; }, "array": function(elements) { - return [ "array", MAP(elements, walk) ]; + return [ this[0], MAP(elements, walk) ]; }, "stat": function(stat) { - return [ "stat", walk(stat) ]; + return [ this[0], walk(stat) ]; }, "seq": function() { - return [ "seq" ].concat(MAP(slice(arguments), walk)); + return [ this[0] ].concat(MAP(slice(arguments), walk)); }, "label": function(name, block) { - return [ "label", name, walk(block) ]; + return [ this[0], name, walk(block) ]; }, "with": function(expr, block) { - return [ "with", walk(expr), walk(block) ]; + return [ this[0], walk(expr), walk(block) ]; }, "atom": function(name) { - return [ "atom", name ]; + return [ this[0], name ]; } }; @@ -260,8 +255,8 @@ function Scope(parent) { this.rev_mangled = {}; // reverse lookup (mangled => orig.name) this.cname = -1; // current mangled name this.refs = {}; // names referenced from this scope - this.uses_with = false; // will become TRUE if eval() is detected in this or any subscopes - this.uses_eval = false; // will become TRUE if with() is detected in this or any subscopes + this.uses_with = false; // will become TRUE if with() is detected in this or any subscopes + this.uses_eval = false; // will become TRUE if eval() is detected in this or any subscopes this.parent = parent; // parent scope this.children = []; // sub-scopes if (parent) { @@ -382,7 +377,7 @@ function ast_add_scope(ast) { }; function _lambda(name, args, body) { - return [ this[0], define(name), args, with_new_scope(function(){ + return [ this[0], this[0] == "defun" ? define(name) : name, args, with_new_scope(function(){ MAP(args, define); return MAP(body, walk); })]; @@ -405,7 +400,7 @@ function ast_add_scope(ast) { }, "try": function(t, c, f) { if (c != null) return [ - "try", + this[0], MAP(t, walk), [ define(c[0]), MAP(c[1], walk) ], f != null ? MAP(f, walk) : null @@ -415,10 +410,6 @@ function ast_add_scope(ast) { if (name == "eval") having_eval.push(current_scope); reference(name); - }, - "for-in": function(has_var, name) { - if (has_var) define(name); - else reference(name); } }, function(){ return walk(ast); @@ -461,11 +452,14 @@ function ast_add_scope(ast) { /* -----[ mangle names ]----- */ -function ast_mangle(ast, do_toplevel) { +function ast_mangle(ast, options) { var w = ast_walker(), walk = w.walk, scope; + options = options || {}; function get_mangled(name, newMangle) { - if (!do_toplevel && !scope.parent) return name; // don't mangle toplevel + if (!options.toplevel && !scope.parent) return name; // don't mangle toplevel + if (options.except && member(name, options.except)) + return name; return scope.get_mangled(name, newMangle); }; @@ -491,9 +485,9 @@ function ast_mangle(ast, do_toplevel) { }; function _vardefs(defs) { - return MAP(defs, function(d){ + return [ this[0], MAP(defs, function(d){ return [ get_mangled(d[0]), walk(d[1]) ]; - }); + }) ]; }; return w.with_walkers({ @@ -510,28 +504,22 @@ function ast_mangle(ast, do_toplevel) { } return ast; }, - "var": function(defs) { - return [ "var", _vardefs(defs) ]; - }, - "const": function(defs) { - return [ "const", _vardefs(defs) ]; - }, + "var": _vardefs, + "const": _vardefs, "name": function(name) { - return [ "name", get_mangled(name) ]; + return [ this[0], get_mangled(name) ]; }, "try": function(t, c, f) { - return [ "try", + return [ this[0], MAP(t, walk), c != null ? [ get_mangled(c[0]), MAP(c[1], walk) ] : null, f != null ? MAP(f, walk) : null ]; }, "toplevel": function(body) { - return with_scope(this.scope, function(){ - return [ "toplevel", MAP(body, walk) ]; + var self = this; + return with_scope(self.scope, function(){ + return [ self[0], MAP(body, walk) ]; }); - }, - "for-in": function(has_var, name, obj, stat) { - return [ "for-in", has_var, get_mangled(name), walk(obj), walk(stat) ]; } }, function() { return walk(ast_add_scope(ast)); @@ -569,28 +557,29 @@ function aborts(t) { } }; -function negate(c) { - var not_c = [ "unary-prefix", "!", c ]; - switch (c[0]) { - case "unary-prefix": - return c[1] == "!" ? c[2] : not_c; - case "binary": - var op = c[1], left = c[2], right = c[3]; - switch (op) { - case "<=": return [ "binary", ">", left, right ]; - case "<": return [ "binary", ">=", left, right ]; - case ">=": return [ "binary", "<", left, right ]; - case ">": return [ "binary", "<=", left, right ]; - case "==": return [ "binary", "!=", left, right ]; - case "!=": return [ "binary", "==", left, right ]; - case "===": return [ "binary", "!==", left, right ]; - case "!==": return [ "binary", "===", left, right ]; - case "&&": return best_of(not_c, [ "binary", "||", negate(left), negate(right) ]); - case "||": return best_of(not_c, [ "binary", "&&", negate(left), negate(right) ]); - } - break; - } - return not_c; +function boolean_expr(expr) { + return ( (expr[0] == "unary-prefix" + && member(expr[1], [ "!", "delete" ])) || + + (expr[0] == "binary" + && member(expr[1], [ "in", "instanceof", "==", "!=", "===", "!==", "<", "<=", ">=", ">" ])) || + + (expr[0] == "binary" + && member(expr[1], [ "&&", "||" ]) + && boolean_expr(expr[2]) + && boolean_expr(expr[3])) || + + (expr[0] == "conditional" + && boolean_expr(expr[2]) + && boolean_expr(expr[3])) || + + (expr[0] == "assign" + && expr[1] === true + && boolean_expr(expr[3])) || + + (expr[0] == "seq" + && boolean_expr(expr[expr.length - 1])) + ); }; function make_conditional(c, t, e) { @@ -605,16 +594,143 @@ function empty(b) { return !b || (b[0] == "block" && (!b[1] || b[1].length == 0)); }; +function is_string(node) { + return (node[0] == "string" || + node[0] == "unary-prefix" && node[1] == "typeof" || + node[0] == "binary" && node[1] == "+" && + (is_string(node[2]) || is_string(node[3]))); +}; + +var when_constant = (function(){ + + var $NOT_CONSTANT = {}; + + // this can only evaluate constant expressions. If it finds anything + // not constant, it throws $NOT_CONSTANT. + function evaluate(expr) { + switch (expr[0]) { + case "string": + case "num": + return expr[1]; + case "name": + case "atom": + switch (expr[1]) { + case "true": return true; + case "false": return false; + } + break; + case "unary-prefix": + switch (expr[1]) { + case "!": return !evaluate(expr[2]); + case "typeof": return typeof evaluate(expr[2]); + case "~": return ~evaluate(expr[2]); + case "-": return -evaluate(expr[2]); + case "+": return +evaluate(expr[2]); + } + break; + case "binary": + var left = expr[2], right = expr[3]; + switch (expr[1]) { + case "&&" : return evaluate(left) && evaluate(right); + case "||" : return evaluate(left) || evaluate(right); + case "|" : return evaluate(left) | evaluate(right); + case "&" : return evaluate(left) & evaluate(right); + case "^" : return evaluate(left) ^ evaluate(right); + case "+" : return evaluate(left) + evaluate(right); + case "*" : return evaluate(left) * evaluate(right); + case "/" : return evaluate(left) / evaluate(right); + case "-" : return evaluate(left) - evaluate(right); + case "<<" : return evaluate(left) << evaluate(right); + case ">>" : return evaluate(left) >> evaluate(right); + case ">>>" : return evaluate(left) >>> evaluate(right); + case "==" : return evaluate(left) == evaluate(right); + case "===" : return evaluate(left) === evaluate(right); + case "!=" : return evaluate(left) != evaluate(right); + case "!==" : return evaluate(left) !== evaluate(right); + case "<" : return evaluate(left) < evaluate(right); + case "<=" : return evaluate(left) <= evaluate(right); + case ">" : return evaluate(left) > evaluate(right); + case ">=" : return evaluate(left) >= evaluate(right); + case "in" : return evaluate(left) in evaluate(right); + case "instanceof" : return evaluate(left) instanceof evaluate(right); + } + } + throw $NOT_CONSTANT; + }; + + return function(expr, yes, no) { + try { + var val = evaluate(expr), ast; + switch (typeof val) { + case "string": ast = [ "string", val ]; break; + case "number": ast = [ "num", val ]; break; + case "boolean": ast = [ "name", String(val) ]; break; + default: throw new Error("Can't handle constant of type: " + (typeof val)); + } + return yes.call(expr, ast, val); + } catch(ex) { + if (ex === $NOT_CONSTANT) { + if (expr[0] == "binary" + && (expr[1] == "===" || expr[1] == "!==") + && ((is_string(expr[2]) && is_string(expr[3])) + || (boolean_expr(expr[2]) && boolean_expr(expr[3])))) { + expr[1] = expr[1].substr(0, 2); + } + return no ? no.call(expr, expr) : null; + } + else throw ex; + } + }; + +})(); + +function warn_unreachable(ast) { + if (!empty(ast)) + warn("Dropping unreachable code: " + gen_code(ast, true)); +}; + function ast_squeeze(ast, options) { options = defaults(options, { make_seqs : true, dead_code : true, - no_warnings : false, - extra : false + keep_comps : true, + no_warnings : false }); var w = ast_walker(), walk = w.walk, scope; + function negate(c) { + var not_c = [ "unary-prefix", "!", c ]; + switch (c[0]) { + case "unary-prefix": + return c[1] == "!" && boolean_expr(c[2]) ? c[2] : not_c; + case "seq": + c = slice(c); + c[c.length - 1] = negate(c[c.length - 1]); + return c; + case "conditional": + return best_of(not_c, [ "conditional", c[1], negate(c[2]), negate(c[3]) ]); + case "binary": + var op = c[1], left = c[2], right = c[3]; + if (!options.keep_comps) switch (op) { + case "<=" : return [ "binary", ">", left, right ]; + case "<" : return [ "binary", ">=", left, right ]; + case ">=" : return [ "binary", "<", left, right ]; + case ">" : return [ "binary", "<=", left, right ]; + } + switch (op) { + case "==" : return [ "binary", "!=", left, right ]; + case "!=" : return [ "binary", "==", left, right ]; + case "===" : return [ "binary", "!==", left, right ]; + case "!==" : return [ "binary", "===", left, right ]; + case "&&" : return best_of(not_c, [ "binary", "||", negate(left), negate(right) ]); + case "||" : return best_of(not_c, [ "binary", "&&", negate(left), negate(right) ]); + } + break; + } + return not_c; + }; + function with_scope(s, cont) { var _scope = scope; scope = s; @@ -624,89 +740,14 @@ function ast_squeeze(ast, options) { return ret; }; - function is_constant(node) { - return node[0] == "string" || node[0] == "num"; - }; - - function find_first_execute(node) { - if (!node) - return false; - - switch (node[0]) { - case "num": - case "string": - case "name": - return node; - case "call": - case "conditional": - case "for": - case "if": - case "new": - case "return": - case "stat": - case "switch": - case "throw": - return find_first_execute(node[1]); - case "binary": - return find_first_execute(node[2]); - case "assign": - if (node[1] === true) - return find_first_execute(node[3]); - break; - case "var": - if (node[1][0].length > 1) - return find_first_execute(node[1][0][1]); - break; - } - return null; - } - - function find_assign_recursive(p, v) { - if (p[0] == "assign" && p[1] != true || p[0] == "unary-prefix") { - if (p[2][0] == "name" && v[0] == "name" && p[2][1] == v[1]) - return true; - return false; - } - - if (p[0] != "assign" || p[1] !== true) - return false; - - if ((is_constant(p[3]) && p[3][0] == v[0] && p[3][1] == v[1]) || - (p[3][0] == "name" && v[0] == "name" && p[3][1] == v[1]) || - (p[2][0] == "name" && v[0] == "name" && p[2][1] == v[1])) - return true; - - return find_assign_recursive(p[3], v); - }; - function rmblock(block) { - if (block != null && block[0] == "block" && block[1] && block[1].length == 1) - block = block[1][0]; - return block; - }; - - function clone(obj) { - if (obj && obj.constructor == Array) - return MAP(obj, clone); - return obj; - }; - - function make_seq_to_statements(node) { - if (node[0] != "seq") { - switch (node[0]) { - case "var": - case "const": - return [ node ]; - default: - return [ [ "stat", node ] ]; - } + if (block != null && block[0] == "block" && block[1]) { + if (block[1].length == 1) + block = block[1][0]; + else if (block[1].length == 0) + block = [ "block" ]; } - - var ret = []; - for (var i = 1; i < node.length; i++) - ret.push.apply(ret, make_seq_to_statements(node[i])); - - return ret; + return block; }; function _lambda(name, args, body) { @@ -734,64 +775,6 @@ function ast_squeeze(ast, options) { return a; }, []); - if (options.extra) { - // Detightening things. We do this because then we can assume that the - // statements are structured in a specific way. - statements = (function(a, prev) { - statements.forEach(function(cur) { - switch (cur[0]) { - case "for": - if (cur[1] != null) { - a.push.apply(a, make_seq_to_statements(cur[1])); - cur[1] = null; - } - a.push(cur); - break; - case "stat": - var stats = make_seq_to_statements(cur[1]); - stats.forEach(function(s) { - if (s[1][0] == "unary-postfix") - s[1][0] = "unary-prefix"; - }); - a.push.apply(a, stats); - break; - default: - a.push(cur); - } - }); - return a; - })([]); - - statements = (function(a, prev) { - statements.forEach(function(cur) { - if (!(prev && prev[0] == "stat")) { - a.push(cur); - prev = cur; - return; - } - - var p = prev[1]; - var c = find_first_execute(cur); - if (c && find_assign_recursive(p, c)) { - var old_cur = clone(cur); - c.splice(0, c.length); - c.push.apply(c, p); - var tmp_cur = best_of(cur, [ "toplevel", [ prev, old_cur ] ]); - if (tmp_cur == cur) { - a[a.length -1] = cur; - } else { - cur = old_cur; - a.push(cur); - } - } else { - a.push(cur); - } - prev = cur; - }); - return a; - })([]); - } - statements = (function(a, prev){ statements.forEach(function(cur){ if (prev && ((cur[0] == "var" && prev[0] == "var") || @@ -812,7 +795,7 @@ function ast_squeeze(ast, options) { a.push(st); } else if (!options.no_warnings) - warn("Removing unreachable code: " + gen_code(st, true)); + warn_unreachable(st); } else { a.push(st); @@ -835,22 +818,6 @@ function ast_squeeze(ast, options) { return a; })([]); - if (options.extra) { - statements = (function(a, prev){ - statements.forEach(function(cur){ - var replaced = false; - if (prev && cur[0] == "for" && cur[1] == null && (prev[0] == "var" || prev[0] == "const" || prev[0] == "stat")) { - cur[1] = prev; - a[a.length - 1] = cur; - } else { - a.push(cur); - } - prev = cur; - }); - return a; - })([]); - } - if (block_type == "lambda") statements = (function(i, a, stat){ while (i < statements.length) { stat = statements[i++]; @@ -874,6 +841,20 @@ function ast_squeeze(ast, options) { }; function make_if(c, t, e) { + return when_constant(c, function(ast, val){ + if (val) { + warn_unreachable(e); + return t; + } else { + warn_unreachable(t); + return e; + } + }, function() { + return make_real_if(c, t, e); + }); + }; + + function make_real_if(c, t, e) { c = walk(c); t = walk(t); e = walk(e); @@ -901,7 +882,10 @@ function ast_squeeze(ast, options) { if (empty(e) && empty(t)) return [ "stat", c ]; var ret = [ "if", c, t, e ]; - if (t[0] == "stat") { + if (t[0] == "if" && empty(t[3]) && empty(e)) { + ret = best_of(ret, walk([ "if", [ "binary", "&&", c, t[1] ], t[2] ])); + } + else if (t[0] == "stat") { if (e) { if (e[0] == "stat") { ret = best_of(ret, [ "stat", make_conditional(c, t[1], e[1]) ]); @@ -911,7 +895,7 @@ function ast_squeeze(ast, options) { ret = best_of(ret, [ "stat", make_conditional(c, t[1]) ]); } } - else if (e && t[0] == e[0] && (t[0] == "return" || t[0] == "throw")) { + else if (e && t[0] == e[0] && (t[0] == "return" || t[0] == "throw") && t[1] && e[1]) { ret = best_of(ret, [ t[0], make_conditional(c, t[1], e[1] ) ]); } else if (e && aborts(t)) { @@ -936,13 +920,25 @@ function ast_squeeze(ast, options) { return ret; }; + function _do_while(cond, body) { + return when_constant(cond, function(cond, val){ + if (!val) { + warn_unreachable(body); + return [ "block" ]; + } else { + return [ "for", null, null, null, walk(body) ]; + } + }); + }; + return w.with_walkers({ "sub": function(expr, subscript) { if (subscript[0] == "string") { var name = subscript[1]; - if (is_identifier(name)) { + if (is_identifier(name)) return [ "dot", walk(expr), name ]; - } + else if (/^[1-9][0-9]*$/.test(name) || name === "0") + return [ "sub", walk(expr), [ "num", parseInt(name, 10) ] ]; } }, "if": make_if, @@ -963,35 +959,23 @@ function ast_squeeze(ast, options) { return [ branch[0] ? walk(branch[0]) : null, block ]; }) ]; }, - "function": _lambda, + "function": function() { + var ret = _lambda.apply(this, arguments); + if (ret[1] && !HOP(scope.refs, ret[1])) { + ret[1] = null; + } + return ret; + }, "defun": _lambda, "block": function(body) { if (body) return rmblock([ "block", tighten(MAP(body, walk)) ]); }, "binary": function(op, left, right) { - left = walk(left); - right = walk(right); - var best = [ "binary", op, left, right ]; - if (is_constant(right)) { - if (is_constant(left)) { - var val = null; - switch (op) { - case "+": val = left[1] + right[1]; break; - case "*": val = left[1] * right[1]; break; - case "/": val = left[1] / right[1]; break; - case "-": val = left[1] - right[1]; break; - case "<<": val = left[1] << right[1]; break; - case ">>": val = left[1] >> right[1]; break; - case ">>>": val = left[1] >>> right[1]; break; - } - if (val != null) { - best = best_of(best, [ typeof val == "string" ? "string" : "num", val ]); - } - } else if (left[0] == "binary" && left[1] == "+" && left[3][0] == "string") { - best = best_of(best, [ "binary", "+", left[2], [ "string", left[3][1] + right[1] ] ]); - } - } - return best; + return when_constant([ "binary", op, walk(left), walk(right) ], function yes(c){ + return best_of(walk(c), this); + }, function no() { + return this; + }); }, "conditional": function(c, t, e) { return make_conditional(walk(c), walk(t), walk(e)); @@ -1004,17 +988,14 @@ function ast_squeeze(ast, options) { f != null ? tighten(MAP(f, walk)) : null ]; }, - "unary-prefix": function(op, cond) { - if (op == "!") { - cond = walk(cond); - if (cond[0] == "unary-prefix" && cond[1] == "!") { - var p = w.parent(); - if (p[0] == "unary-prefix" && p[1] == "!") - return cond[2]; - return [ "unary-prefix", "!", cond ]; - } - return best_of(this, negate(cond)); - } + "unary-prefix": function(op, expr) { + expr = walk(expr); + var ret = [ "unary-prefix", op, expr ]; + if (op == "!") + ret = best_of(ret, negate(expr)); + return when_constant(ret, function(ast, val){ + return walk(ast); // it's either true or false, so minifies to !0 or !1 + }, function() { return ret }); }, "name": function(name) { switch (name) { @@ -1035,7 +1016,9 @@ function ast_squeeze(ast, options) { if (expr[0] == "name" && expr[1] == "Array" && args.length != 1 && !scope.has("Array")) { return [ "array", args ]; } - } + }, + "while": _do_while, + "do": _do_while }, function() { return walk(ast_add_scope(ast)); }); @@ -1046,6 +1029,7 @@ function ast_squeeze(ast, options) { var DOT_CALL_NO_PARENS = jsp.array_to_hash([ "name", "array", + "object", "string", "dot", "sub", @@ -1053,9 +1037,9 @@ var DOT_CALL_NO_PARENS = jsp.array_to_hash([ "regexp" ]); -function make_string(str) { +function make_string(str, ascii_only) { var dq = 0, sq = 0; - str = str.replace(/[\\\b\f\n\r\t\x22\x27]/g, function(s){ + str = str.replace(/[\\\b\f\n\r\t\x22\x27\u2028\u2029]/g, function(s){ switch (s) { case "\\": return "\\\\"; case "\b": return "\\b"; @@ -1063,34 +1047,56 @@ function make_string(str) { case "\n": return "\\n"; case "\r": return "\\r"; case "\t": return "\\t"; + case "\u2028": return "\\u2028"; + case "\u2029": return "\\u2029"; case '"': ++dq; return '"'; case "'": ++sq; return "'"; } return s; }); - if (dq > sq) { - return "'" + str.replace(/\x27/g, "\\'") + "'"; - } else { - return '"' + str.replace(/\x22/g, '\\"') + '"'; - } + if (ascii_only) str = to_ascii(str); + if (dq > sq) return "'" + str.replace(/\x27/g, "\\'") + "'"; + else return '"' + str.replace(/\x22/g, '\\"') + '"'; }; -function gen_code(ast, beautify) { - if (beautify) beautify = defaults(beautify, { +function to_ascii(str) { + return str.replace(/[\u0080-\uffff]/g, function(ch) { + var code = ch.charCodeAt(0).toString(16); + while (code.length < 4) code = "0" + code; + return "\\u" + code; + }); +}; + +function gen_code(ast, options) { + options = defaults(options, { indent_start : 0, indent_level : 4, quote_keys : false, - space_colon : false + space_colon : false, + beautify : false, + ascii_only : false }); + var beautify = !!options.beautify; var indentation = 0, newline = beautify ? "\n" : "", space = beautify ? " " : ""; + function encode_string(str) { + return make_string(str, options.ascii_only); + }; + + function make_name(name) { + name = name.toString(); + if (options.ascii_only) + name = to_ascii(name); + return name; + }; + function indent(line) { if (line == null) line = ""; if (beautify) - line = repeat_string(" ", beautify.indent_start + indentation * beautify.indent_level) + line; + line = repeat_string(" ", options.indent_start + indentation * options.indent_level) + line; return line; }; @@ -1144,7 +1150,7 @@ function gen_code(ast, beautify) { }; function needs_parens(expr) { - if (expr[0] == "function") { + if (expr[0] == "function" || expr[0] == "object") { // dot/call on a literal function requires the // function literal itself to be parenthesized // only if it's the first "thing" in a @@ -1156,9 +1162,8 @@ function gen_code(ast, beautify) { var a = slice($stack), self = a.pop(), p = a.pop(); while (p) { if (p[0] == "stat") return true; - if ((p[0] == "seq" && p[1] === self) || - (p[0] == "call" && p[1] === self) || - (p[0] == "binary" && p[2] === self)) { + if (((p[0] == "seq" || p[0] == "call" || p[0] == "dot" || p[0] == "sub" || p[0] == "conditional") && p[1] === self) || + ((p[0] == "binary" || p[0] == "assign" || p[0] == "unary-postfix") && p[2] === self)) { self = p; p = a.pop(); } else { @@ -1185,7 +1190,7 @@ function gen_code(ast, beautify) { }; var generators = { - "string": make_string, + "string": encode_string, "num": make_num, "name": make_name, "toplevel": function(statements) { @@ -1253,9 +1258,10 @@ function gen_code(ast, beautify) { }, "dot": function(expr) { var out = make(expr), i = 1; - if (expr[0] == "num") - out += "."; - else if (needs_parens(expr)) + if (expr[0] == "num") { + if (!/\./.test(expr[1])) + out += "."; + } else if (needs_parens(expr)) out = "(" + out + ")"; while (i < arguments.length) out += "." + make_name(arguments[i++]); @@ -1288,12 +1294,11 @@ function gen_code(ast, beautify) { out.push("(" + args + ")", make(block)); return add_spaces(out); }, - "for-in": function(has_var, key, hash, block) { - var out = add_spaces([ "for", "(" ]); - if (has_var) - out += "var "; - out += add_spaces([ make_name(key) + " in " + make(hash) + ")", make(block) ]); - return out; + "for-in": function(vvar, key, hash, block) { + return add_spaces([ "for", "(" + + (vvar ? make(vvar).replace(/;+$/, "") : make(key)), + "in", + make(hash) + ")", make(block) ]); }, "while": function(condition, block) { return add_spaces([ "while", "(" + make(condition) + ")", make(block) ]); @@ -1316,7 +1321,8 @@ function gen_code(ast, beautify) { left = "(" + left + ")"; } if (member(rvalue[0], [ "assign", "conditional", "seq" ]) || - rvalue[0] == "binary" && PRECEDENCE[operator] >= PRECEDENCE[rvalue[1]]) { + rvalue[0] == "binary" && PRECEDENCE[operator] >= PRECEDENCE[rvalue[1]] && + !(rvalue[1] == operator && member(operator, [ "&&", "||", "*" ]))) { right = "(" + right + ")"; } return add_spaces([ left, operator, right ]); @@ -1350,14 +1356,15 @@ function gen_code(ast, beautify) { return indent(make_function(p[0], p[1][2], p[1][3], p[2])); } var key = p[0], val = make(p[1]); - if (beautify && beautify.quote_keys) { - key = make_string(key); - } else if (typeof key == "number" || !beautify && +key + "" == key) { + if (options.quote_keys) { + key = encode_string(key); + } else if ((typeof key == "number" || !beautify && +key + "" == key) + && parseFloat(key) >= 0) { key = make_num(+key); } else if (!is_identifier(key)) { - key = make_string(key); + key = encode_string(key); } - return indent(add_spaces(beautify && beautify.space_colon + return indent(add_spaces(beautify && options.space_colon ? [ key, ":", val ] : [ key + ":", val ])); }).join("," + newline); @@ -1369,6 +1376,7 @@ function gen_code(ast, beautify) { "array": function(elements) { if (elements.length == 0) return "[]"; return add_spaces([ "[", add_commas(MAP(elements, function(el){ + if (!beautify && el[0] == "atom" && el[1] == "undefined") return ""; return parenthesize(el, "seq"); })), "]" ]); }, @@ -1386,12 +1394,6 @@ function gen_code(ast, beautify) { }, "atom": function(name) { return make_name(name); - }, - "comment1": function(text) { - return "//" + text + "\n"; - }, - "comment2": function(text) { - return "/*" + text + "*/"; } }; @@ -1435,17 +1437,21 @@ function gen_code(ast, beautify) { return add_spaces([ out, make_block(body) ]); }; - function make_name(name) { - return name.toString(); - }; - function make_block_statements(statements) { for (var a = [], last = statements.length - 1, i = 0; i <= last; ++i) { var stat = statements[i]; var code = make(stat); if (code != ";") { - if (!beautify && i == last) - code = code.replace(/;+\s*$/, ""); + if (!beautify && i == last) { + if ((stat[0] == "while" && empty(stat[2])) || + (member(stat[0], [ "for", "for-in"] ) && empty(stat[4])) || + (stat[0] == "if" && empty(stat[2]) && !stat[3]) || + (stat[0] == "if" && stat[3] && empty(stat[3]))) { + code = code.replace(/;*\s*$/, ";"); + } else { + code = code.replace(/;+\s*$/, ""); + } + } a.push(code); } } @@ -1480,7 +1486,7 @@ function gen_code(ast, beautify) { function make_1vardef(def) { var name = def[0], val = def[1]; if (val != null) - name = add_spaces([ name, "=", make(val) ]); + name = add_spaces([ make_name(name), "=", parenthesize(val, "seq") ]); return name; }; @@ -1500,6 +1506,49 @@ function gen_code(ast, beautify) { return make(ast); }; +function split_lines(code, max_line_length) { + var splits = [ 0 ]; + jsp.parse(function(){ + var next_token = jsp.tokenizer(code); + var last_split = 0; + var prev_token; + function current_length(tok) { + return tok.pos - last_split; + }; + function split_here(tok) { + last_split = tok.pos; + splits.push(last_split); + }; + function custom(){ + var tok = next_token.apply(this, arguments); + out: { + if (prev_token) { + if (prev_token.type == "keyword") break out; + } + if (current_length(tok) > max_line_length) { + switch (tok.type) { + case "keyword": + case "atom": + case "name": + case "punc": + split_here(tok); + break out; + } + } + } + prev_token = tok; + return tok; + }; + custom.context = function() { + return next_token.context.apply(this, arguments); + }; + return custom; + }()); + return splits.map(function(pos, i){ + return code.substring(pos, splits[i + 1] || code.length); + }).join("\n"); +}; + /* -----[ Utilities ]----- */ function repeat_string(str, i) { @@ -1558,5 +1607,10 @@ exports.ast_mangle = ast_mangle; exports.ast_squeeze = ast_squeeze; exports.gen_code = gen_code; exports.ast_add_scope = ast_add_scope; -exports.ast_squeeze_more = require("./squeeze-more").ast_squeeze_more; exports.set_logger = function(logger) { warn = logger }; +exports.make_string = make_string; +exports.split_lines = split_lines; +exports.MAP = MAP; + +// keep this last! +exports.ast_squeeze_more = require("./squeeze-more").ast_squeeze_more;