From 4d4f8247e1e8d3b5422c957a6adc85d355b0abbd Mon Sep 17 00:00:00 2001 From: Anthony Ramine Date: Thu, 18 Apr 2013 09:38:28 +0200 Subject: [PATCH 1/2] Teach erl_scan how to disambiguate '=' '<<' and '=<' '<<' --- lib/stdlib/src/erl_scan.erl | 22 +++++++++++++++++++++- lib/stdlib/test/erl_scan_SUITE.erl | 20 +++++++++++++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/stdlib/src/erl_scan.erl b/lib/stdlib/src/erl_scan.erl index d988a4d8c741..cc080a3237f5 100644 --- a/lib/stdlib/src/erl_scan.erl +++ b/lib/stdlib/src/erl_scan.erl @@ -580,7 +580,7 @@ scan1("=/="++Cs, St, Line, Col, Toks) -> scan1("=/"=Cs, _St, Line, Col, Toks) -> {more,{Cs,Col,Toks,Line,[],fun scan/6}}; scan1("=<"++Cs, St, Line, Col, Toks) -> - tok2(Cs, St, Line, Col, Toks, "=<", '=<', 2); + scan_lts(Cs, St, Line, Col, Toks, {{"=<",'=<',2},{"=",'=',1},1}); scan1("=="++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "==", '==', 2); scan1("="=Cs, _St, Line, Col, Toks) -> @@ -652,6 +652,26 @@ scan1([]=Cs, _St, Line, Col, Toks) -> scan1(eof=Cs, _St, Line, Col, Toks) -> {ok,Toks,Cs,Line,Col}. +scan_lts([], _St, Line, Col, Toks, Acc={_,_,_}) -> + {more,{[],Col,Toks,Line,Acc,fun scan_lts/6}}; +scan_lts("<"++Cs, St, Line, Col, Toks, {OddTokInfo,EvenTokInfo,LtCount}) -> + scan_lts(Cs, St, Line, Col, Toks, {OddTokInfo,EvenTokInfo,LtCount+1}); +scan_lts(Cs, St, Line, Col, Toks, {{Wcs,P,N},_,1}) -> + tok2(Cs, St, Line, Col, Toks, Wcs, P, N); +scan_lts(Cs, St, Line, Col, Toks, + {{Wcs,P,N},_,LtCount}) when LtCount rem 2 =:= 1 -> + Attrs = attributes(Line, Col, St, Wcs), + scan_lts2(Cs, St, Line, incr_column(Col, N), [{P,Attrs}|Toks], LtCount-1); +scan_lts(Cs, St, Line, Col, Toks, {_,{Wcs,P,N},LtCount}) -> + Attrs = attributes(Line, Col, St, Wcs), + scan_lts2(Cs, St, Line, incr_column(Col, N), [{P,Attrs}|Toks], LtCount). + +scan_lts2(Cs, St, Line, Col, Toks, 0) -> + scan1(Cs, St, Line, Col, Toks); +scan_lts2(Cs, St, Line, Col, Toks0, LtCount) -> + Toks = [{'<<',attributes(Line, Col, St, "<<")}|Toks0], + scan_lts2(Cs, St, Line, incr_column(Col, 2), Toks, LtCount-2). + scan_atom(Cs0, St, Line, Col, Toks, Ncs0) -> case scan_name(Cs0, Ncs0) of {more,Ncs} -> diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl index 361abbb771ee..30bb68a474a5 100644 --- a/lib/stdlib/test/erl_scan_SUITE.erl +++ b/lib/stdlib/test/erl_scan_SUITE.erl @@ -233,8 +233,9 @@ punctuations() -> Ts = [{W,1}], ?line test_string(S, Ts) end || S <- L], - Three = ["/=:=", "<=:=", "==:=", ">=:="], % three tokens... - No = Three ++ L, + Three = ["/=:=", "<=:=", "==:=", ">=:=", "=<<-", "=<<="], % three tokens... + EqualBin = ["=<<"], + No = Three ++ EqualBin ++ L, SL0 = [{S1++S2,{-length(S1),S1,S2}} || S1 <- L, S2 <- L, @@ -242,7 +243,6 @@ punctuations() -> SL = family_list(SL0), %% Two tokens. When there are several answers, the one with %% the longest first token is chosen: - %% [the special case "=<<" is among the tested ones] [begin W1 = list_to_atom(S1), W2 = list_to_atom(S2), @@ -250,6 +250,20 @@ punctuations() -> ?line test_string(S, Ts) end || {S,[{_,S1,S2}|_]} <- SL], + %% Special case of "=" followed by one or more "<" + test_string("=<<", [{'=',1},{'<<',1}]), + test_string("=<<-", [{'=',1},{'<<',1},{'-',1}]), + test_string("=<<=", [{'=',1},{'<<',1},{'=',1}]), + test_string("=<<<", [{'=<',1},{'<<',1}]), + test_string("=<<<-", [{'=<',1},{'<<',1},{'-',1}]), + test_string("=<<<=", [{'=<',1},{'<<',1},{'=',1}]), + test_string("=<<<<", [{'=',1},{'<<',1},{'<<',1}]), + {more,C0} = erl_scan:tokens([], "=<", 1), + {done,{ok,[{'=<',1},{dot,1}],1},eof} = erl_scan:tokens(C0, eof, 1), + {done,{ok,[{'=',1},{'<<',1},{dot,1}],1},""} = erl_scan:tokens(C0, "<. ", 1), + {done,{ok,[{'=',1},{'<<',1},{'<<',1},{dot,1}],1},""} = + erl_scan:tokens(C0, "<<<. ", 1), + PTs1 = [{'!',1},{'(',1},{')',1},{',',1},{';',1},{'=',1},{'[',1}, {']',1},{'{',1},{'|',1},{'}',1}], ?line test_string("!(),;=[]{|}", PTs1), From 590bb0083169557ac68feb817994caaa8112dd67 Mon Sep 17 00:00:00 2001 From: Anthony Ramine Date: Wed, 1 May 2013 14:35:44 +0200 Subject: [PATCH 2/2] Teach erl_scan how to disambiguate '<' '<<' and '<<' '<<' --- lib/stdlib/src/erl_scan.erl | 2 +- lib/stdlib/test/erl_scan_SUITE.erl | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/stdlib/src/erl_scan.erl b/lib/stdlib/src/erl_scan.erl index cc080a3237f5..cec81a361b78 100644 --- a/lib/stdlib/src/erl_scan.erl +++ b/lib/stdlib/src/erl_scan.erl @@ -544,7 +544,7 @@ scan1([C|Cs], St, Line, Col, Toks) when ?WHITE_SPACE(C) -> %% Punctuation characters and operators, first recognise multiples. %% << <- <= scan1("<<"++Cs, St, Line, Col, Toks) -> - tok2(Cs, St, Line, Col, Toks, "<<", '<<', 2); + scan_lts(Cs, St, Line, Col, Toks, {{"<<",'<<',2},{"<",'<',1},1}); scan1("<-"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "<-", '<-', 2); scan1("<="++Cs, St, Line, Col, Toks) -> diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl index 30bb68a474a5..bd824640a317 100644 --- a/lib/stdlib/test/erl_scan_SUITE.erl +++ b/lib/stdlib/test/erl_scan_SUITE.erl @@ -233,9 +233,10 @@ punctuations() -> Ts = [{W,1}], ?line test_string(S, Ts) end || S <- L], - Three = ["/=:=", "<=:=", "==:=", ">=:=", "=<<-", "=<<="], % three tokens... - EqualBin = ["=<<"], - No = Three ++ EqualBin ++ L, + Three = ["/=:=", "<=:=", "==:=", ">=:=", "=<<-", "=<<=", + "<<<-", "<<<="], % three tokens... + BinSpecialCases = ["<<<", "=<<"], + No = Three ++ BinSpecialCases ++ L, SL0 = [{S1++S2,{-length(S1),S1,S2}} || S1 <- L, S2 <- L, @@ -264,6 +265,20 @@ punctuations() -> {done,{ok,[{'=',1},{'<<',1},{'<<',1},{dot,1}],1},""} = erl_scan:tokens(C0, "<<<. ", 1), + %% Special case of "<" followed by one or more "<" + test_string("<<<", [{'<',1},{'<<',1}]), + test_string("<<<-", [{'<',1},{'<<',1},{'-',1}]), + test_string("<<<=", [{'<',1},{'<<',1},{'=',1}]), + test_string("<<<<", [{'<<',1},{'<<',1}]), + test_string("<<<<-", [{'<<',1},{'<<',1},{'-',1}]), + test_string("<<<<=", [{'<<',1},{'<<',1},{'=',1}]), + test_string("<<<<<", [{'<',1},{'<<',1},{'<<',1}]), + {more,C1} = erl_scan:tokens([], "<<", 1), + {done,{ok,[{'<<',1}],1},eof} = erl_scan:tokens(C1, eof, 1), + {done,{ok,[{'<',1},{'<<',1},{dot,1}],1},""} = erl_scan:tokens(C1, "<. ", 1), + {done,{ok,[{'<',1},{'<<',1},{'<<',1},{dot,1}],1},""} = + erl_scan:tokens(C1, "<<<. ", 1), + PTs1 = [{'!',1},{'(',1},{')',1},{',',1},{';',1},{'=',1},{'[',1}, {']',1},{'{',1},{'|',1},{'}',1}], ?line test_string("!(),;=[]{|}", PTs1),