Skip to content
Browse files

Merge branch 'outline' of git://github.com/caox/erlide into pu

  • Loading branch information...
2 parents 2018191 + be5e0be commit 006496cb430e92142ef75de3ca6a9fe0ab87faaf @vladdu vladdu committed Mar 24, 2012
Showing with 33 additions and 3 deletions.
  1. +33 −3 org.erlide.kernel.ide/src/erlide_scan.erl
View
36 org.erlide.kernel.ide/src/erlide_scan.erl
@@ -82,6 +82,7 @@ string_thing(_) -> "string".
%% Returns:
%% {ok,[Tok]}
%% {error,{ErrorPos,?MODULE,What},EndPos}
+
string(Cs) ->
case string_ws(Cs) of
{ok, Toks, Pos} ->
@@ -94,6 +95,7 @@ string_ws(Cs) ->
string(Cs, {1, 1}).
string(Cs, Pos) when is_list(Cs) ->
+ %%S = unicode:characters_to_list(Cs),
scan(Cs, [], [], Pos, [], []).
%% tokens(Continuation, CharList, StartPos) ->
@@ -449,11 +451,33 @@ escape_char($s) -> $\s; %\s = SPC
escape_char($d) -> $\d; %\d = DEL
escape_char(C) -> C.
+utf8_str_length(List) ->
+ case get_length(List, 0) of
+ -1 ->
+ length(List);
+ Int ->
+ Int
+ end.
+
+get_length([], Len) ->
+ Len;
+get_length([C|Rest], Len) when C <16#80 ->
+ get_length(Rest, Len +1);
+get_length([C1,C2|Rest], Len) when C1 =< 16#FF, C2 =< 16#FF, C1 band 16#E0 =:= 16#C0, C2 band 16#C0 =:= 16#80 ->
+ get_length(Rest, Len+1);
+get_length([C1,_C2,_C3|Rest], Len) when C1 =< 16#FF, C1 band 16#F0 =:= 16#E0 ->
+ get_length(Rest, Len +1);
+get_length([C1,_C2,_C3,_C4|Rest], Len) when C1 =< 16#FF, C1 band 16#F8 =:= 16#F0 ->
+ get_length(Rest, Len +1);
+get_length(_Bad, _Len) ->
+ -1.
scan_string([$"|Cs], Stack, Toks, Pos, State, Errors) ->
[StartPos,$"|S] = reverse(Stack),
{VS, SS} = unstack(S),
- scan(Cs, [], [{string,{StartPos, length(SS)+2},VS,[$"|SS]++[$"]}|Toks], inc(Pos,length(SS)+2), State, Errors);
+ Len = utf8_str_length(SS),
+ %%io:format("the ss: ~p and length :~p ~n", [SS, Len]),
+ scan(Cs, [], [{string,{StartPos, Len+2},VS,[$"|SS]++[$"]}|Toks], inc(Pos,Len+2), State, Errors);
scan_string([$\n|Cs]=_ACs, Stack, Toks, Pos, State, Errors) ->
scan_string(Cs, [{$\n,"\n"}|Stack], Toks, incrow0(Pos), State, Errors);
%% [StartPos, $"|S] = reverse(Stack),
@@ -609,9 +633,15 @@ scan_exponent(Cs, Stack, Toks, Pos, State, Errors) ->
scan_comment([$\r|Cs], Stack, Toks, Pos, State, Errors) ->
- scan([$\r|Cs], [], [{comment, {Pos, length(Stack)}, lists:reverse(Stack)}|Toks], inc(Pos,length(Stack)), State, Errors);
+ L = lists:reverse(Stack),
+ Len = utf8_str_length(L),
+ %%io:format("the comment :~p and length :~p ~n" , [L, Len]),
+ scan([$\r|Cs], [], [{comment, {Pos, Len}, lists:reverse(Stack)}|Toks], inc(Pos,Len), State, Errors);
scan_comment([$\n|Cs], Stack, Toks, Pos, State, Errors) ->
- scan([$\n|Cs], [], [{comment, {Pos, length(Stack)}, lists:reverse(Stack)}|Toks], inc(Pos,length(Stack)), State, Errors);
+ L = lists:reverse(Stack),
+ Len = utf8_str_length(L),
+ %%io:format("the comment :~p and length :~p ~n" , [L, Len]),
+ scan([$\n|Cs], [], [{comment, {Pos, Len}, lists:reverse(Stack)}|Toks], inc(Pos, Len), State, Errors);
scan_comment([C|Cs], Stack, Toks, Pos, State, Errors) ->
scan_comment(Cs, [C|Stack], Toks, Pos, State, Errors);
scan_comment([], Stack, Toks, Pos, State, Errors) ->

0 comments on commit 006496c

Please sign in to comment.
Something went wrong with that request. Please try again.