Support calling lexer.lineno before or after lexing

lunarmodules · Feb 2, 2016 · 8b62502 · 8b62502
1 parent 5baf4d0
commit 8b62502
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 26 deletions.
diff --git a/lua/pl/lexer.lua b/lua/pl/lexer.lua
@@ -158,12 +158,41 @@ function lexer.scan(s,matches,filter,options)
         end
         matches = plain_matches
     end
-    local function lex()
-        local line_nr = file and 0 or 1
+    local function lex(first_arg)
+        local line_nr = 0
         local next_line = file and file:read()
         local sz = file and 0 or #s
         local idx = 1
 
+        -- res is the value used to resume the coroutine.
+        local function handle_requests(res)
+            while res do
+                local tp = type(res)
+                -- insert a token list
+                if tp == 'table' then
+                    res = yield('','')
+                    for _,t in ipairs(res) do
+                        res = yield(t[1],t[2])
+                    end
+                elseif tp == 'string' then -- or search up to some special pattern
+                    local i1,i2 = strfind(s,res,idx)
+                    if i1 then
+                        local tok = strsub(s,i1,i2)
+                        idx = i2 + 1
+                        res = yield('',tok)
+                    else
+                        res = yield('','')
+                        idx = sz + 1
+                    end
+                else
+                    res = yield(line_nr,idx)
+                end
+            end
+        end
+
+        handle_requests(first_arg)
+        if not file then line_nr = 1 end
+
         while true do
             if idx > sz then
                 if file then
@@ -176,7 +205,9 @@ function lexer.scan(s,matches,filter,options)
                     end
                     idx, sz = 1, #s
                 else
-                    return
+                    while true do
+                        handle_requests(yield())
+                    end
                 end
             end
 
@@ -198,29 +229,7 @@ function lexer.scan(s,matches,filter,options)
                         local _, newlines = tok:gsub("\n", {})
                         line_nr = line_nr + newlines
                     end
-                    while res do
-                        local tp = type(res)
-                        -- insert a token list
-                        if tp == 'table' then
-                            res = yield('','')
-                            for _,t in ipairs(res) do
-                                res = yield(t[1],t[2])
-                            end
-                        elseif tp == 'string' then -- or search up to some special pattern
-                            i1,i2 = strfind(s,res,idx)
-                            if i1 then
-                                tok = strsub(s,i1,i2)
-                                idx = i2 + 1
-                                res = yield('',tok)
-                            else
-                                res = yield('','')
-                                idx = sz + 1
-                            end
-                        else
-                            res = yield(line_nr,idx)
-                        end
-                    end
-
+                    handle_requests(res)
                     break
                 end
             end

diff --git a/tests/test-lexer.lua b/tests/test-lexer.lua
@@ -126,8 +126,14 @@ foo
 bar
 ]])
 
+asserteq(lexer.lineno(iter), 0)
 iter()
 asserteq(lexer.lineno(iter), 1)
 asserteq(lexer.lineno(iter), 1)
 iter()
 asserteq(lexer.lineno(iter), 2)
+iter()
+asserteq(lexer.lineno(iter), 3)
+iter()
+iter()
+asserteq(lexer.lineno(iter), 3)