Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Many lua parser fixes. #302

Merged
merged 6 commits into from
May 11, 2023
Merged

Many lua parser fixes. #302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ad9a4bb
Fix Lua parser to handle semicolon separators.
DanAlbert May 11, 2023
9286e28
Fix the lua parser to handle empty files.
DanAlbert May 11, 2023
5a48016
Improve lua parser handling of unknown variables.
DanAlbert May 11, 2023
fdef52f
Improve error message for Lua unexpected EOF.
DanAlbert May 11, 2023
ba07893
Handle whitespace with trailing comments at EOF.
DanAlbert May 11, 2023
b34d8e0
Handle block comments in Lua.
DanAlbert May 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 100 additions & 53 deletions dcs/lua/parse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,44 @@
from typing import Dict, Any, Optional, List, Union
from typing import Dict, Any, Callable, Optional, List, Union


def loads(tablestr, _globals: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
def loads(
tablestr,
_globals: Optional[Dict[str, Any]] = None,
unknown_variable_lookup: Optional[Callable[[str], Any]] = None,
) -> Dict[str, Any]:

class Parser:
def __init__(self, buffer: str, _globals: Optional[Dict[str, Any]] = None):
def __init__(
self,
buffer: str,
_globals: Optional[Dict[str, Any]] = None,
unknown_variable_lookup: Optional[Callable[[str], Any]] = None,
) -> None:
self.buffer: str = buffer
if _globals:
self.variables = _globals.copy()
else:
self.variables = {}
self.unknown_variable_lookup = unknown_variable_lookup

self._variable_assignment_queue: List[str] = []

self.buflen: int = len(buffer)
self.pos: int = 0
self.lineno: int = 1

# Tracks whether we are at the global scope or in an object. The parser
# doesn't currently handle function declarations. Variable declarations are
# only allowed when object_depth is zero.
self.object_depth = 0

def parse(self):
self.eat_ws()

c = self.buffer[self.pos]
if self.eob():
return

c = self.char()
if c == '{':
return self.object()
elif c == '"':
Expand Down Expand Up @@ -48,12 +66,20 @@ def parse(self):
elif varname in self.variables:
# substitute value from variables
return self.variables[varname]
else:
elif not self.object_depth:
# shortcut syntax without `local`
self.push_assigned_variable_names([varname])
else:
if self.unknown_variable_lookup is None:
se = SyntaxError()
se.text = "Use of undefined variable {}".format(varname)
se.lineno = self.lineno
se.offset = self.pos
raise se
return self.unknown_variable_lookup(varname)

self.eat_ws()
if not self.eob() and self.buffer[self.pos] == '=':
if not self.eob() and self.char() == '=':
while True:
# skip comma or whitespace
self.advance()
Expand All @@ -76,29 +102,29 @@ def parse(self):
return self.parse()
else:
se = SyntaxError()
se.text = varname + " '" + self.buffer[self.pos] + "'"
se.text = varname + " '" + self.char() + "'"
se.lineno = self.lineno
se.offset = self.pos
raise se

def str_function(self) -> str:
if self.buffer[self.pos] != '_':
if self.char() != '_':
se = SyntaxError()
se.lineno = self.lineno
se.offset = self.pos
se.text = "Expected character '_', got '{char}'".format(char=self.buffer[self.pos])
se.text = "Expected character '_', got '{char}'".format(char=self.char())
raise se

if self.advance():
raise self.eob_exception()

self.eat_ws()

if self.buffer[self.pos] != '(':
if self.char() != '(':
se = SyntaxError()
se.lineno = self.lineno
se.offset = self.pos
se.text = "Expected character '(', got '{char}'".format(char=self.buffer[self.pos])
se.text = "Expected character '(', got '{char}'".format(char=self.char())
raise se

self.advance()
Expand All @@ -108,22 +134,22 @@ def str_function(self) -> str:

self.eat_ws()

if self.buffer[self.pos] != ')':
if self.char() != ')':
se = SyntaxError()
se.lineno = self.lineno
se.offset = self.pos
se.text = "Expected character ')', got '{char}'".format(char=self.buffer[self.pos])
se.text = "Expected character ')', got '{char}'".format(char=self.char())
raise se

self.pos += 1
return s

def string(self) -> str:
if self.buffer[self.pos] != '"':
if self.char() != '"':
se = SyntaxError()
se.lineno = self.lineno
se.offset = self.pos
se.text = "Expected character '\"', got '{char}'".format(char=self.buffer[self.pos])
se.text = "Expected character '\"', got '{char}'".format(char=self.char())
raise se

state = 0
Expand All @@ -132,7 +158,7 @@ def string(self) -> str:
if self.advance():
raise self.eob_exception()

c = self.buffer[self.pos]
c = self.char()
if state == 0:
if c == '"':
state = 1
Expand All @@ -149,34 +175,34 @@ def string(self) -> str:
def number(self) -> Union[int, float]:
n = ''
sign = 1
if self.buffer[self.pos] == '-':
if self.char() == '-':
sign = -1
if self.advance():
raise self.eob_exception()

found_exponent, found_exponent_sign, found_separator = False, False, False
while not self.eob():
if self.buffer[self.pos].isnumeric():
if self.char().isnumeric():
pass
elif self.buffer[self.pos] == '.':
elif self.char() == '.':
if not found_separator:
found_separator = True
else:
raise SyntaxError()
elif self.buffer[self.pos].lower() == 'e':
elif self.char().lower() == 'e':
if not found_exponent:
found_exponent = True
else:
raise SyntaxError()
elif self.buffer[self.pos] == '-':
elif self.char() == '-':
if not found_exponent_sign:
found_exponent_sign = True
else:
raise SyntaxError()
else:
break

n += self.buffer[self.pos]
n += self.char()
self.pos += 1

num = float(n) * sign
Expand All @@ -185,12 +211,19 @@ def number(self) -> Union[int, float]:
return num

def object(self) -> Dict[Union[int, str], Any]:
self.object_depth += 1
try:
return self._object()
finally:
self.object_depth -= 1

def _object(self) -> Dict[Union[int, str], Any]:
d = {}
if self.buffer[self.pos] != '{':
if self.char() != '{':
se = SyntaxError()
se.lineno = self.lineno
se.offset = self.pos
se.text = "Expected character '{{', got '{char}'".format(char=self.buffer[self.pos])
se.text = "Expected character '{{', got '{char}'".format(char=self.char())
raise se

if self.advance():
Expand All @@ -199,17 +232,17 @@ def object(self) -> Dict[Union[int, str], Any]:
self.eat_ws()

inc_key = 1
while self.buffer[self.pos] != '}':
while self.char() != '}':
self.eat_ws()

if self.buffer[self.pos] == '[':
if self.char() == '[':
# key given
if self.advance():
raise self.eob_exception()

self.eat_ws()
key: Union[int, str]
if self.buffer[self.pos] == '"':
if self.char() == '"':
key = self.string()
else:
number = self.number()
Expand All @@ -226,23 +259,23 @@ def object(self) -> Dict[Union[int, str], Any]:

self.eat_ws()

if self.buffer[self.pos] != ']':
if self.char() != ']':
se = SyntaxError()
se.lineno = self.lineno
se.offset = self.pos
se.text = "Expected character ']', got '{char}'".format(char=self.buffer[self.pos])
se.text = "Expected character ']', got '{char}'".format(char=self.char())
raise se

if self.advance():
raise self.eob_exception()

self.eat_ws()

if self.buffer[self.pos] != '=':
if self.char() != '=':
se = SyntaxError()
se.lineno = self.lineno
se.offset = self.pos
se.text = "Expected character '=', got '{char}'".format(char=self.buffer[self.pos])
se.text = "Expected character '=', got '{char}'".format(char=self.char())
raise se

if self.advance():
Expand All @@ -261,18 +294,18 @@ def object(self) -> Dict[Union[int, str], Any]:
if self.eob():
raise self.eob_exception()

c = self.buffer[self.pos]
c = self.char()
if c == '}':
break
elif c == ',':
elif c in {',', ';'}:
if self.advance():
raise self.eob_exception()
self.eat_ws()
else:
se = SyntaxError()
se.lineno = self.lineno
se.offset = self.pos
se.text = "Unexpected character '{char}'".format(char=self.buffer[self.pos])
se.text = "Unexpected character '{char}'".format(char=self.char())
raise se

self.pos += 1
Expand All @@ -287,8 +320,8 @@ def assign_local_variable(self, value):

def eatvarname(self) -> str:
varname = ''
while (not self.eob()) and (self.buffer[self.pos].isalnum() or self.buffer[self.pos] == '_'):
varname += self.buffer[self.pos]
while (not self.eob()) and (self.char().isalnum() or self.char() == '_'):
varname += self.char()
self.pos += 1

return varname
Expand All @@ -309,28 +342,33 @@ def eatvarnamelist(self) -> List[str]:

return varnames

def eat_comment(self):
if (self.pos + 1 < self.buflen
and self.buffer[self.pos] == '-'
and self.buffer[self.pos + 1] == '-'):
while not self.eob() and self.buffer[self.pos] != '\n':
self.pos += 1
def eat_line(self) -> None:
while not self.eob() and self.char() != '\n':
self.pos += 1

def eat_block_comment(self) -> None:
while not self.eob() and self.next_n_chars(4) != "--]]":
self.pos += 1
if not self.eob():
self.pos += 4

def eat_ws(self):
"""
Advances the internal buffer until it reaches a non comment or whitespace.
:return: None
"""
self.eat_comment()
while True:
if self.pos >= self.buflen:
return
c: str = self.buffer[self.pos]
c: str = self.char()
if c == '\n':
self.lineno += 1
if c == '-':
self.eat_comment()
c = self.buffer[self.pos]
if self.next_n_chars(4) == "--[[":
self.eat_block_comment()
continue
if self.next_n_chars(2) == "--":
self.eat_line()
continue
if not c.isspace():
return

Expand All @@ -344,15 +382,24 @@ def eob(self) -> bool:
"""
return self.pos >= self.buflen

def eob_exception(self):
def eob_exception(self, lookahead: int = 0) -> SyntaxError:
offset = self.pos + lookahead
se = SyntaxError()
se.lineno = self.lineno
se.offset = self.pos
se.text = "Unexpected end of buffer"
se.offset = offset
se.text = "Unexpected end of buffer. Previous 20 characters: {}".format(
self.buffer[offset - 20:offset]
)
return se

def char(self):
return self.buffer[self.pos]
def char(self, lookahead: int = 0) -> str:
try:
return self.buffer[self.pos + lookahead]
except IndexError as ex:
raise self.eob_exception(lookahead) from ex

def next_n_chars(self, n: int) -> str:
return self.buffer[self.pos:self.pos + n]

def advance(self) -> bool:
"""
Expand All @@ -362,6 +409,6 @@ def advance(self) -> bool:
self.pos += 1
return self.eob()

p = Parser(tablestr, _globals)
p = Parser(tablestr, _globals, unknown_variable_lookup)
p.parse()
return p.variables
Loading