Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use string buffer to store generated code #211

Merged
merged 7 commits into from May 19, 2020
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion m2cgen/interpreters/c/interpreter.py
Expand Up @@ -63,7 +63,7 @@ def interpret(self, expr):
if self.with_math_module:
self._cg.add_dependency("<math.h>")

return self._cg.code
return self._cg.finalize_and_get_generated_code()

# Both methods supporting linear algebra do several things:
#
Expand Down
2 changes: 1 addition & 1 deletion m2cgen/interpreters/c_sharp/interpreter.py
Expand Up @@ -58,4 +58,4 @@ def interpret(self, expr):
if self.with_math_module:
self._cg.add_dependency("System.Math")

return self._cg.code
return self._cg.finalize_and_get_generated_code()
43 changes: 37 additions & 6 deletions m2cgen/interpreters/code_generator.py
@@ -1,4 +1,6 @@
from io import StringIO
from string import Template
from weakref import finalize
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.



class CodeTemplate:
Expand Down Expand Up @@ -29,11 +31,38 @@ class BaseCodeGenerator:

def __init__(self, indent=4):
self._indent = indent
self._code_buf = None
self.reset_state()

def reset_state(self):
self._current_indent = 0
self.code = ""
self._finalize_buffer()
self._code_buf = StringIO()
self._code = None
self._finalizer = finalize(self, self._finalize_buffer)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice, yet another Python TIL for me!


def _finalize_buffer(self):
if self._code_buf is not None and not self._code_buf.closed:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Minor] if self._code_buf and not self._code_buf.closed:

self._code_buf.close()

def _write_to_code_buffer(self, text, prepend=False):
if self._code_buf.closed:
raise BufferError(
"Cannot modify code after getting generated code and "
"closing the underlying buffer!\n"
"Call reset_state() to allocate new buffer.")
if prepend:
self._code_buf.seek(0)
old_content = self._code_buf.read()
self._code_buf.seek(0)
text += old_content
self._code_buf.write(text)

def finalize_and_get_generated_code(self):
if not self._code_buf.closed:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shall we do the if self._code here as well (similarly to _finalize_buffer check)?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope. We call reset_state() in __init__() where _code_buf is set to StringIO(). So we have a guarantee here that _code_buf has been already created (not None from __init__()).

self._code = self._code_buf.getvalue()
self._finalize_buffer()
return self._code if self._code is not None else ""
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Minor] return self._code if self._code else ""

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I always prefer to check for None explicitly according to the recommendation from PEP.

Also, beware of writing if x when you really mean if x is not None -- e.g. when testing whether a variable or argument that defaults to None was set to some other value. The other value might have a type (such as a container) that could be false in a boolean context!
https://www.python.org/dev/peps/pep-0008/#programming-recommendations

If you only used if key: here, then an argument which evaluated to false would not be considered. Explicitly comparing with is None is the correct idiom to make this check. See Truth Value Testing.
https://stackoverflow.com/a/17117361

https://stackoverflow.com/a/7816439


def increase_indent(self):
self._current_indent += self._indent
Expand All @@ -48,22 +77,24 @@ def decrease_indent(self):
def add_code_line(self, line):
if not line:
return
indent = " " * self._current_indent
self.code += indent + line + "\n"
self.add_code_lines([line.strip()])

def add_code_lines(self, lines):
if isinstance(lines, str):
lines = lines.strip().split("\n")
indent = " " * self._current_indent
self.code += indent + "\n{}".format(indent).join(lines) + "\n"
self._write_to_code_buffer(
indent + "\n{}".format(indent).join(lines) + "\n")

def prepend_code_line(self, line):
self.code = line + "\n" + self.code
if not line:
return
self.prepend_code_lines([line.strip()])

def prepend_code_lines(self, lines):
if isinstance(lines, str):
lines = lines.strip().split("\n")
self.code = "\n".join(lines) + "\n" + self.code
self._write_to_code_buffer("\n".join(lines) + "\n", prepend=True)

# Following methods simply compute expressions using templates without
# changing result.
Expand Down
2 changes: 1 addition & 1 deletion m2cgen/interpreters/dart/interpreter.py
Expand Up @@ -62,7 +62,7 @@ def interpret(self, expr):
if self.with_math_module:
self._cg.add_dependency("dart:math")

return self._cg.code
return self._cg.finalize_and_get_generated_code()

def interpret_tanh_expr(self, expr, **kwargs):
self.with_tanh_expr = True
Expand Down
2 changes: 1 addition & 1 deletion m2cgen/interpreters/go/interpreter.py
Expand Up @@ -50,4 +50,4 @@ def interpret(self, expr):
if self.with_math_module:
self._cg.add_dependency("math")

return self._cg.code
return self._cg.finalize_and_get_generated_code()
5 changes: 3 additions & 2 deletions m2cgen/interpreters/haskell/interpreter.py
Expand Up @@ -52,7 +52,7 @@ def interpret(self, expr):
self._cg.prepend_code_line(self._cg.tpl_module_definition(
module_name=self.module_name))

return self._cg.code
return self._cg.finalize_and_get_generated_code()

def interpret_if_expr(self, expr, if_code_gen=None, **kwargs):
if if_code_gen is None:
Expand All @@ -72,7 +72,8 @@ def interpret_if_expr(self, expr, if_code_gen=None, **kwargs):
code_gen.add_if_termination()

if not nested:
return self._cache_reused_expr(expr, code_gen.code)
return self._cache_reused_expr(
expr, code_gen.finalize_and_get_generated_code())

def interpret_pow_expr(self, expr, **kwargs):
base_result = self._do_interpret(expr.base_expr, **kwargs)
Expand Down
2 changes: 1 addition & 1 deletion m2cgen/interpreters/java/interpreter.py
Expand Up @@ -60,7 +60,7 @@ def interpret(self, expr):
os.path.dirname(__file__), "linear_algebra.java")
top_cg.add_code_lines(utils.get_file_content(filename))

return top_cg.code
return top_cg.finalize_and_get_generated_code()

# Required by SubroutinesMixin to create new code generator for
# each subroutine.
Expand Down
2 changes: 1 addition & 1 deletion m2cgen/interpreters/javascript/interpreter.py
Expand Up @@ -49,4 +49,4 @@ def interpret(self, expr):
os.path.dirname(__file__), "linear_algebra.js")
self._cg.add_code_lines(utils.get_file_content(filename))

return self._cg.code
return self._cg.finalize_and_get_generated_code()
2 changes: 1 addition & 1 deletion m2cgen/interpreters/mixins.py
Expand Up @@ -187,7 +187,7 @@ def _process_subroutine(self, subroutine):
last_result = self._do_interpret(subroutine.expr)
self._cg.add_return_statement(last_result)

return self._cg.code
return self._cg.finalize_and_get_generated_code()

def _get_subroutine_name(self):
subroutine_name = "subroutine" + str(self._subroutine_idx)
Expand Down
2 changes: 1 addition & 1 deletion m2cgen/interpreters/php/interpreter.py
Expand Up @@ -45,4 +45,4 @@ def interpret(self, expr):

self._cg.prepend_code_line("<?php")

return self._cg.code
return self._cg.finalize_and_get_generated_code()
2 changes: 1 addition & 1 deletion m2cgen/interpreters/powershell/interpreter.py
Expand Up @@ -46,7 +46,7 @@ def interpret(self, expr):
os.path.dirname(__file__), "linear_algebra.ps1")
self._cg.prepend_code_lines(utils.get_file_content(filename))

return self._cg.code
return self._cg.finalize_and_get_generated_code()

def interpret_exp_expr(self, expr, **kwargs):
nested_result = self._do_interpret(expr.expr, **kwargs)
Expand Down
2 changes: 1 addition & 1 deletion m2cgen/interpreters/python/interpreter.py
Expand Up @@ -37,7 +37,7 @@ def interpret(self, expr):
if self.with_linear_algebra:
self._cg.add_dependency("numpy", alias="np")

return self._cg.code
return self._cg.finalize_and_get_generated_code()

def interpret_bin_vector_expr(self, expr, **kwargs):
self.with_linear_algebra = True
Expand Down
2 changes: 1 addition & 1 deletion m2cgen/interpreters/r/interpreter.py
Expand Up @@ -38,7 +38,7 @@ def interpret(self, expr):
self.enqueue_subroutine(self.function_name, expr)
self.process_subroutine_queue(top_cg)

return top_cg.code
return top_cg.finalize_and_get_generated_code()

def create_code_generator(self):
return RCodeGenerator(indent=self.indent)
Expand Down
2 changes: 1 addition & 1 deletion m2cgen/interpreters/visual_basic/interpreter.py
Expand Up @@ -60,7 +60,7 @@ def interpret(self, expr):
self._cg.add_code_line(self._cg.tpl_block_termination(
block_name="Module"))

return self._cg.code
return self._cg.finalize_and_get_generated_code()

def interpret_pow_expr(self, expr, **kwargs):
base_result = self._do_interpret(expr.base_expr, **kwargs)
Expand Down