Some improvements for the expression REPL and a basic test.

It's still not a very powerful REPL but I wanted to use it to evaluate some
strange width expressions and use std::smul.

PiperOrigin-RevId: 318185538

xls/dslx/BUILD

@@ -567,6 +567,10 @@ py_library(
 
 py_library(
     name = "fakefs_util",
+    # If we don't mark this as testonly we can get warnings (when the modified
+    # third party version drags in googletest as a dependency and the test
+    # main is not called).
+    testonly = True,
     srcs = ["fakefs_util.py"],
     srcs_version = "PY3",
     deps = [

xls/dslx/bit_helpers.py

@@ -1,3 +1,5 @@
+# Lint as: python3
+#
 # Copyright 2020 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Lint as: python3
 """Helper routines dealing with bits (in arbitrary-width integers)."""
 
 from typing import Iterable, Text, Optional, Tuple
@@ -220,3 +221,17 @@ def to_hex_string(value: int, width: int) -> Text:
     width: The width of the encoding to generate.
   """
   return '{:x}'.format(value & to_mask(width))
+
+
+def to_bits_string(value: int) -> str:
+  """Converts unsigned value to a bit string with _ separators every nibble."""
+  if value < 0:
+    raise ValueError(f'Value is not unsigned: {value!r}')
+  bits = bin(value)[2:]
+  rev = bits[::-1]
+  pieces = []
+  i = 0
+  while i < len(rev):
+    pieces.append(rev[i:i + 4])
+    i += 4
+  return '0b' + '_'.join(pieces)[::-1]

xls/dslx/bit_helpers_test.py

@@ -103,6 +103,19 @@ def test_from_twos_complement(self):
     self.assertEqual(15,
                      bit_helpers.from_twos_complement(value=0xf, bit_count=5))
 
+  def test_to_bits_string(self):
+    for want, input_ in [
+        ('0b0', 0b0),
+        ('0b1', 0b1),
+        ('0b10', 0b10),
+        ('0b1010', 0b1010),
+        ('0b1_0000', 0b1_0000),
+        ('0b10_0000', 0b10_0000),
+        ('0b1010_0101', 0b1010_0101),
+        ('0b1_1010_0101', 0b1_1010_0101),
+    ]:
+      self.assertEqual(want, bit_helpers.to_bits_string(input_))
+
 
 if __name__ == '__main__':
   absltest.main()

xls/dslx/fakefs_util.py

@@ -1,3 +1,5 @@
+# Lint as: python3
+#
 # Copyright 2020 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Lint as: python3
 """Helpers for working with fake filesystems.
 
 Helper for making a scoped fake filesystem; e.g. for use in tests or synthetic

xls/dslx/interpreter/BUILD

@@ -144,6 +144,7 @@ py_library(
         "//xls/dslx:bit_helpers",
         "//xls/dslx:concrete_type",
         "//xls/dslx:deduce",
+        "//xls/dslx:import_fn",
         "//xls/dslx:parametric_expression",
         "//xls/dslx:scanner",
         "//xls/dslx:span",
@@ -200,18 +201,35 @@ py_binary(
     srcs_version = "PY3",
     deps = [
         ":interpreter",
+        ":value",
         "//xls/dslx:ast",
         "//xls/dslx:bindings",
+        "//xls/dslx:bit_helpers",
         "//xls/dslx:concrete_type",
         "//xls/dslx:deduce",
-        "//xls/dslx:fakefs_util",
+        "//xls/dslx:import_routines",
         "//xls/dslx:parser",
         "//xls/dslx:parser_helpers",
         "//xls/dslx:scanner",
         "//xls/dslx:span",
         "//xls/dslx:typecheck",
+        "//xls/dslx:xls_type_error",
         "@com_google_absl_py//absl:app",
         "@com_google_absl_py//absl/flags",
+        "@pyfakefs_archive//:pyfakefs",
+    ],
+)
+
+py_test(
+    name = "repl_test",
+    srcs = ["repl_test.py"],
+    data = [":repl"],
+    python_version = "PY3",
+    srcs_version = "PY3",
+    deps = [
+        ":repl@pytype@lib",
+        "//xls/common:runfiles",
+        "//xls/common:test_base",
     ],
 )
 

xls/dslx/interpreter/interpreter.py

@@ -32,6 +32,7 @@
 from xls.dslx import ast
 from xls.dslx import bit_helpers
 from xls.dslx import deduce
+from xls.dslx import import_fn
 from xls.dslx.concrete_type import ArrayType
 from xls.dslx.concrete_type import BitsType
 from xls.dslx.concrete_type import ConcreteType
@@ -1228,6 +1229,16 @@ def _evaluate_fn(self,
 
     return result
 
+  def _do_import(self, subject: import_fn.ImportTokens,
+                 span: Span) -> ast.Module:
+    """Handles an import as specified by a top level module statement."""
+    if self._f_import is None:
+      raise EvaluateError(span,
+                          'Cannot import, no import capability was provided.')
+    imported_module, imported_node_to_type = self._f_import(subject)
+    self._node_to_type.update(imported_node_to_type)
+    return imported_module
+
   def _make_top_level_bindings(self, m: ast.Module) -> Bindings:
     """Creates a fresh set of bindings for use in module-level evaluation.
 
@@ -1292,12 +1303,7 @@ def _make_top_level_bindings(self, m: ast.Module) -> Bindings:
       elif isinstance(member, ast.Enum):
         b.add_enum(member.identifier, member)
       elif isinstance(member, ast.Import):
-        if self._f_import is None:
-          raise EvaluateError(
-              member.span, 'Cannot import, no import capability was provided.')
-        subject = member.name
-        imported_module, imported_node_to_type = self._f_import(subject)
-        self._node_to_type.update(imported_node_to_type)
+        imported_module = self._do_import(member.name, member.span)
         b.add_mod(member.identifier, imported_module)
     return b
 

xls/dslx/interpreter/repl.py

@@ -14,24 +14,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Read-eval-print-loop (REPL) for DSLX input."""
+"""Minimal read-eval-print-loop (REPL) for DSL input, just for expressions."""
 
+import functools
 import readline  # pylint: disable=unused-import
 import sys
 
 from absl import app
 from absl import flags
+from pyfakefs import fake_filesystem
+
 from xls.dslx import ast
-from xls.dslx import bindings as bindings_mod
+from xls.dslx import bit_helpers
 from xls.dslx import concrete_type as concrete_type_mod
-from xls.dslx import deduce
-from xls.dslx import fakefs_util
+from xls.dslx import import_routines
 from xls.dslx import parser
 from xls.dslx import parser_helpers
 from xls.dslx import scanner
 from xls.dslx import span
 from xls.dslx import typecheck
+from xls.dslx import xls_type_error
 from xls.dslx.interpreter import interpreter as interpreter_mod
+from xls.dslx.interpreter import value as value_mod
 
 FLAGS = flags.FLAGS
 FILENAME = '/fake/repl.x'
@@ -45,69 +49,108 @@
 
 def concrete_type_to_annotation(
     concrete_type: concrete_type_mod.ConcreteType) -> ast.TypeAnnotation:
-  if concrete_type.is_bits():
-    assert concrete_type.is_ubits() or concrete_type.is_sbits()
-    keyword = UN_KEYWORD if concrete_type.is_ubits() else SN_KEYWORD
+  if isinstance(concrete_type, concrete_type_mod.BitsType):
+    keyword = SN_KEYWORD if concrete_type.get_signedness() else UN_KEYWORD
     num_tok = scanner.Token(scanner.TokenKind.NUMBER, FAKE_SPAN,
                             concrete_type.get_total_bit_count())
     return ast.TypeAnnotation(FAKE_SPAN, keyword, dims=(ast.Number(num_tok),))
 
   raise NotImplementedError(concrete_type)
 
 
+def handle_line(line: str, stmt_index: int, f_import):
+  """Runs a single user-provided line as a REPL input."""
+  fn_name = f'repl_{stmt_index}'
+  module_text = f"""
+  import std
+  fn {fn_name}() -> () {{
+    {line}
+  }}
+  """
+
+  # For error reporting we use a helper that puts this into a fake filesystem
+  # location.
+  def make_fakefs_open():
+    fs = fake_filesystem.FakeFilesystem()
+    fs.CreateFile(FILENAME, module_text)
+    return fake_filesystem.FakeFileOpen(fs)
+
+  while True:
+    try:
+      fake_module = parser.Parser(scanner.Scanner(
+          FILENAME, module_text)).parse_module(fn_name)
+    except span.PositionalError as e:
+      parser_helpers.pprint_positional_error(e, fs_open=make_fakefs_open())
+      return
+
+    # First attempt at type checking, we expect this may fail the first time
+    # around and we'll substitute the real return type we observe.
+    try:
+      node_to_type = typecheck.check_module(fake_module, f_import=f_import)
+    except xls_type_error.XlsTypeError as e:
+      # We use nil as a placeholder, and swap it with the type that was expected
+      # and retry once we determine what that should be.
+      if e.rhs_type == concrete_type_mod.ConcreteType.NIL:
+        module_text = module_text.replace(' -> ()', ' -> ' + str(e.lhs_type))
+        continue
+      # Any other errors are likely real type errors in the code and we should
+      # report them.
+      parser_helpers.pprint_positional_error(e, fs_open=make_fakefs_open())
+      return
+
+    # It type checked ok, and we can proceed.
+    break
+
+  # Interpret the line and print the result.
+  # TODO(leary): 2020-06-20 No let bindings for the moment, just useful for
+  # evaluating expressions -- could put them into the module scope as consts.
+  interpreter = interpreter_mod.Interpreter(
+      fake_module, node_to_type, f_import=f_import, trace_all=False)
+  result = interpreter.run_function(fn_name, args=())
+  print(result)
+  return result
+
+
 def main(argv):
   if len(argv) > 1:
     raise app.UsageError('Too many command-line arguments.')
 
-  bindings = bindings_mod.Bindings()
-  fake_module = ast.Module(name='repl', top=())
+  stmt_index = 0
 
-  prompt = 'dslx> ' if sys.stdin.isatty() else ''
+  import_cache = {}
+  f_import = functools.partial(import_routines.do_import, cache=import_cache)
+  last_result = None
 
-  stmt_index = 0
   while True:
+    prompt = f'dslx[{stmt_index}]> ' if sys.stdin.isatty() else ''
     try:
       # TODO(leary): Update this to support multi-line input.
       line = input(prompt)
     except EOFError:
-      print('\r', end='')
+      if sys.stdin.isatty():
+        print('\r', end='')
       break
-    try:
-      expr = parser.Parser(scanner.Scanner(FILENAME,
-                                           line)).parse_expression(bindings)
-    except span.PositionalError as e:
-      with fakefs_util.scoped_fakefs(FILENAME, line):
-        parser_helpers.pprint_positional_error(e)
-      continue
 
-    try:
-      result_type = deduce.deduce(expr, deduce.NodeToType())
-    except span.PositionalError as e:
-      with fakefs_util.scoped_fakefs(FILENAME, line):
-        parser_helpers.pprint_positional_error(e)
+    # Some helper 'magic' commands for printing out values in different ways.
+    if line == '%int':
+      if last_result is None:
+        print('No last result for magic command.')
+        continue
+      assert isinstance(last_result, value_mod.Value), last_result
+      print(last_result.get_bits_value_signed())
+      continue
+    if line == '%bin':
+      if last_result is None:
+        print('No last result for magic command.')
+      assert isinstance(last_result, value_mod.Value), last_result
+      print(bit_helpers.to_bits_string(last_result.get_bits_value()))
       continue
 
-    stmt_name = f'repl_{stmt_index}'
-    name_tok = scanner.Token(
-        scanner.TokenKind.IDENTIFIER, span=FAKE_SPAN, value=stmt_name)
-    fn = ast.Function(
-        FAKE_SPAN,
-        ast.NameDef(name_tok), (), (),
-        concrete_type_to_annotation(result_type),
-        expr,
-        public=False)
-    fake_module.top = fake_module.top + (fn,)
-    try:
-      node_to_type = typecheck.check_module(fake_module, f_import=None)
-    except span.PositionalError as e:
-      with fakefs_util.scoped_fakefs(FILENAME, line):
-        parser_helpers.pprint_positional_error(e)
+    result = handle_line(line, stmt_index, f_import)
+    if result is None:
       continue
+    last_result = result
     stmt_index += 1
-    interpreter = interpreter_mod.Interpreter(
-        fake_module, node_to_type, f_import=None, trace_all=False)
-    result = interpreter.run_function(stmt_name, args=())
-    print(result)
 
 
 if __name__ == '__main__':

xls/dslx/interpreter/repl_test.py

@@ -0,0 +1,44 @@
+# Lint as: python3
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for xls.dslx.interpreter.repl binary."""
+
+import subprocess as subp
+
+from xls.common import runfiles
+from xls.common import test_base
+
+REPL_PATH = runfiles.get_path('xls/dslx/interpreter/repl')
+
+
+class ReplTest(test_base.TestCase):
+
+  def test_simple_command(self):
+    # For some unclear reason pylint doesn't like the input keyword here but it
+    # works fine.
+    # pylint: disable=unexpected-keyword-arg
+    output = subp.check_output([REPL_PATH], input=b'u32:2+u32:2')\
+        .decode('utf-8')
+    self.assertEqual(output, 'bits[32]:0x4\n')
+
+  def test_stdlib_expr(self):
+    # pylint: disable=unexpected-keyword-arg
+    output = subp.check_output([REPL_PATH], input=b'std::smul(sN[2]:0b11, sN[3]:0b001)')\
+        .decode('utf-8')
+    self.assertEqual(output, 'bits[5]:0x1f\n')
+
+
+if __name__ == '__main__':
+  test_base.main()

xls/dslx/interpreter/value.py

@@ -42,7 +42,7 @@ class Tag(enum_mod.Enum):
   FUNCTION = 'function'
 
 
-class Value(object):
+class Value:
   """Represents a value in the interpreter evaluation.
 
   The value type is capable of representing all expression evaluation results.