ovm-build.sh runs on two machines and 'stage1' aggregates the results.

Also: - Create an asdl/visitor.py module to fix a circular build dependency. This was caught by building on a different machine. - asdl/gen_python shouldn't depend on asdl/gen_cpp, because the latter depends on Id. - Download clang and put it in _deps/
oilshell · Feb 16, 2018 · 6f86d6b281317fcf8afc6a7ccb2689d4a55f8817 · 6f86d6b
1 parent 297593f
commit 6f86d6b281317fcf8afc6a7ccb2689d4a55f8817
diff --git a/asdl/gen_cpp.py b/asdl/gen_cpp.py
@@ -29,76 +29,10 @@

 from asdl import asdl_ as asdl
 from asdl import encode
+from asdl import visitor

 from osh.meta import Id

-TABSIZE = 2
-MAX_COL = 80
-
-# Copied from asdl_c.py
-
-
-def ReflowLines(s, depth):
-  """Reflow the line s indented depth tabs.
-
-  Return a sequence of lines where no line extends beyond MAX_COL when properly
-  indented.  The first line is properly indented based exclusively on depth *
-  TABSIZE.  All following lines -- these are the reflowed lines generated by
-  this function -- start at the same column as the first character beyond the
-  opening { in the first line.
-  """
-  size = MAX_COL - depth * TABSIZE
-  if len(s) < size:
-    return [s]
-
-  lines = []
-  cur = s
-  padding = ""
-  while len(cur) > size:
-    i = cur.rfind(' ', 0, size)
-    # XXX this should be fixed for real
-    if i == -1 and 'GeneratorExp' in cur:
-      i = size + 3
-    assert i != -1, "Impossible line %d to reflow: %r" % (size, s)
-    lines.append(padding + cur[:i])
-    if len(lines) == 1:
-      # find new size based on brace
-      j = cur.find('{', 0, i)
-      if j >= 0:
-        j += 2  # account for the brace and the space after it
-        size -= j
-        padding = " " * j
-      else:
-        j = cur.find('(', 0, i)
-        if j >= 0:
-          j += 1  # account for the paren (no space after it)
-          size -= j
-          padding = " " * j
-    cur = cur[i + 1:]
-  else:
-    lines.append(padding + cur)
-  return lines
-
-
-def FormatLines(s, depth, reflow=True):
-  """Make the generated code readable.
-
-  Args:
-    depth: controls indentation
-    reflow: line wrapping.
-  """
-  if reflow:
-    lines = ReflowLines(s, depth)
-  else:
-    lines = [s]
-
-  result = []
-  for line in lines:
-    line = (" " * TABSIZE * depth) + line + "\n"
-    result.append(line)
-  return result
-
-
 class ChainOfVisitors:
  def __init__(self, *visitors):
    self.visitors = visitors
@@ -115,53 +49,7 @@ def VisitModule(self, module):
    'id': 'Id',  # Application specific hack for now
 }

-class AsdlVisitor:
-  """Base class for visitors.
-
-  TODO:
-  - It might be useful to separate this into VisitChildren() / generic_visit()
-    like Python's ast.NodeVisitor  does.
-  - Also remove self.f and self.Emit.  Those can go in self.output?
-  - Move to common location, since gen_python uses it as well.
-  """
-  def __init__(self, f):
-    self.f = f
-
-  def Emit(self, s, depth, reflow=True):
-    for line in FormatLines(s, depth):
-      self.f.write(line)
-
-  def VisitModule(self, mod):
-    for dfn in mod.dfns:
-      self.VisitType(dfn)
-    self.EmitFooter()
-
-  def VisitType(self, typ, depth=0):
-    if isinstance(typ.value, asdl.Sum):
-      self.VisitSum(typ.value, typ.name, depth)
-    elif isinstance(typ.value, asdl.Product):
-      self.VisitProduct(typ.value, typ.name, depth)
-    else:
-      raise AssertionError(typ)
-
-  def VisitSum(self, sum, name, depth):
-    if asdl.is_simple(sum):
-      self.VisitSimpleSum(sum, name, depth)
-    else:
-      self.VisitCompoundSum(sum, name, depth)
-
-  # Optionally overridden.
-  def VisitProduct(self, value, name, depth):
-    pass
-  def VisitSimpleSum(self, value, name, depth):
-    pass
-  def VisitCompoundSum(self, value, name, depth):
-    pass
-  def EmitFooter(self):
-    pass
-
-
-class ForwardDeclareVisitor(AsdlVisitor):
+class ForwardDeclareVisitor(visitor.AsdlVisitor):
  """Print forward declarations.

  ASDL allows forward references of types, but C++ doesn't.
@@ -176,11 +64,11 @@ def EmitFooter(self):
    self.Emit("", 0)  # blank line


-class ClassDefVisitor(AsdlVisitor):
+class ClassDefVisitor(visitor.AsdlVisitor):
  """Generate C++ classes and type-safe enums."""

  def __init__(self, f, enc_params, type_lookup, enum_types=None):
-    AsdlVisitor.__init__(self, f)
+    visitor.AsdlVisitor.__init__(self, f)
    self.ref_width = enc_params.ref_width
    self.type_lookup = type_lookup
    self.enum_types = enum_types or {}
@@ -347,9 +235,9 @@ def VisitField(self, field, type_name, offset, depth):
            'return static_cast<const %(ctype)s>('
            'Ref(base, %(offset)d).Ref(base, a));')

-        self.footer.extend(FormatLines(func_def % locals(), 0))
-        self.footer.extend(FormatLines(ARRAY_OFFSET, 1))
-        self.footer.extend(FormatLines(func_body % locals(), 1))
+        self.footer.extend(visitor.FormatLines(func_def % locals(), 0))
+        self.footer.extend(visitor.FormatLines(ARRAY_OFFSET, 1))
+        self.footer.extend(visitor.FormatLines(func_body % locals(), 1))
        self.footer.append('}\n\n')
        maybe_qual_name = name  # RESET for later

@@ -385,8 +273,8 @@ def VisitField(self, field, type_name, offset, depth):
              'return static_cast<const %(ctype)s>(Ref(base, %(offset)d));')

        # depth 0 for bodies
-        self.footer.extend(FormatLines(func_def % locals(), 0))
-        self.footer.extend(FormatLines(func_body % locals(), 1))
+        self.footer.extend(visitor.FormatLines(func_def % locals(), 0))
+        self.footer.extend(visitor.FormatLines(func_body % locals(), 1))
        self.footer.append('}\n\n')
        maybe_qual_name = name  # RESET for later

@@ -401,7 +289,7 @@ def VisitField(self, field, type_name, offset, depth):


 # Used by osh/ast_gen.py
-class CEnumVisitor(AsdlVisitor):
+class CEnumVisitor(visitor.AsdlVisitor):

  def VisitSimpleSum(self, sum, name, depth):
    # Just use #define, since enums aren't namespaced.
diff --git a/asdl/gen_python.py b/asdl/gen_python.py
@@ -10,11 +10,11 @@

 import sys

-from asdl import gen_cpp
 from asdl import asdl_ as asdl
+from asdl import visitor


-class GenClassesVisitor(gen_cpp.AsdlVisitor):
+class GenClassesVisitor(visitor.AsdlVisitor):

  def VisitSimpleSum(self, sum, name, depth):
    self.Emit('class %s_e(py_meta.SimpleObj):' % name, depth)
diff --git a/asdl/visitor.py b/asdl/visitor.py
@@ -0,0 +1,118 @@
+#!/usr/bin/python
+"""
+visitor.py
+"""
+
+from asdl import asdl_ as asdl
+
+
+class AsdlVisitor:
+  """Base class for visitors.
+
+  TODO:
+  - It might be useful to separate this into VisitChildren() / generic_visit()
+    like Python's ast.NodeVisitor  does.
+  - Also remove self.f and self.Emit.  Those can go in self.output?
+  - Move to common location, since gen_python uses it as well.
+  """
+  def __init__(self, f):
+    self.f = f
+
+  def Emit(self, s, depth, reflow=True):
+    for line in FormatLines(s, depth):
+      self.f.write(line)
+
+  def VisitModule(self, mod):
+    for dfn in mod.dfns:
+      self.VisitType(dfn)
+    self.EmitFooter()
+
+  def VisitType(self, typ, depth=0):
+    if isinstance(typ.value, asdl.Sum):
+      self.VisitSum(typ.value, typ.name, depth)
+    elif isinstance(typ.value, asdl.Product):
+      self.VisitProduct(typ.value, typ.name, depth)
+    else:
+      raise AssertionError(typ)
+
+  def VisitSum(self, sum, name, depth):
+    if asdl.is_simple(sum):
+      self.VisitSimpleSum(sum, name, depth)
+    else:
+      self.VisitCompoundSum(sum, name, depth)
+
+  # Optionally overridden.
+  def VisitProduct(self, value, name, depth):
+    pass
+  def VisitSimpleSum(self, value, name, depth):
+    pass
+  def VisitCompoundSum(self, value, name, depth):
+    pass
+  def EmitFooter(self):
+    pass
+
+
+TABSIZE = 2
+MAX_COL = 80
+
+# Copied from asdl_c.py
+
+def _ReflowLines(s, depth):
+  """Reflow the line s indented depth tabs.
+
+  Return a sequence of lines where no line extends beyond MAX_COL when properly
+  indented.  The first line is properly indented based exclusively on depth *
+  TABSIZE.  All following lines -- these are the reflowed lines generated by
+  this function -- start at the same column as the first character beyond the
+  opening { in the first line.
+  """
+  size = MAX_COL - depth * TABSIZE
+  if len(s) < size:
+    return [s]
+
+  lines = []
+  cur = s
+  padding = ""
+  while len(cur) > size:
+    i = cur.rfind(' ', 0, size)
+    # XXX this should be fixed for real
+    if i == -1 and 'GeneratorExp' in cur:
+      i = size + 3
+    assert i != -1, "Impossible line %d to reflow: %r" % (size, s)
+    lines.append(padding + cur[:i])
+    if len(lines) == 1:
+      # find new size based on brace
+      j = cur.find('{', 0, i)
+      if j >= 0:
+        j += 2  # account for the brace and the space after it
+        size -= j
+        padding = " " * j
+      else:
+        j = cur.find('(', 0, i)
+        if j >= 0:
+          j += 1  # account for the paren (no space after it)
+          size -= j
+          padding = " " * j
+    cur = cur[i + 1:]
+  else:
+    lines.append(padding + cur)
+  return lines
+
+
+def FormatLines(s, depth, reflow=True):
+  """Make the generated code readable.
+
+  Args:
+    depth: controls indentation
+    reflow: line wrapping.
+  """
+  if reflow:
+    lines = _ReflowLines(s, depth)
+  else:
+    lines = [s]
+
+  result = []
+  for line in lines:
+    line = (" " * TABSIZE * depth) + line + "\n"
+    result.append(line)
+  return result
diff --git a/benchmarks/auto.sh b/benchmarks/auto.sh
@@ -81,6 +81,15 @@ all() {
  # Note this has to happen AFTER a tarball is built.
 }

+ovm-build() {
+  local base_dir=${1:-../benchmark-data}
+
+  local provenance
+  provenance=$(benchmarks/id.sh compiler-provenance)  # capture the filename
+
+  benchmarks/ovm-build.sh measure $provenance $base_dir/ovm-build
+}
+
 #
 # Other
 #
diff --git a/benchmarks/id.sh b/benchmarks/id.sh
@@ -330,6 +330,9 @@ shell-provenance() {
  done > $out

  log "Wrote $out"
+
+  # Return value used in command sub
+  echo $out
 }

 compiler-provenance() {
@@ -362,6 +365,9 @@ compiler-provenance() {
  done > $out

  log "Wrote $out"
+
+  # Return value used in command sub
+  echo $out
 }

 "$@"
diff --git a/benchmarks/osh-parser.sh b/benchmarks/osh-parser.sh
@@ -102,7 +102,7 @@ measure() {
  local times_out="$raw_dir/$prefix.times.csv"
  local lines_out="$raw_dir/$prefix.lines.csv"

-  mkdir -p $BASE_DIR/{tmp,raw,stage1}
+  mkdir -p $BASE_DIR/{tmp,raw,stage1} $raw_dir

  write-sorted-manifest '' $lines_out
  local sorted=$SORTED
@@ -144,10 +144,9 @@ fake-other-host() {
 #

 stage1() {
-  local raw_dir=${1:-_tmp/osh-parser/raw}
-  #local raw_dir=${1:-../benchmark-data/osh-parser}
+  local raw_dir=${1:-$BASE_DIR/raw}

-  local out=_tmp/osh-parser/stage1
+  local out=$BASE_DIR/stage1
  mkdir -p $out

  local vm_csv=$out/virtual-memory.csv
diff --git a/benchmarks/osh-runtime.sh b/benchmarks/osh-runtime.sh
@@ -232,7 +232,7 @@ measure() {
  local prefix=${name%.provenance.txt}  # strip suffix

  local times_out="$raw_dir/$prefix.times.csv"
-  mkdir -p $BASE_DIR/{raw,stage1}
+  mkdir -p $BASE_DIR/{raw,stage1} $raw_dir

  # Write Header of the CSV file that is appended to.
  echo $HEADER > $times_out