[external modules] compatibility with plaitpy-ipc (0.1.0)

this diff includes features and fixes added to support external modules
like plaitpy-ipc. plaitpy-ipc is a module that provides IPC like
features for multiple plaitpy processes, like locks and queues.

new features:

* add "effect" field for "exec" type statements with side effects
* add "init / setup" field for custom template setup code
* expose GLOBALS variable in templates that can hold data

cleanup:

* update Makefile, install scripts and tests to run py2 and py3
* move debug printing to its own module
* rename fields.py -> template.py (thats what it really is)
* all field errors should cause process exit except "suppressed" fields

speed improvements:

* add cache for parsed YAML templates
* memoize lambda compilation

tests:

* add test for effect, init fields
* add test for CSV and JSON printing
* add test for custom printer

README.md

@@ -146,10 +146,13 @@ plait.py also simplifies looking up faker fields:
 
 * see docs/TROUBLESHOOTING.md
 
-### future direction
 
-Currently, plait.py models independent markov processes - future investigation
-into modeling processes that can interact with each other is needed.
+### Dependent Markov Processes
+
+To simulate data that comes from many markov processes (a markov ecosystem),
+see the [plaitpy-ipc](https://github.com/plaitpy/plaitpy-ipc) repository.
+
+### future direction
 
 If you have ideas on features to add, open an issue - Feedback is appreciated!
 

docs/FORMAT.md

@@ -41,6 +41,7 @@ field can be of multiple types, including:
 * **switch**: specifies that this field is a switch field (similar to if / else if clauses)
 * **csv**: specifies that this field is generated via sampling from a CSV file
 * **lambda**: specifies that this field is generated via custom lambda function
+* **effect**: specifies that this field is an effect field with sideeffects
 * **mixture**: specifies that this field is a mixture field (similar to a probabilistic if clause)
 * **random**: specifies that this field should be generated from random module
 * **template**: specifies that this field is generated from another template
@@ -55,6 +56,8 @@ params:
   * cast: the type to cast this field to, "int", "float", "str", etc
   * initial: the initial value for this field (if it is self-referential)
   * finalize: a final lambda expression to be run on this field (a finalizer)
+  * suppress: suppress errors from this field (if its known to throw any)
+  * onlyif: only add this field if it matches the supplied expr
 
 ##  fields API
 

scripts/install_package.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 
 VERSION=`python src/version.py`
-sudo pip install dist/plaitpy-${VERSION}.tar.gz --upgrade
+sudo pip2 install dist/plaitpy-${VERSION}.tar.gz --upgrade
+sudo pip3 install dist/plaitpy-${VERSION}.tar.gz --upgrade

src/__init__.py

@@ -1,6 +1,9 @@
-from . fields import Template
+from . template import Template
 from . import cli
 from . version import VERSION
-from .  ecosystem import Ecosystem
+from . import helpers
 
-__all__ = [ "Template", "cli" ]
+import sys
+sys.modules['plaitpy'] = sys.modules[__name__]
+
+__all__ = [ "Template", "cli", "helpers" ]

src/cli.py

@@ -8,9 +8,9 @@
 import os
 
 from . import fakerb
-from . import fields
+from . import template
 from . import helpers
-from .helpers import debug
+from . import debug
 
 def setup_args():
     parser = argparse.ArgumentParser(description='Generate fake datasets from yaml template files')
@@ -33,7 +33,7 @@ def setup_args():
     parser.add_argument('--debug', dest='debug', action="store_true", default=False,
                         help='Turn on debugging output for plait.py')
     parser.add_argument('--exit-on-error', dest='exit_error', action="store_true", default=False,
-                        help='Exit loudly on error')
+                        help='Exit loudly on any error')
 
     args = parser.parse_args()
 
@@ -44,20 +44,19 @@ def main():
     args, parser = setup_args()
 
     if args.csv:
-        fields.CSV = True
-        fields.JSON = False
+        template.CSV = True
+        template.JSON = False
 
     elif args.json:
-        fields.JSON = True
-        fields.CSV = False
+        template.JSON = True
+        template.CSV = False
 
     if args.exit_error:
         args.debug = True
-        fields.EXIT_ON_ERROR = True
+        template.EXIT_ON_ERROR = True
 
     if args.debug:
-        fields.DEBUG = True
-        helpers.DEBUG = True
+        debug.DEBUG = True
 
 
 
@@ -105,7 +104,7 @@ def main():
     helpers.add_template_path(args.dir)
     helpers.setup_globals()
 
-    tmpl = fields.Template(template_file)
-    debug("*** GENERATING %s RECORDS" % args.num_records)
+    tmpl = template.Template(template_file)
+    debug.debug("*** GENERATING %s RECORDS" % args.num_records)
     tmpl.print_records(args.num_records)
     fakerb.save_cache()

src/debug.py

@@ -0,0 +1,15 @@
+from __future__ import print_function
+from os import environ as ENV
+
+import sys
+
+DEBUG="DEBUG" in ENV
+VERBOSE=False
+
+def debug(*args):
+    if DEBUG:
+        print(" ".join(map(str, args)), file=sys.stderr)
+
+def verbose(*args):
+    if VERBOSE:
+        print(" ".join(map(str, args)), file=sys.stderr)

src/fakerb.py

@@ -5,7 +5,8 @@
 import re
 import hashlib
 
-from .helpers import debug, verbose, exit_error, readfile, LAMBDA_TYPE
+from .debug import debug, verbose
+from .helpers import LAMBDA_TYPE, readfile, exit_error
 
 # a key looks like: base.field
 # base maps to faker/lib/locales/en/base.yaml

src/helpers.py

@@ -8,40 +8,56 @@
 import math
 import os
 
-from . import tween
+from . import debug
 
 from os import environ as ENV
 
-DEBUG="DEBUG" in ENV
 LAMBDA_TYPE = type(lambda w: w)
-VERBOSE=False
 TRACEBACK=True
 
-def debug(*args):
-    if DEBUG:
-        print(" ".join(map(str, args)), file=sys.stderr)
+class DotWrapper(dict):
+    def __getattr__(self, attr):
+        if attr in self:
+            return self[attr]
+
+        if not attr in self:
+            debug.debug("MISSING ATTR", attr)
+
+    def __setattr__(self, attr, val):
+        self[attr] = val
 
-def verbose(*args):
-    if VERBOSE:
-        print(" ".join(map(str, args)), file=sys.stderr)
 
 def exit():
     sys.exit(1)
 
 def exit_error(e=None):
     import traceback
     if e:
-        debug("Error:", e)
+        debug.debug("Error:", e)
 
     if TRACEBACK:
         traceback.print_exc()
     sys.exit(1)
 
-
+# from comment on http://code.activestate.com/recipes/578231-probably-the-fastest-memoization-decorator-in-the-/
+def memoize(f):
+    """ Memoization decorator for functions taking one or more arguments. """
+    class memodict(dict):
+        def __init__(self, f):
+            self.f = f
+        def __call__(self, *args):
+            return self[args]
+        def __missing__(self, key):
+            ret = self[key] = self.f(*key)
+            return ret
+    return memodict(f)
+
+@memoize
 def make_func(expr, name):
     func = compile_lambda(str(expr), name, 'exec')
     return lambda: eval(func, GLOBALS, LOCALS)
 
+@memoize
 def make_lambda(expr, name):
     func = compile_lambda(str(expr), name)
     return lambda: eval(func, GLOBALS, LOCALS)
@@ -60,27 +76,47 @@ def __setattr__(self, attr, val):
         self[attr] = val
 
 
-
 GLOBALS = ObjWrapper({})
 RAND_GLOBALS = ObjWrapper({})
 LOCALS = ObjWrapper()
 
+class GlobalAssigner(dict):
+    def __str__(self):
+        return "GLOBAL ASSIGNER %s" % (id(self))
+
+    def __setitem__(self, attr, val):
+        GLOBALS[attr] = val
+        RAND_GLOBALS[attr] = val
+
+    def __setattr__(self, attr, val):
+        GLOBALS[attr] = val
+        RAND_GLOBALS[attr] = val
+
+    def __getitem__(self, attr):
+        if attr in RAND_GLOBALS:
+            return RAND_GLOBALS[attr]
+
+    def __getattr__(self, attr):
+        if attr in RAND_GLOBALS:
+            return RAND_GLOBALS[attr]
 
 def setup_globals():
     if "__plaitpy__" in GLOBALS:
         return
 
-    g = globals()
+    ga = GlobalAssigner()
+    ga["__plaitpy__"] = True
+    ga.time = time
+    ga.random = random
+    ga.re = re
+    ga.GLOBALS = ga
+    ga.globals = ga
 
-    GLOBALS.time = time
-    GLOBALS.random = random
-    GLOBALS.tween = tween
-    GLOBALS.re = re
-    GLOBALS["__plaitpy__"] = True
+    from . import tween
+    ga.tween = tween
 
     for field in dir(math):
-        GLOBALS[field] = getattr(math, field)
-        RAND_GLOBALS[field] = getattr(math, field)
+        ga[field] = getattr(math, field)
 
     for field in dir(random):
         RAND_GLOBALS[field] = getattr(random, field)