Merge pull request #10 from synthicity/yaml-configs

Support for YAML configuration files

.travis.yml

@@ -8,7 +8,8 @@ install:
     # install dependencies
     - >
       conda create -p $HOME/py --yes pip jinja2 numpy pandas patsy scipy
-      statsmodels pytables pytest "python=$TRAVIS_PYTHON_VERSION"
+      statsmodels pytables pytest pyyaml
+      "python=$TRAVIS_PYTHON_VERSION"
     - export PATH=$HOME/py/bin:$PATH
     - pip install Django shapely simplejson
     # install test depencies

sandbox/init.py

@@ -8,7 +8,7 @@
 
 from urbansim.urbansimd import urbansimd
 from urbansim.utils import misc
-from urbansim.utils.misc import run_model
+from urbansim.urbansim.modelcompile import run_model
 
 pd.set_option('precision', 3)
 pd.set_option('display.width', 160)

scripts/run.py

@@ -4,7 +4,7 @@
 
 import simplejson
 
-from urbansim.utils import misc
+from urbansim.urbansim import modelcompile
 sys.path.insert(0, ".")
 import dataset
 
@@ -19,5 +19,5 @@
         model, mode = item[0], "run"
     else:
         model, mode = item
-    d = misc.run_model(model, dset, mode)
+    d = modelcompile.run_model(model, dset, mode)
     # print d

setup.py

@@ -24,6 +24,7 @@
         'numpy>=1.8.0',
         'pandas>=0.13.1',
         'patsy>=0.2.1',
+        'pyyaml>=3.10',
         'scipy>=0.13.3',
         'shapely>=1.3.0',
         'simplejson>=3.3.3',

urbansim/urbansim/compilecli.py

@@ -3,13 +3,18 @@
 import argparse
 import os.path
 
+from urbansim.urbansim import modelcompile
 from urbansim.utils import misc
 
 
 def model_save(config):
     print('Generating model for config {}.'.format(config))
 
-    basename, d = misc.gen_model(config)
+    basename = None
+    if isinstance(config, str):
+        config, basename = modelcompile.load_config(config)
+
+    basename, d = modelcompile.gen_model(config, configname=basename)
 
     for mode, code in d.items():
         outname = os.path.join(

urbansim/urbansim/modelcompile.py

@@ -0,0 +1,137 @@
+import os.path
+import sys
+import time
+import urllib2
+import urlparse
+from collections import defaultdict
+
+import simplejson as json
+import yaml
+from jinja2 import Environment, FileSystemLoader
+
+# these are the lists of modes available for each model
+MODES_D = defaultdict(lambda: ["estimate", "simulate"], {
+    "minimodel": ["run"],
+    "modelset": ["run"],
+    "transitionmodel": ["run"],
+    "transitionmodel2": ["run"],
+    "networks": ["run"]
+})
+
+
+def droptable(d):
+    d = d.copy()
+    del d['table']
+    return d
+
+J2_ENV = Environment(
+    loader=FileSystemLoader(
+        os.path.join(os.path.dirname(__file__), 'templates')),
+    trim_blocks=True)
+J2_ENV.filters['droptable'] = droptable
+
+
+def load_config(config):
+    """
+    Load a configuration from a JSON or YAML file.
+
+    Parameters
+    ----------
+    config : str
+        Path to config file.
+
+    Returns
+    -------
+    conf : dict
+        Configuration parameters as a dictionary.
+    basename : str
+        Base name (without path) of the config file.
+
+    """
+    base = os.path.basename(config)
+    ext = os.path.splitext(base)[1]
+
+    if ext == '.json':
+        loader = json.load
+    elif ext in {'.yaml', '.yml'}:
+        loader = yaml.load
+    else:
+        raise ValueError('Only JSON and YAML configs are supported.')
+
+    with open(config) as f:
+        conf = loader(f)
+
+    return conf, base
+
+
+def gen_model(config, configname=None, mode=None):
+    """
+    Generate a Python model based on a configuration stored in a JSON file.
+
+    Parameters
+    ----------
+    config : dict
+        Dictionary of config parameters.
+    configname : str, optional
+        Name of configuration file from which config came,
+        if it came from a file.
+    mode : str, optional
+
+    Returns
+    -------
+    basename : str
+    d : dict
+
+    """
+    configname = configname or 'autorun'
+
+    if 'model' not in config:
+        print('Not generating {}'.format(configname))
+        return '', {}
+
+    model = config['model']
+    d = {}
+    modes = [mode] if mode else MODES_D[model]
+    for mode in modes:
+        assert mode in MODES_D[model]
+
+        basename = os.path.splitext(configname)[0]
+        dirname = os.path.dirname(configname)
+        print('Running {} with mode {}'.format(basename, mode))
+
+        if 'var_lib_file' in config:
+            if 'var_lib_db' in config:
+                # should not be hardcoded
+                githubroot = ('https://raw.github.com/fscottfoti'
+                              '/bayarea/master/configs/')
+                var_lib = json.loads(
+                    urllib2.urlopen(
+                        urlparse.urljoin(
+                            githubroot, config['var_lib_file'])).read())
+            else:
+                with open(
+                    os.path.join(configs_dir(), config['var_lib_file'])
+                ) as f:
+                    var_lib = json.load(f)
+
+            config['var_lib'] = config.get('var_lib', {})
+            config['var_lib'].update(var_lib)
+
+        config['modelname'] = basename
+        config['template_mode'] = mode
+        d[mode] = J2_ENV.get_template(model + '.py.template').render(**config)
+
+    return basename, d
+
+COMPILED_MODELS = {}
+
+
+def run_model(config, dset, mode="estimate"):
+    basename, model = gen_model(config, mode)
+    model = model[mode]
+    code = compile(model, '<string>', 'exec')
+    ns = {}
+    exec code in ns
+    print(basename, mode)
+    out = ns['%s_%s' % (basename, mode)](dset, 2010)
+    return out

urbansim/urbansim/tests/test_compilecli.py

@@ -7,6 +7,7 @@
 import tempfile
 
 import pytest
+import yaml
 import simplejson as json
 
 from .. import compilecli
@@ -39,7 +40,7 @@ def test_model_save_with_dict():
     assert os.path.exists(os.path.join(TEST_DIR, 'models', 'autorun_run.py'))
 
 
-def test_model_save_with_file():
+def test_model_save_with_file_json():
     test_file = os.path.join(TEST_DIR, 'test_config.json')
 
     with open(test_file, 'w') as f:
@@ -48,3 +49,15 @@ def test_model_save_with_file():
     compilecli.model_save(test_file)
     assert os.path.exists(
         os.path.join(TEST_DIR, 'models', 'test_config_run.py'))
+
+
+def test_model_save_with_file_yaml():
+    test_file = os.path.join(TEST_DIR, 'test_config.yaml')
+
+    with open(test_file, 'w') as f:
+        yaml.dump(
+            TEST_CONFIG, f, default_flow_style=False, indent=4, width=50)
+
+    compilecli.model_save(test_file)
+    assert os.path.exists(
+        os.path.join(TEST_DIR, 'models', 'test_config_run.py'))

urbansim/utils/tests/test_gen_model.py → urbansim/urbansim/tests/test_gen_model.py

@@ -4,7 +4,7 @@
 
 """
 
-from ..misc import gen_model
+from ..modelcompile import gen_model
 
 
 def check_estimate_simulate(basename, d):

urbansim/urbansim/tests/test_modelcompile.py

@@ -0,0 +1,49 @@
+import os.path
+import shutil
+import tempfile
+
+import simplejson as json
+import yaml
+
+from .. import modelcompile
+
+TEST_CONFIG = {
+    'growth_rate': 0.05,
+    'internalname': 'households',
+    'model': 'transitionmodel2',
+    'output_varname': 'household_id',
+    'table': 'dset.households',
+    'zero_out_names': ['building_id']
+}
+
+
+def setup_module(module):
+    module.TEST_DIR = tempfile.mkdtemp()
+
+
+def teardown_module(module):
+    shutil.rmtree(module.TEST_DIR)
+
+
+def test_load_config_json():
+    test_file = os.path.join(TEST_DIR, 'test_config.json')
+
+    with open(test_file, 'w') as f:
+        json.dump(TEST_CONFIG, f)
+
+    config, basename = modelcompile.load_config(test_file)
+    assert config == TEST_CONFIG
+    assert basename == 'test_config.json'
+
+
+def test_load_config_yaml():
+    test_file = os.path.join(TEST_DIR, 'test_config.yaml')
+
+    with open(test_file, 'w') as f:
+        yaml.dump(
+            TEST_CONFIG, f, default_flow_style=False, indent=4, width=50)
+
+    config, basename = modelcompile.load_config(test_file)
+    assert config == TEST_CONFIG
+    assert basename == 'test_config.yaml'
+