Merge 03c50b9 into dab0b4d

.travis.yml

@@ -23,7 +23,7 @@ install:
   - pip show urbansim_templates
 
 script:
-  - cd urbansim_templates/tests
+  - cd tests
   - coverage run --source urbansim_templates --module pytest --verbose
 
 after_success:

CHANGELOG.md

@@ -0,0 +1,69 @@
+# UrbanSim Templates change log
+
+### 0.1.dev24 (2018-12-20)
+
+- fixes a string comparison bug that caused problems with binary logit output in Windows
+
+- adds `model` as an attribute of large MNL model steps, which provides a `choicemodels.MultinomialLogitResults` object and is available any time after a model step is fitted
+
+- enables on-the-fly creation of output columns in large MNL
+
+- fixes a large MNL simulation bug when there are no valid choosers or alternatives after evaluating the filters
+
+- moves unit tests out of the module directory
+
+### 0.1.dev23 (2018-12-13)
+
+- fixes a bug with interaction terms passed into `LargeMultinomialLogitStep.run()`
+
+### 0.1.dev22 (2018-12-13)
+
+- narrows the output of `utils.get_data()` to include only the columns requested (plus the index of the primary table) -- previously Orca had also provided some extra columns such as join keys
+
+### 0.1.dev21 (2018-12-11)
+
+- adds a new function `utils.get_data()` to assemble data from Orca, automatically detecting columns included in model expressions and filters
+
+- implements `SegmentedLargeMultinomialLogit.run_all()`
+
+### 0.1.dev20 (2018-12-11)
+
+- fixes a model expression persistence bug in the small MNL template
+
+### 0.1.dev19 (2018-12-06)
+
+- fixes a bug to allow large MNL simulation with multiple chooser tables
+
+### 0.1.dev18 (2018-11-19)
+
+- improves installation and testing
+
+### 0.1.dev17 (2018-11-15)
+
+- adds an `interaction_terms` parameter that users can manually pass to `LargeMultinomialLogitStep.run()`, as a temporary solution until interaction terms are fully handled by the templates
+
+- also adds a `chooser_batch_size` parameter in the same place, to reduce memory pressure when there are large numbers of choosers
+
+### 0.1.dev16 (2018-11-06)
+
+- adds a tool for testing template validity
+
+### 0.1.dev15 (2018-10-15)
+
+- adds new `LargeMultinomialLogitStep` parameters related to choice simulation: `constrained_choices`, `alt_capacity`, `chooser_size`, and `max_iter`
+
+- updates `LargeMultinomialLogitStep.run()` to use improved simulation utilities from ChoiceModels 0.2.dev4
+
+### 0.1.dev14 (2018-09-25)
+
+- adds a template for segmented large MNL models: `SegmentedLargeMultinomialLogitStep`, which can automatically generate a set of large MNL models based on segmentation rules
+
+### 0.1.dev13 (2018-09-24)
+
+- adds a `@modelmanager.template` decorator that makes a class available to the currently running instance of ModelManager
+
+### 0.1.dev12 (2018-09-19)
+
+- moves the `register()` operation to `modelmanager` (previously it was a method implemented by the individual templates)
+
+- adds general ModelManager support for supplemental objects like pickled model results

setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name='urbansim_templates',
-    version='0.1.dev23',
+    version='0.1.dev24',
     description='UrbanSim extension for managing model steps',
     author='UrbanSim Inc.',
     author_email='info@urbansim.com',

tests/.gitignore

@@ -0,0 +1,3 @@
+.cache/*
+.coverage
+__pycache__/*

tests/pytest.ini

@@ -0,0 +1,9 @@
+[pytest]
+filterwarnings =
+    ignore:::orca
+    ignore:::urbansim
+    ignore:::pandas
+    ignore:::past
+    ignore:::prettytable
+    ignore:::statsmodels
+    ignore:::yaml

urbansim_templates/tests/test_large_multinomial_logit.py → tests/test_large_multinomial_logit.py

@@ -3,6 +3,8 @@
 import pandas as pd
 import pytest
 
+from choicemodels import MultinomialLogitResults
+
 from urbansim_templates import modelmanager
 from urbansim_templates.models import LargeMultinomialLogitStep
 from urbansim_templates.utils import validate_template
@@ -178,3 +180,82 @@ def test_simulation_constrained(m):
     obs = orca.get_table('obs').to_frame()
     assert all(~obs.choice.isin([-1]))
 
+
+def test_simulation_no_valid_choosers(m):
+    """
+    If there are no valid choosers after applying filters, simulation should exit.
+    
+    """
+    m.out_chooser_filters = 'choice == -1'
+    m.run()
+
+
+def test_simulation_no_valid_alternatives(m):
+    """
+    If there are no valid alternatives after applying filters, simulation should exit.
+    
+    """
+    m.out_alt_filters = 'altval == -1'
+    m.run()
+
+
+def test_output_column_autocreation(m):
+    """
+    Test on-the-fly creation of the output column.
+    
+    """
+    m.out_column = 'potato_chips'
+    m.run()
+
+    assert('potato_chips' in orca.get_table('obs').columns)
+    assert(m.choices.equals(orca.get_table('obs').to_frame()['potato_chips']))
+
+
+def test_diagnostic_attributes(data):
+    """
+    Test that diagnostic attributes are available when expected.
+    
+    """
+    m = LargeMultinomialLogitStep()
+    m.choosers = 'obs'
+    m.alternatives = 'alts'
+    m.choice_column = 'choice'
+    m.model_expression = 'obsval + altval'
+    m.alt_sample_size = 10
+
+    assert(m.model is None)
+    assert(m.mergedchoicetable is None)
+    assert(m.probabilities is None)
+    assert(m.choices is None)
+
+    m.fit()
+
+    assert(isinstance(m.model, MultinomialLogitResults))
+
+    len_mct = len(m.mergedchoicetable.to_frame())
+    len_obs_alts = len(orca.get_table(m.choosers).to_frame()) * m.alt_sample_size
+
+    assert(len_mct == len_obs_alts)
+
+    name = m.name
+    modelmanager.register(m)
+    modelmanager.initialize()
+    m = modelmanager.get_step(name)
+
+    assert(isinstance(m.model, MultinomialLogitResults))
+
+    m.run()
+
+    len_mct = len(m.mergedchoicetable.to_frame())
+    len_probs = len(m.probabilities)
+    len_choices = len(m.choices)
+    len_obs = len(orca.get_table(m.choosers).to_frame())
+    len_obs_alts = len_obs * m.alt_sample_size
+
+    assert(len_mct == len_obs_alts)
+    assert(len_probs == len_obs_alts)
+    assert(len_choices == len_obs)
+
+    modelmanager.remove_step(name)
+
+

urbansim_templates/tests/test_utils.py → tests/test_utils.py

@@ -87,5 +87,31 @@ def test_get_data_bad_columns(orca_session):
     assert(set(df.columns) == set(['tenure', 'pop']))
 
 
+def test_update_column(orca_session):
+    """
+    General test.
+    
+    Additional tests to add: series without index, adding column on the fly.
+    
+    """
+    table = 'buildings'
+    column = 'pop'
+    data = pd.Series([3,3,3], index=[1,2,3])
+
+    utils.update_column(table, column, data)
+    assert(orca.get_table(table).to_frame()[column].tolist() == [3,3,3])
+
 
-
+def test_update_column_incomplete_series(orca_session):
+    """
+    Update certain values but not others, with non-matching index orders.
+    
+    """
+    table = 'buildings'
+    column = 'pop'
+    data = pd.Series([10,5], index=[3,1])
+
+    utils.update_column(table, column, data)
+    assert(orca.get_table(table).to_frame()[column].tolist() == [5,2,10])
+
+

urbansim_templates/__init__.py

@@ -1 +1 @@
-version = __version__ = '0.1.dev23'
+version = __version__ = '0.1.dev24'

urbansim_templates/modelmanager.py

@@ -223,7 +223,7 @@ def save_supplemental_object(step_name, name, content, content_type, required=Tr
         Whether the supplemental object is required (not yet supported).
     
     """
-    if content_type is 'pickle':
+    if content_type == 'pickle':
         content.to_pickle(os.path.join(_disk_store, step_name+'-'+name+'.pkl'))
 
 
@@ -282,7 +282,7 @@ def remove_supplemental_object(step_name, name, content_type):
     """
     # TO DO - check that the file exists first
 
-    if content_type is 'pickle':
+    if content_type == 'pickle':
         os.remove(os.path.join(_disk_store, step_name+'-'+name+'.pkl'))
 
 

urbansim_templates/models/binary_logit.py

@@ -241,10 +241,10 @@ def run(self):
         colname = self._get_out_column()
         tabname = self._get_out_table()
 
-        if self.out_value_true is not 'nothing':
+        if self.out_value_true != 'nothing':
             df.loc[df._choices==True, colname] = self.out_value_true
 
-        if self.out_value_false is not 'nothing':
+        if self.out_value_false != 'nothing':
             df.loc[df._choices==False, colname] = self.out_value_false
 
         orca.get_table(tabname).update_col_from_series(colname, df[colname], cast=True)