Merge pull request #118 from UDST/segment_mct

Fully enable "backdoor" MergedChoiceTable functionality for Segmented Large MNL

.travis.yml

@@ -4,12 +4,8 @@ python:
   - '2.7'
   - '3.5'
   - '3.6'
-
-matrix:
-  include:
-    - python: '3.7'  # temp solution until python 3.7 is more cleanly supported
-      dist: xenial
-      sudo: true
+  - '3.7'
+  - '3.8'
 
 install:
   - pip install .

setup.py

@@ -1,12 +1,8 @@
 from setuptools import setup, find_packages
 
-with open('requirements.txt') as f:
-    requirements = f.readlines()
-requirements = [item.strip() for item in requirements]
-
 setup(
     name='urbansim_templates',
-    version='0.2.dev7',
+    version='0.2.dev8', 
     description='UrbanSim extension for managing model steps',
     author='UrbanSim Inc.',
     author_email='info@urbansim.com',
@@ -17,8 +13,19 @@
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'License :: OSI Approved :: BSD License'
     ],
     packages=find_packages(exclude=['*.tests']),
-    install_requires=requirements
-)
+    install_requires=[
+        'choicemodels >= 0.2.2.dev1',
+        'numpy >= 1.14',
+        'orca >= 1.4',
+        'pandas >= 0.23',
+        'patsy >= 0.4',
+        'statsmodels >= 0.8, <0.11; python_version <"3.6"',
+        'statsmodels >= 0.8; python_version >="3.6"',
+        'urbansim >= 3.1'
+    ]
+)

urbansim_templates/models/large_multinomial_logit.py

@@ -1,13 +1,15 @@
 from __future__ import print_function
 
 import orca
-from urbansim.models.util import columns_in_formula
+from urbansim.models.util import columns_in_formula, apply_filter_query
+from choicemodels.tools import MergedChoiceTable
 
 from .. import modelmanager
 from ..utils import get_data, update_column, to_list, version_greater_or_equal
 from .shared import TemplateStep
 
 
+
 def check_choicemodels_version():
     try:
         import choicemodels
@@ -458,8 +460,13 @@ def fit(self, mct=None):
         from choicemodels.tools import MergedChoiceTable
 
         if (mct is not None):
-            data = mct
-
+            df_from_mct = mct.to_frame()
+            idx_names = df_from_mct.index.names
+            df_from_mct = df_from_mct.reset_index()
+            df_from_mct = apply_filter_query(
+                df_from_mct, self.chooser_filters).set_index(idx_names)
+            mct = MergedChoiceTable.from_df(df_from_mct)
+
         else:        
             observations = get_data(tables = self.choosers, 
                                     filters = self.chooser_filters,
@@ -606,4 +613,3 @@ def probs(mct):
                       column = self.out_column, 
                       fallback_column = self.choice_column,
                       data = choices)
-

urbansim_templates/models/segmented_large_multinomial_logit.py

@@ -120,43 +120,60 @@ def to_dict(self):
         return d
 
 
-    def get_segmentation_column(self):
+    def get_segmentation_column(self, mct=None):
         """
         Get the column of segmentation values from Orca. Chooser and alternative filters
         are applied to identify valid observations.
 
+        Parameters
+        ----------
+        mct : choicemodels.tools.MergedChoiceTable
+            This parameter is a temporary backdoor allowing us to pass in a more
+            complicated choice table than can be generated within the template, for
+            example including sampling weights or interaction terms.
+
         Returns
         -------
         pd.Series
 
         """
-        obs = get_data(tables = self.defaults.choosers,
-                       filters = self.defaults.chooser_filters,
-                       extra_columns = [self.defaults.choice_column,
-                                        self.segmentation_column])
+        if mct is not None:
+            df = mct.to_frame()
+        else:
+            obs = get_data(tables = self.defaults.choosers,
+                           filters = self.defaults.chooser_filters,
+                           extra_columns = [self.defaults.choice_column,
+                                            self.segmentation_column])
 
-        alts = get_data(tables = self.defaults.alternatives,
-                        filters = self.defaults.alt_filters)
+            alts = get_data(tables = self.defaults.alternatives,
+                            filters = self.defaults.alt_filters)
 
-        df = pd.merge(obs, alts, how='inner',
-                      left_on=self.defaults.choice_column, right_index=True)
+            df = pd.merge(obs, alts, how='inner',
+                          left_on=self.defaults.choice_column, right_index=True)
 
         return df[self.segmentation_column]
 
 
-    def build_submodels(self):
+    def build_submodels(self, mct=None):
         """
         Create a submodel for each category of choosers identified in the segmentation
         column. Only categories with at least one observation remaining after applying
         chooser and alternative filters will be included.
 
         Running this method will overwrite any previous submodels.
 
+        Parameters
+        ----------
+        mct : choicemodels.tools.MergedChoiceTable
+            This parameter is a temporary backdoor allowing us to pass in a more
+            complicated choice table than can be generated within the template, for
+            example including sampling weights or interaction terms.
+
         """
         self.submodels = {}
         submodel = LargeMultinomialLogitStep.from_dict(self.defaults.to_dict())
 
-        col = self.get_segmentation_column()
+        col = self.get_segmentation_column(mct=mct)
 
         if (len(col) == 0):
             print("Warning: No valid observations after applying the chooser and "+
@@ -214,7 +231,7 @@ def update_submodels(self, param, value):
 
     def fit_all(self, mct=None):
         """
-        Fit all the submodels. Build the subomdels first, if they don't exist yet. This
+        Fit all the submodels. Build the submodels first, if they don't exist yet. This
         method can be run as many times as desired.
 
         Parameters
@@ -227,7 +244,7 @@ def fit_all(self, mct=None):
 
         """
         if (len(self.submodels) == 0):
-            self.build_submodels()
+            self.build_submodels(mct=mct)
 
         for k, m in self.submodels.items():
             print(' SEGMENT: {0} = {1} '.format(