From 0b55c25bd17eb5d0a9e9887ebae3f74a90de4fbb Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Mon, 13 Apr 2020 17:48:48 +0000 Subject: [PATCH 1/9] add chooser filtering for backdoor mct to accommodate segmented mnl use case --- .../models/large_multinomial_logit.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/urbansim_templates/models/large_multinomial_logit.py b/urbansim_templates/models/large_multinomial_logit.py index 12004cc..a3f1fd2 100644 --- a/urbansim_templates/models/large_multinomial_logit.py +++ b/urbansim_templates/models/large_multinomial_logit.py @@ -1,13 +1,15 @@ from __future__ import print_function import orca -from urbansim.models.util import columns_in_formula +from urbansim.models.util import columns_in_formula, apply_filter_query +from choicemodels.tools import MergedChoiceTable from .. import modelmanager from ..utils import get_data, update_column, to_list, version_greater_or_equal from .shared import TemplateStep + def check_choicemodels_version(): try: import choicemodels @@ -458,8 +460,12 @@ def fit(self, mct=None): from choicemodels.tools import MergedChoiceTable if (mct is not None): - data = mct - + df_from_mct = mct.to_frame() + idx_names = df_from_mct.index.names + df_from_mct = df_from_mct.reset_index() + df_from_mct = apply_filter_query(df_from_mct, self.chooser_filters) + mct = MergedChoiceTable.from_df(df_from_mct).set_index(idx_names) + else: observations = get_data(tables = self.choosers, filters = self.chooser_filters, @@ -606,4 +612,3 @@ def probs(mct): column = self.out_column, fallback_column = self.choice_column, data = choices) - From 6d6f9383046e318091a181ae76ad9fd3ed7a43e7 Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Mon, 13 Apr 2020 20:59:33 +0000 Subject: [PATCH 2/9] ensure mct in large mnl is filtered by segmentation column for segmented large mnl use case --- urbansim_templates/models/large_multinomial_logit.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/urbansim_templates/models/large_multinomial_logit.py b/urbansim_templates/models/large_multinomial_logit.py index a3f1fd2..fab92ce 100644 --- a/urbansim_templates/models/large_multinomial_logit.py +++ b/urbansim_templates/models/large_multinomial_logit.py @@ -463,8 +463,9 @@ def fit(self, mct=None): df_from_mct = mct.to_frame() idx_names = df_from_mct.index.names df_from_mct = df_from_mct.reset_index() - df_from_mct = apply_filter_query(df_from_mct, self.chooser_filters) - mct = MergedChoiceTable.from_df(df_from_mct).set_index(idx_names) + df_from_mct = apply_filter_query( + df_from_mct, self.chooser_filters).set_index(idx_names) + mct = MergedChoiceTable.from_df(df_from_mct) else: observations = get_data(tables = self.choosers, From 86adbac4021bf1a3eb012a897180b5c0873487af Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Mon, 13 Apr 2020 21:01:44 +0000 Subject: [PATCH 3/9] enable backdoor mct for segmented large mnl --- .../segmented_large_multinomial_logit.py | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/urbansim_templates/models/segmented_large_multinomial_logit.py b/urbansim_templates/models/segmented_large_multinomial_logit.py index 1d0b19a..d290072 100644 --- a/urbansim_templates/models/segmented_large_multinomial_logit.py +++ b/urbansim_templates/models/segmented_large_multinomial_logit.py @@ -120,31 +120,41 @@ def to_dict(self): return d - def get_segmentation_column(self): + def get_segmentation_column(self, mct=None): """ Get the column of segmentation values from Orca. Chooser and alternative filters are applied to identify valid observations. + Parameters + ---------- + mct : choicemodels.tools.MergedChoiceTable + This parameter is a temporary backdoor allowing us to pass in a more + complicated choice table than can be generated within the template, for + example including sampling weights or interaction terms. + Returns ------- pd.Series """ - obs = get_data(tables = self.defaults.choosers, - filters = self.defaults.chooser_filters, - extra_columns = [self.defaults.choice_column, - self.segmentation_column]) + if mct is not None: + df = mct.to_frame() + else: + obs = get_data(tables = self.defaults.choosers, + filters = self.defaults.chooser_filters, + extra_columns = [self.defaults.choice_column, + self.segmentation_column]) - alts = get_data(tables = self.defaults.alternatives, - filters = self.defaults.alt_filters) + alts = get_data(tables = self.defaults.alternatives, + filters = self.defaults.alt_filters) - df = pd.merge(obs, alts, how='inner', - left_on=self.defaults.choice_column, right_index=True) + df = pd.merge(obs, alts, how='inner', + left_on=self.defaults.choice_column, right_index=True) return df[self.segmentation_column] - def build_submodels(self): + def build_submodels(self, mct=None): """ Create a submodel for each category of choosers identified in the segmentation column. Only categories with at least one observation remaining after applying @@ -152,11 +162,18 @@ def build_submodels(self): Running this method will overwrite any previous submodels. + Parameters + ---------- + mct : choicemodels.tools.MergedChoiceTable + This parameter is a temporary backdoor allowing us to pass in a more + complicated choice table than can be generated within the template, for + example including sampling weights or interaction terms. + """ self.submodels = {} submodel = LargeMultinomialLogitStep.from_dict(self.defaults.to_dict()) - col = self.get_segmentation_column() + col = self.get_segmentation_column(mct=mct) if (len(col) == 0): print("Warning: No valid observations after applying the chooser and "+ @@ -214,7 +231,7 @@ def update_submodels(self, param, value): def fit_all(self, mct=None): """ - Fit all the submodels. Build the subomdels first, if they don't exist yet. This + Fit all the submodels. Build the submodels first, if they don't exist yet. This method can be run as many times as desired. Parameters @@ -227,7 +244,7 @@ def fit_all(self, mct=None): """ if (len(self.submodels) == 0): - self.build_submodels() + self.build_submodels(mct=mct) for k, m in self.submodels.items(): print(' SEGMENT: {0} = {1} '.format( From 0f345b2cabc648002626eb74bfd1cac4664425fe Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 17 Apr 2020 10:32:44 -0700 Subject: [PATCH 4/9] Updating requirements to fix travis failure --- requirements.txt | 9 --------- setup.py | 15 ++++++++++----- 2 files changed, 10 insertions(+), 14 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c3584b4..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -# minimal requirements for model management and core templates - -choicemodels >= 0.2.dev4 -numpy >= 1.14 -orca >= 1.4 -pandas >= 0.23 -patsy >= 0.4 -statsmodels >= 0.8 -urbansim >= 3.1 diff --git a/setup.py b/setup.py index 511b093..a12d56f 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,5 @@ from setuptools import setup, find_packages -with open('requirements.txt') as f: - requirements = f.readlines() -requirements = [item.strip() for item in requirements] - setup( name='urbansim_templates', version='0.2.dev7', @@ -20,5 +16,14 @@ 'License :: OSI Approved :: BSD License' ], packages=find_packages(exclude=['*.tests']), - install_requires=requirements + install_requires=[ + 'choicemodels >= 0.2.dev4', + 'numpy >= 1.14', + 'orca >= 1.4', + 'pandas >= 0.23', + 'patsy >= 0.4', + 'statsmodels >= 0.8, <0.11; python_version <"3.6"', + 'statsmodels >= 0.8; python_version >="3.6"' + 'urbansim >= 3.1' + ] ) \ No newline at end of file From 5ae4fe5cc2625cf82f8e2b90397db9c448fabfc5 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 17 Apr 2020 10:37:08 -0700 Subject: [PATCH 5/9] Fixing syntax --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a12d56f..97ca0ad 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ 'pandas >= 0.23', 'patsy >= 0.4', 'statsmodels >= 0.8, <0.11; python_version <"3.6"', - 'statsmodels >= 0.8; python_version >="3.6"' + 'statsmodels >= 0.8; python_version >="3.6"', 'urbansim >= 3.1' ] ) \ No newline at end of file From 6ab79ab4c87941ff1679fdacc0e5c70f4102a167 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 17 Apr 2020 10:41:09 -0700 Subject: [PATCH 6/9] Adding newer python versions --- .travis.yml | 8 ++------ setup.py | 2 ++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e294b14..423a934 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,8 @@ python: - '2.7' - '3.5' - '3.6' - -matrix: - include: - - python: '3.7' # temp solution until python 3.7 is more cleanly supported - dist: xenial - sudo: true + - '3.7' + - '3.8' install: - pip install . diff --git a/setup.py b/setup.py index 97ca0ad..b06e97b 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,8 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: BSD License' ], packages=find_packages(exclude=['*.tests']), From 246a8dc616cbceda246ed9343889389046c3a6b3 Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Fri, 17 Apr 2020 18:51:29 +0000 Subject: [PATCH 7/9] version bump and requirements bump --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b06e97b..ec63593 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='urbansim_templates', - version='0.2.dev7', + version='0.2.dev8', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', @@ -19,7 +19,7 @@ ], packages=find_packages(exclude=['*.tests']), install_requires=[ - 'choicemodels >= 0.2.dev4', + 'choicemodels >= 0.2.2.dev1', 'numpy >= 1.14', 'orca >= 1.4', 'pandas >= 0.23', From 58e8556271f09ab50bc20267234a8e6705967609 Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Fri, 17 Apr 2020 20:07:21 +0000 Subject: [PATCH 8/9] version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ec63593..3fb451e 100644 --- a/setup.py +++ b/setup.py @@ -28,4 +28,4 @@ 'statsmodels >= 0.8; python_version >="3.6"', 'urbansim >= 3.1' ] -) \ No newline at end of file +) From 27672150c5d511b2b6b48fe162f3835527c6f764 Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Fri, 17 Apr 2020 20:44:20 +0000 Subject: [PATCH 9/9] rebuild trigger --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3fb451e..2d7a28f 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='urbansim_templates', - version='0.2.dev8', + version='0.2.dev8', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com',