Merge branch 'release/1.7.2'

.travis.yml

@@ -1,5 +1,5 @@
 dist: xenial
-sudo: false
+os: linux
 
 cache:
   directories:

dev-requirements.txt

@@ -1,5 +1,6 @@
-pytest==3.5.0
+pytest==5.4.2
 Sphinx==1.4.1
 mock==2.0.0
 jupyter==1.0.0
-tox==3.7.0
+tox==3.15.1
+pytest-cov==2.9.0

docs/conf.py

@@ -32,7 +32,7 @@ def __getattr__(cls, name):
     'eemeter.weather.location',
     'eemeter.location',
     'eemeter.evaluation',
-    'eemeter.weather.cache',
+    'eeweather.cache',
     'scipy',
     'scipy.optimize',
     'scipy.stats',
@@ -58,6 +58,7 @@ def __getattr__(cls, name):
     'sphinx.ext.mathjax',
     'sphinx.ext.viewcode',
     'sphinx.ext.napoleon',
+    'sphinx.ext.autosectionlabel',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -85,9 +86,9 @@ def __getattr__(cls, name):
 #
 # The short X.Y version.
 
-version = '1.5'
+version = '1.7'
 # The full version, including alpha/beta/rc tags.
-release = '1.5.0'
+release = '1.7.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/index.rst

@@ -13,6 +13,7 @@ Usage
    :maxdepth: 2
 
    tutorial
+   data_files
    api
 
 License
@@ -24,6 +25,6 @@ Contact
 -------
 
 Please feel free to reach out to either
-Dan Baldewicz (Dan.Baldewicz@icfi.com, 518-452-6426) or
-Phil Ngo (phil@theimpactlab.co) with questions or for
+Abhishek Jathar (Abhishek.Jathar@icf.com) or
+Craig Maloney (craig@intellovations.com) with questions or for
 technical support.

setup.py

@@ -4,21 +4,6 @@
 
 long_description = "Calculate connected thermostat temperature/run-time savings."
 
-class PyTest(Command):
-    user_options = []
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        import subprocess
-        import sys
-        errno = subprocess.call([sys.executable, 'runtests.py', '--runslow', '--assert=plain', '--verbose','--cov-report', 'term-missing', '--cov', 'thermostat'  ])
-        raise SystemExit(errno)
-
-
 setup(name='thermostat',
     version=version,
     description='Calculate connected thermostat savings',
@@ -32,7 +17,6 @@ def run(self):
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
     ],
-    cmdclass = {'test': PyTest},
     keywords='thermostat savings EPA',
     packages=find_packages(),
     package_data={'': ['*.csv', '*.json']},

tests/conftest.py

@@ -1,8 +1,23 @@
+# content of conftest.py
+
 import pytest
 
+
 def pytest_addoption(parser):
-    parser.addoption("--runslow", action="store_true", help="run slow tests")
+    parser.addoption(
+        "--runslow", action="store_true", default=False, help="run slow tests"
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "slow: mark test as slow to run")
+
 
-def pytest_runtest_setup(item):
-    if 'slow' in item.keywords and not item.config.getoption("--runslow"):
-        pytest.skip("need --runslow option to run")
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow"):
+        # --runslow given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    for item in items:
+        if "slow" in item.keywords:
+            item.add_marker(skip_slow)

tests/fixtures/thermostats.py

@@ -169,7 +169,7 @@ def metrics_type_1_data():
 
     # this data comes from a script in scripts/test_data_generation.ipynb
     data = [{
-        'sw_version': '1.7.1',
+        'sw_version': '1.7.2',
         'ct_identifier': '8465829e-df0d-449e-97bf-96317c24dec3',
         'equipment_type': 1,
         'heating_or_cooling': 'cooling_ALL',
@@ -211,7 +211,7 @@ def metrics_type_1_data():
         'core_mean_indoor_temperature': 73.95971753003002,
         'core_mean_outdoor_temperature': 79.8426321875
         }, {
-        'sw_version': '1.7.1',
+        'sw_version': '1.7.2',
         'ct_identifier': '8465829e-df0d-449e-97bf-96317c24dec3',
         'equipment_type': 1,
         'heating_or_cooling': 'heating_ALL',

tests/test_baseline.py

@@ -5,11 +5,11 @@
 
 import pytest
 
-from fixtures.thermostats import thermostat_type_1
-from fixtures.thermostats import core_heating_day_set_type_1_entire
-from fixtures.thermostats import core_cooling_day_set_type_1_entire
-from fixtures.thermostats import metrics_type_1_data
-from fixtures.thermostats import thermostat_template
+from .fixtures.thermostats import thermostat_type_1
+from .fixtures.thermostats import core_heating_day_set_type_1_entire
+from .fixtures.thermostats import core_cooling_day_set_type_1_entire
+from .fixtures.thermostats import metrics_type_1_data
+from .fixtures.thermostats import thermostat_template
 
 from thermostat.core import CoreDaySet
 from datetime import datetime

tests/test_core.py

@@ -8,22 +8,22 @@
 
 import pytest
 
-from fixtures.thermostats import thermostat_type_1
-from fixtures.thermostats import thermostat_type_2
-from fixtures.thermostats import thermostat_type_3
-from fixtures.thermostats import thermostat_type_4
-from fixtures.thermostats import thermostat_type_5
-from fixtures.thermostats import core_heating_day_set_type_1_entire
-from fixtures.thermostats import core_heating_day_set_type_2
-from fixtures.thermostats import core_heating_day_set_type_3
-from fixtures.thermostats import core_heating_day_set_type_4
-from fixtures.thermostats import core_cooling_day_set_type_1_entire
-from fixtures.thermostats import core_cooling_day_set_type_2
-from fixtures.thermostats import core_cooling_day_set_type_3
-from fixtures.thermostats import core_cooling_day_set_type_5
-from fixtures.thermostats import metrics_type_1_data
-from fixtures.thermostats import thermostat_zero_days
-from fixtures.thermostats import thermostats_multiple_same_key
+from .fixtures.thermostats import thermostat_type_1
+from .fixtures.thermostats import thermostat_type_2
+from .fixtures.thermostats import thermostat_type_3
+from .fixtures.thermostats import thermostat_type_4
+from .fixtures.thermostats import thermostat_type_5
+from .fixtures.thermostats import core_heating_day_set_type_1_entire
+from .fixtures.thermostats import core_heating_day_set_type_2
+from .fixtures.thermostats import core_heating_day_set_type_3
+from .fixtures.thermostats import core_heating_day_set_type_4
+from .fixtures.thermostats import core_cooling_day_set_type_1_entire
+from .fixtures.thermostats import core_cooling_day_set_type_2
+from .fixtures.thermostats import core_cooling_day_set_type_3
+from .fixtures.thermostats import core_cooling_day_set_type_5
+from .fixtures.thermostats import metrics_type_1_data
+from .fixtures.thermostats import thermostat_zero_days
+from .fixtures.thermostats import thermostats_multiple_same_key
 
 from numpy.testing import assert_allclose
 from numpy import isnan

tests/test_demand.py

@@ -4,12 +4,12 @@
 
 import pytest
 
-from fixtures.thermostats import thermostat_type_1
-from fixtures.thermostats import core_heating_day_set_type_1_entire as core_heating_day_set_type_1
-from fixtures.thermostats import core_cooling_day_set_type_1_entire as core_cooling_day_set_type_1
-from fixtures.thermostats import core_heating_day_set_type_1_empty
-from fixtures.thermostats import core_cooling_day_set_type_1_empty
-from fixtures.thermostats import metrics_type_1_data
+from .fixtures.thermostats import thermostat_type_1
+from .fixtures.thermostats import core_heating_day_set_type_1_entire as core_heating_day_set_type_1
+from .fixtures.thermostats import core_cooling_day_set_type_1_entire as core_cooling_day_set_type_1
+from .fixtures.thermostats import core_heating_day_set_type_1_empty
+from .fixtures.thermostats import core_cooling_day_set_type_1_empty
+from .fixtures.thermostats import metrics_type_1_data
 
 RTOL = 1e-3
 ATOL = 1e-3

tests/test_importers.py

@@ -7,7 +7,7 @@
 
 import pytest
 
-from fixtures.thermostats import (
+from .fixtures.thermostats import (
         thermostat_type_1,
         thermostat_type_1_utc,
         thermostat_type_1_utc_bad,)

tests/test_metrics.py

@@ -9,12 +9,12 @@
 
 import pytest
 
-from fixtures.thermostats import thermostat_type_1
-from fixtures.thermostats import thermostat_type_2
-from fixtures.thermostats import thermostat_type_3
-from fixtures.thermostats import thermostat_type_4
-from fixtures.thermostats import thermostat_type_5
-from fixtures.thermostats import metrics_type_1_data
+from .fixtures.thermostats import thermostat_type_1
+from .fixtures.thermostats import thermostat_type_2
+from .fixtures.thermostats import thermostat_type_3
+from .fixtures.thermostats import thermostat_type_4
+from .fixtures.thermostats import thermostat_type_5
+from .fixtures.thermostats import metrics_type_1_data
 import six
 
 @pytest.fixture(scope="session")

tests/test_stats.py

@@ -1,7 +1,7 @@
 from thermostat.stats import combine_output_dataframes
 from thermostat.stats import compute_summary_statistics
 from thermostat.stats import summary_statistics_to_csv
-from fixtures.thermostats import thermostat_emg_aux_constant_on_outlier
+from .fixtures.thermostats import thermostat_emg_aux_constant_on_outlier
 from thermostat.multiple import multiple_thermostat_calculate_epa_field_savings_metrics
 from thermostat.exporters import COLUMNS
 
@@ -538,25 +538,29 @@ def test_summary_statistics_to_csv(combined_dataframe):
 
 def test_iqr_filteringa(thermostat_emg_aux_constant_on_outlier):
 
-    thermostats = list(thermostat_emg_aux_constant_on_outlier)
+    thermostats_iqflt = list(thermostat_emg_aux_constant_on_outlier)
     # Run the metrics / statistics with the outlier thermostat in place
-    iqflt_metrics = multiple_thermostat_calculate_epa_field_savings_metrics(thermostats)
+    iqflt_metrics = multiple_thermostat_calculate_epa_field_savings_metrics(thermostats_iqflt)
     iqflt_output_dataframe = pd.DataFrame(iqflt_metrics, columns=COLUMNS)
     iqflt_summary_statistics = compute_summary_statistics(iqflt_output_dataframe)
 
     # Remove the outlier thermostat
-    for i in range(0, len(thermostats) - 1):
-        if thermostats[i].thermostat_id == 'thermostat_single_emg_aux_constant_on_outlier':
-            outlier_thermostat = thermostats.pop(i)
+    thermostats_noiq = []
+    for thermostat in list(thermostats_iqflt):
+        if thermostat.thermostat_id != 'thermostat_single_emg_aux_constant_on_outlier':
+            thermostats_noiq.append(thermostat)
+
+    if len(thermostats_noiq) == 5:
+        raise ValueError("Try again")
 
     # Re-run the metrics / statistics with the outlier thermostat removed
-    noiq_metrics = multiple_thermostat_calculate_epa_field_savings_metrics(thermostats)
+    noiq_metrics = multiple_thermostat_calculate_epa_field_savings_metrics(thermostats_noiq)
     noiq_output_dataframe = pd.DataFrame(noiq_metrics, columns=COLUMNS)
     noiq_summary_statistics = compute_summary_statistics(noiq_output_dataframe)
 
     # Verify that the IQFLT removed the outliers by comparing this with the
     # metrics with the outlier thermostat already removed.
-    for column in range(0, len(iqflt_summary_statistics) - 1):
+    for column in range(0, len(iqflt_summary_statistics)):
         fields_iqflt = [x for x in iqflt_summary_statistics[column] if '_IQFLT' in x]
         for field_iqflt in fields_iqflt:
             field_noiq = field_iqflt.replace('IQFLT', 'NOIQ')

thermostat/__init__.py

@@ -1,4 +1,4 @@
-VERSION = (1, 7, 1)
+VERSION = (1, 7, 2)
 
 def get_version():
     return '{}.{}.{}'.format(VERSION[0], VERSION[1], VERSION[2])

thermostat/core.py

@@ -22,6 +22,10 @@
 except TypeError:
     pass  # Documentation mocks out pd, so ignore if not present.
 
+# Ignore divide-by-zero errors
+np.seterr(divide='ignore', invalid='ignore')
+
+
 CoreDaySet = namedtuple("CoreDaySet", ["name", "daily", "hourly", "start_date", "end_date"])
 
 logger = logging.getLogger('epathermostat')
@@ -35,7 +39,7 @@
     RESISTANCE_HEAT_USE_BINS_MIN_TEMP,
     RESISTANCE_HEAT_USE_BINS_MAX_TEMP + RESISTANCE_HEAT_USE_BIN_TEMP_WIDTH,
     RESISTANCE_HEAT_USE_BIN_TEMP_WIDTH))
-RESISTANCE_HEAT_USE_BIN_TUPLE_FIRST = [(RESISTANCE_HEAT_USE_BIN_FIRST[i], RESISTANCE_HEAT_USE_BIN_FIRST[i+1])
+RESISTANCE_HEAT_USE_BIN_FIRST_TUPLE = [(RESISTANCE_HEAT_USE_BIN_FIRST[i], RESISTANCE_HEAT_USE_BIN_FIRST[i+1])
                                        for i in range(0, len(RESISTANCE_HEAT_USE_BIN_FIRST) - 1)]
 
 RESISTANCE_HEAT_USE_BIN_SECOND = [-np.inf, 10, 20, 30, 40, 50, 60]
@@ -606,6 +610,7 @@ def get_resistance_heat_utilization_bins(self, runtime_temp, bins, core_heating_
         # Calculate the RHU based on the bins
         runtime_rhu['rhu'] = (runtime_rhu['aux_runtime'] + runtime_rhu['emg_runtime']) / (runtime_rhu['heat_runtime'] + runtime_rhu['emg_runtime'])
 
+        # Currently treating aux_runtime as separate from heat_runtime
         runtime_rhu['total_runtime'] = runtime_rhu.heat_runtime + runtime_rhu.aux_runtime + runtime_rhu.emg_runtime
         # Changed to use the number of minutes per eligible day
         runtime_rhu['aux_duty_cycle'] = runtime_rhu.aux_runtime / runtime_rhu.total_minutes
@@ -1559,7 +1564,6 @@ def percent_savings(avoided, baseline):
                         heat_runtime = rhu_runtime.heat_runtime.sum()
                         aux_runtime = rhu_runtime.aux_runtime.sum()
                         emg_runtime = rhu_runtime.emg_runtime.sum()
-                        # total_runtime = heat_runtime + aux_runtime + emg_runtime
                         total_minutes = rhu_runtime.total_minutes.sum()
                         additional_outputs[rhu_type + '_aux_duty_cycle'] = aux_runtime / total_minutes
                         additional_outputs[rhu_type + '_emg_duty_cycle'] = emg_runtime / total_minutes

thermostat/importers.py

@@ -172,6 +172,18 @@ def from_csv(metadata_filename, verbose=False, save_cache=False, shuffle=True, c
     # Bad thermostats return None so remove those.
     results = [x for x in result_list if x is not None]
 
+    # Check for thermostats that were not loaded and log them
+    metadata_thermostat_ids = set(metadata.thermostat_id)
+    loaded_thermostat_ids = set([x.thermostat_id for x in results])
+    missing_thermostats = metadata_thermostat_ids.difference(loaded_thermostat_ids)
+    missing_thermostats_num = len(missing_thermostats)
+    if missing_thermostats_num > 0:
+        logging.warning("Unable to load {} thermostat records because of "
+                        "errors. Please check the logs for the following thermostats:".format(
+                            missing_thermostats_num))
+        for thermostat in missing_thermostats:
+            logging.warning(thermostat)
+
     # Convert this to an iterator to maintain compatibility
     return iter(results)
 
@@ -276,7 +288,7 @@ def get_single_thermostat(thermostat_id, zipcode, equipment_type,
     if not all(dates == daily_index):
         message = ("Dates provided for thermostat_id={} may contain some "
                    "which are out of order, missing, or duplicated.".format(thermostat_id))
-        raise ValueError(message)
+        raise RuntimeError(message)
 
     # load hourly time series values
     temp_in = pd.Series(_get_hourly_block(df, "temp_in"), hourly_index)
@@ -304,7 +316,7 @@ def get_single_thermostat(thermostat_id, zipcode, equipment_type,
     if station is None:
         message = "Could not locate a valid source of outdoor temperature " \
                 "data for ZIP code {}".format(zipcode)
-        raise ValueError(message)
+        raise RuntimeError(message)
 
     utc_offset = normalize_utc_offset(utc_offset)
     temp_out = get_indexed_temperatures_eeweather(station, hourly_index_utc - utc_offset)