Merge pull request #1058 from NREL/data-dictionary

Data dictionary + automated RTD generation

.github/workflows/config.yml

@@ -17,7 +17,7 @@ jobs:
         with:
           ref: ${{ github.head_ref }}
 
-      - name: Install gems and software
+      - name: Install gems
         run: |
           rm -f Gemfile.lock && bundle install
 
@@ -37,7 +37,13 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           path: resources/buildstock.csv
-          name: samples
+          name: feature_samples
+
+      - name: Store code coverage
+        uses: actions/upload-artifact@v3
+        with:
+          path: coverage
+          name: coverage
 
   analysis-tests:
     runs-on: ubuntu-latest
@@ -64,7 +70,7 @@ jobs:
             project_testing/results-Baseline.csv
             project_national/results-AllUpgrades.csv
             project_testing/results-AllUpgrades.csv
-          name: run_analysis
+          name: run_analysis_results_csvs
 
   integration-tests:
     runs-on: ubuntu-latest
@@ -97,29 +103,22 @@ jobs:
           tar -xvf project_national/national_upgrades/simulation_output/*.tar.gz -C project_national/national_upgrades/simulation_output
           tar -xvf project_testing/testing_upgrades/simulation_output/*.tar.gz -C project_testing/testing_upgrades/simulation_output
 
-      - name: Run tests
-        run: |
-          ruby test/test_bsb_analysis.rb
-
       - name: Process results
         run: |
           pip install pandas==1.5.3
-          python test/util.py
+          python test/process_bsb_analysis.py
+
+      - name: Run tests
+        run: |
+          ruby test/test_bsb_analysis.rb
 
       - name: Upload integration results
         uses: actions/upload-artifact@v3
         with:
           path: |
             baseline
             upgrades
-          name: results
-
-      - name: Upload output files
-        uses: actions/upload-artifact@v3
-        with:
-          path: |
-            outputs
-          name: outputs
+          name: feature_results
 
       - name: Upload buildstockbatch results
         uses: actions/upload-artifact@v3
@@ -129,11 +128,11 @@ jobs:
             project_testing/testing_baseline/results_csvs/results_up00.csv
             project_national/national_upgrades/results_csvs/results_up15.csv
             project_testing/testing_upgrades/results_csvs/results_up15.csv
-          name: buildstockbatch
+          name: buildstockbatch_results_csvs
 
   compare-tools:
     runs-on: ubuntu-latest
-    needs: [unit-tests, analysis-tests, integration-tests]
+    needs: [analysis-tests, integration-tests]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -143,22 +142,22 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           path: buildstockbatch
-          name: buildstockbatch
+          name: buildstockbatch_results_csvs
 
       - name: Download run_analysis results
         uses: actions/download-artifact@v3
         with:
           path: run_analysis
-          name: run_analysis
+          name: run_analysis_results_csvs
 
       - name: Run tests
         run: |
-          ruby test/test_tools.rb
+          ruby test/test_analysis_tools.rb
 
   compare-results:
     if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
-    needs: [unit-tests, analysis-tests, integration-tests]
+    needs: [integration-tests]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -184,13 +183,13 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           path: samples
-          name: samples
+          name: feature_samples
 
       - name: Download feature results
         uses: actions/download-artifact@v3
         with:
           path: results
-          name: results
+          name: feature_results
 
       - name: Compare samples and results
         run: |          
@@ -239,35 +238,42 @@ jobs:
 
   update-results:
     runs-on: ubuntu-latest
-    needs: [unit-tests, analysis-tests, integration-tests]
+    needs: [integration-tests]
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
 
-      - name: Install gems and software
+      - name: Install software
         run: |
+          export DEBIAN_FRONTEND=noninteractive
           sudo apt update
-          sudo apt-get install -y wget unzip libxml2-utils python3-lxml python3-pip
-          sudo pip3 install sphinx sphinx-rtd-theme
+          sudo -E apt-get install -y wget unzip python3-sphinx-rtd-theme
 
       - name: Download feature samples
         uses: actions/download-artifact@v3
         with:
           path: samples
-          name: samples
+          name: feature_samples
 
       - name: Download feature results
         uses: actions/download-artifact@v3
         with:
           path: results
-          name: results
+          name: feature_results
 
-      - name: Download output files
-        uses: actions/download-artifact@v3
+      - name: Build documentation
+        run: |
+          ruby docs/read_the_docs/source/workflow_inputs/create_characteristics_rst.rb
+          ruby docs/read_the_docs/source/workflow_outputs/csv_tables.rb
+          cd docs/read_the_docs
+          make html SPHINXOPTS="-W --keep-going -n"
+
+      - name: Save Docs
+        uses: actions/upload-artifact@v3
         with:
-          path: outputs
-          name: outputs
+          name: documentation
+          path: docs/read_the_docs/_build/html/
 
       - name: Commit latest results
         shell: bash        
@@ -276,11 +282,11 @@ jobs:
           git pull origin $branch_name
           cp -r samples/buildstock*.csv test/base_results/baseline/annual
           cp -r results/baseline/annual/results*.csv test/base_results/baseline/annual
-          cp -r results/baseline/timeseries/results*.csv test/base_results/baseline/timeseries
+          cp -r results/baseline/timeseries/*.csv test/base_results/baseline/timeseries
           cp -r results/upgrades/annual/results*.csv test/base_results/upgrades/annual
-          cp -r results/upgrades/timeseries/results*.csv test/base_results/upgrades/timeseries
-          cp -r outputs/*.csv test/base_results/outputs
+          cp -r results/upgrades/timeseries/*.csv test/base_results/upgrades/timeseries
           git add test/base_results
+          git add docs
           git status
           if [[ $(git diff --cached --exit-code) ]]; then
             git config --global user.email "github-action@users.noreply.github.com"
@@ -289,14 +295,3 @@ jobs:
             echo "Pushing to branch: $branch_name"
             git push -u origin $branch_name
           fi
-
-      - name: Build documentation
-        run: |
-          cd docs/read_the_docs
-          make html SPHINXOPTS="-W --keep-going -n"
-
-      - name: Save Docs
-        uses: actions/upload-artifact@v3
-        with:
-          name: documentation
-          path: docs/read_the_docs/_build/html/

.gitignore

@@ -8,9 +8,11 @@
 /project_testing/testing_upgrades
 /resources/buildstock.csv
 /project_*/**/*.csv
+!/project_*/resources/source_report.csv
 /project_*/**/*.log
 /weather
 .vscode/
 .DS_Store
 **/.ipynb_checkpoints/*
 **/data/*
+!/resources/data/dictionary/outputs.csv

CHANGELOG.md

@@ -7,6 +7,7 @@ Features
 - Ability to specify a "sample_weight" column in the precomputed buildstock.csv ([#1056](https://github.com/NREL/resstock/pull/1056))
 - Add descriptions to the housing characteristics ([#1069](https://github.com/NREL/resstock/pull/1069))
 - Connect ASHP to optional capacity retention temperature and fraction arguments (that already exist for MSHP) ([#1071](https://github.com/NREL/resstock/pull/1071))
+- Add data dictionary files for describing various outputs. Use these files to (1) check against integration test results, and (2) generate documentation tables ([#1058](https://github.com/NREL/resstock/pull/1058))
 
 Fixes
 - Pulls in upstream OS-HPXML fix related to [avoiding possible OpenStudio temporary directory collision](https://github.com/NREL/OpenStudio-HPXML/pull/1316) causing random errors ([#1054](https://github.com/NREL/resstock/pull/1054))

docs/read_the_docs/source/advanced_tutorial/index.rst

@@ -10,10 +10,10 @@ At this point in the tutorial, it is assumed that you have checked out a new bra
 If your changes are intended to be merged into the ``develop`` branch of the `ResStock <https://github.com/NREL/resstock>`_ repository, a pull request review is required.
 
 .. toctree::
+   :maxdepth: 2
 
-   modifying_probability_distributions
-   writing_housing_characteristics
    installer_setup
    tasks
    options_lookup
-   increasing_upgrade_options
+   increasing_upgrade_options
+   upgrade_scenario_config

docs/read_the_docs/source/upgrade_scenario_config.rst → docs/read_the_docs/source/advanced_tutorial/upgrade_scenario_config.rst

@@ -1,7 +1,7 @@
 Upgrade Scenario Configuration
 ==============================
 
-There is quite a bit more flexibility and capability in defining an upgrade scenario than was discussed in the :ref:`tutorial <tutorial-apply-upgrade>`. Here we will go through each field in the **Apply Upgrade** measure and discuss how it can be used to build more complicated real-life scenarios for upgrades.
+There is quite a bit more flexibility and capability in defining an upgrade scenario than was discussed in the :ref:`basic tutorial <tutorial-apply-upgrade>`. Here we will go through each field in the **Apply Upgrade** measure and discuss how it can be used to build more complicated real-life scenarios for upgrades.
 
 Upgrade Name
 ------------

docs/read_the_docs/source/tutorial/architecture.rst → docs/read_the_docs/source/basic_tutorial/architecture.rst

@@ -136,7 +136,7 @@ They contribute to the generation of the model.
   Also, we'll give the upgrade scenario a name: "Triple-Pane Windows" and a cost of $40/ft\ :superscript:`2` of window area by entering the number in **Option 1 Cost Value** and selecting "Window Area (ft^2)" for **Option 1 Cost Multiplier**.
   Like the **downselect logic**, excluded datapoints (i.e., datapoints for which the upgrade does not apply) will result in "completed invalid workflow".
   Note that using no downselect logic will apply the option to all housing units.
-  For a full explanation of how to set up the options and logic surrounding them, see :doc:`../upgrade_scenario_config`.
+  For a full explanation of how to set up the options and logic surrounding them, see :doc:`../advanced_tutorial/upgrade_scenario_config`.
 
 **HPXMLtoOpenStudio**
 

docs/read_the_docs/source/tutorial/index.rst → docs/read_the_docs/source/basic_tutorial/index.rst

@@ -1,11 +1,11 @@
-.. _tutorial:
+.. _basic_tutorial:
 
-Tutorial
-========
+Basic Tutorial
+==============
 
 .. toctree::
+   :maxdepth: 2
 
    installation
    architecture
    run_project
-   conclusion

docs/read_the_docs/source/tutorial/installation.rst → docs/read_the_docs/source/basic_tutorial/installation.rst

@@ -26,7 +26,7 @@ Install OpenStudio
 
 Download the version of OpenStudio software (corresponding to the ResStock version that has been selected) from the `OpenStudio developer website <https://www.openstudio.net/developers>`_.
 
-Developer instructions
+Developer Instructions
 ----------------------
 
 If you will be developing residential measures and testing residential building models, see the :ref:`advanced_tutorial`. If you are a developer, make sure that you have checked out the ``develop`` branch of the repository.

docs/read_the_docs/source/conf.py

@@ -104,6 +104,11 @@ def setup(app):
 # 'searchbox.html']``.
 #
 # html_sidebars = {}
+html_logo = 'images/logo.jpg'
+html_theme_options = {
+    'logo_only': True,
+    'style_nav_header_background': 'white',
+}
 
 
 # -- Options for HTMLHelp output ---------------------------------------------

docs/read_the_docs/source/index.rst

@@ -20,7 +20,7 @@ ResStock Documentation
    :maxdepth: 2
    :caption: Contents:
 
-   tutorial/index
+   basic_tutorial/index
    advanced_tutorial/index
-   upgrade_scenario_config
-   outputs/index
+   workflow_inputs/index
+   workflow_outputs/index