Merge pull request #24 from WISDEM/issue_23_separate_numeric_nonnumeric

Issue #23 separate numeric and non-numeric values on details sheet and other validation improvements.

CHANGELOG.md

@@ -8,3 +8,17 @@
 - Enhancements to all modules in model.
 - Black box tests.
 - Dictionary based interface to integrate with other modeling codes.
+
+## 2.1.1 (October 9, 2019)
+
+- In the `costs_by_module_type_operation` tab, standardize all costs to USD/kW per project, cost per project, cost per turbine.
+- Improve docstrings in source code.
+- Refactor more functionality into a new `CostModule` class.
+- Clean up logging to use simple `print()` statements which are safe to use in multi-process parallel logging operations.
+
+## 2.1.2 (October 24, 2019)
+
+- Add separated "numeric value" and "non-numeric value" to columns on the details sheet.
+- Add support to test current model output against previously known good model output to guard against regressions when the model is changed.
+- Add support for command line options to control validation, input folder and output folder so that environment variables are not needed.
+- Added documentation about command line operation with flowcharts about how LandBOSSE processes data according to the command line.

installation_instructions/operation_and_folder_structure.md

@@ -117,7 +117,7 @@ For validating LandBOSSE, the input data are stored in the same folder structure
 To validate the model, you would run the following command.
 
 ```
-python main.py --input PATH_TO_VALIDATION_FOLDER --validate
+python main.py --input PATH_TO_FOLDER_WITH_VALIDATION_DATA --output PATH_TO_YOUR_OUTPUT_FOLDER --validate
 ```
 
-This command will run the model to obtain the actual data and will create the `landbosse-validation-result.xlsx`.
+This command will run the model to obtain the actual data and will create the `landbosse-validation-result.xlsx`. It will write the validation 

landbosse/excelio/XlsxFileOperations.py

@@ -153,6 +153,15 @@ def copy_input_data(self):
         copy2(src_project_list_xlsx, dst_project_list_xlsx)
         copytree(src_project_data_dir, dst_project_data_dir)
 
+        src_expected_validation_data = os.path.join(self.landbosse_input_dir(),
+                                                    'landbosse-expected-validation-data.xlsx')
+
+        dst_expected_validation_data = os.path.join(self.landbosse_output_dir(),
+                                                    'landbosse-expected-validation-data.xlsx')
+
+        if os.path.isfile(src_expected_validation_data):
+            copy2(src_expected_validation_data, dst_expected_validation_data)
+
     def timestamp_filename(self, directory, basename, extension):
         """
         This function creates a timestamped filename. It uses a filename in the

landbosse/excelio/XlsxGenerator.py

@@ -150,7 +150,7 @@ def tab_details(self, rows):
         This writes a detailed outputs tab. It takes a list of dictionaries
         as the parameters and in each of those dictionaries it looks at the keys:
 
-        ['project_id', 'module', 'type', 'variable_df_key_col_name', 'unit', 'value']
+        ['project_id', 'module', 'type', 'variable_df_key_col_name', 'unit', 'numeric value', 'non_numeric_value']
 
         The values of each of those keys become each cell in the row.
 
@@ -164,21 +164,53 @@ def tab_details(self, rows):
         worksheet.set_column(4, 4, 17)
         worksheet.set_column(5, 5, 66)
         worksheet.set_column(0, 2, 17)
-        for idx, col_name in enumerate(['project_id', 'module', 'type', 'variable_df_key_col_name', 'unit', 'value', 'last number']):
+
+        for idx, col_name in enumerate(['Project ID', 'Module', 'Variable or DataFrame', 'name', 'unit', 'Numeric value', 'Non-numeric value']):
             worksheet.write(0, idx, col_name, self.header_format)
+
+        # Go through each row and create Excel rows from each of those rows.
         for row_idx, row in enumerate(rows):
             worksheet.write(row_idx + 1, 0, row['project'])
             worksheet.write(row_idx + 1, 1, row['module'])
             worksheet.write(row_idx + 1, 2, row['type'])
             worksheet.write(row_idx + 1, 3, row['variable_df_key_col_name'])
             worksheet.write(row_idx + 1, 4, row['unit'])
-            if type(row['value']) is str or type(row['value']) is int or type(row['value']) is float:
-                worksheet.write(row_idx + 1, 5, row['value'], self.scientific_format)
+
+            value = row['value']
+            value_is_number = self._is_numeric(value)
+            if value_is_number:
+                worksheet.write(row_idx + 1, 5, value, self.scientific_format)
             else:
-                worksheet.write(row_idx + 1, 5, str(row['value']))
+                worksheet.write(row_idx + 1, 6, value)
+
+            # If there is a last_number, which means this is a dataframe row that has a number
+            # at the end, write this into the numeric value column.
+
             if 'last_number' in row:
-                if type(row['last_number']) is int or type(row['last_number']) is float:
-                    worksheet.write(row_idx + 1, 6, row['last_number'], self.scientific_format)
-                else:
-                    worksheet.write(row_idx + 1, 6, str(row['last_number']))
+                worksheet.write(row_idx + 1, 5, row['last_number'], self.scientific_format)
+
         worksheet.freeze_panes(1, 0)  # Freeze the first row.
+
+    def _is_numeric(self, value):
+        """
+        This method tests if a value is a numeric (that is, can be parsed
+        by float()) or non numeric (which cannot be parsed).
+
+        The decision from this method determines whether values go into
+        the numeric or non-numeric columns.
+
+        Parameters
+        ----------
+        value
+            The value to be tested.
+
+        Returns
+        -------
+        bool
+            True if the value is numeric, False otherwise.
+        """
+        try:
+            float(value)
+        except ValueError:
+            return False
+        return True

main.py

@@ -42,7 +42,7 @@
         # Generated based on input_path from command line when --validate option is specified
         # (validation output file must be in inputs folder and must be called 'landbosse-output-validation.xlsx')
         expected_validation_data_path = os.path.join(input_path, 'landbosse-expected-validation-data.xlsx')
-        validation_result_path = os.path.join(input_path, 'landbosse-validation-result.xlsx')
+        validation_result_path = os.path.join(file_ops.landbosse_output_dir(), 'landbosse-validation-result.xlsx')
 
         validator = XlsxValidator()
         validation_was_successful = validator.compare_expected_to_actual(
@@ -54,11 +54,11 @@
             print('Validation passed.')
         else:
             print('Validation failed. See mismatched data above.')
-    else:
-        # XlsxGenerator has a context manager that writes each individual
-        # worksheet to the output .xlsx. Also, copy file input structure.
-        print('Writing final output folder')
-        with XlsxGenerator('landbosse-output', file_ops) as xlsx:
-            xlsx.tab_costs_by_module_type_operation(rows=final_result['module_type_operation_list'])
-            xlsx.tab_details(rows=final_result['details_list'])
-        file_ops.copy_input_data()
+
+    # XlsxGenerator has a context manager that writes each individual
+    # worksheet to the output .xlsx. Also, copy file input structure.
+    print('Writing final output folder')
+    with XlsxGenerator('landbosse-output', file_ops) as xlsx:
+        xlsx.tab_costs_by_module_type_operation(rows=final_result['module_type_operation_list'])
+        xlsx.tab_details(rows=final_result['details_list'])
+    file_ops.copy_input_data()