Re-create tests from VPP Statistics using ReporterStatistics

(refs idaholab#16584)
aeslaughter · Jun 2, 2021 · 5edb540 · 5edb540
1 parent 8da09ac
commit 5edb540
Show file tree

Hide file tree

Showing 31 changed files with 855 additions and 3 deletions.
diff --git a/modules/stochastic_tools/src/reporters/StatisticsReporter.C b/modules/stochastic_tools/src/reporters/StatisticsReporter.C
@@ -122,8 +122,11 @@ StatisticsReporter::StatisticsReporter(const InputParameters & parameters)
     }
   }
 
-  if (!isParamValid("reporters") && !isParamValid("vectorpostprocessors"))
-    mooseError("The 'vectorpostprocessors' and/or 'reporters' parameters must be defined.");
+  if ((!isParamValid("reporters") && !isParamValid("vectorpostprocessors")) ||
+      (getParam<std::vector<ReporterName>>("reporters").empty() &&
+       getParam<std::vector<VectorPostprocessorName>>("vectorpostprocessors").empty()))
+    mooseError(
+        "The 'vectorpostprocessors' and/or 'reporters' parameters must be defined and non-empty.");
 }
 
 void

diff --git a/modules/stochastic_tools/src/utils/BootstrapCalculators.C b/modules/stochastic_tools/src/utils/BootstrapCalculators.C
@@ -175,7 +175,8 @@ BiasCorrectedAccelerated<InType, OutType>::compute(const InType & data,
                                                    const bool is_distributed) const
 {
   if (is_distributed)
-    mooseError("BiasCorrectedAccelerated does not work with distributed data at this time.");
+    mooseError("Due to the computational demands, the BiasCorrectedAccelerated does not work with "
+               "distributed data.");
 
   // Bootstrap estimates
   const std::vector<OutType> values = this->computeBootstrapEstimates(data, is_distributed);

diff --git a/...rter/gold/statistics_reporter_ci_out.json → ...asic/gold/statistics_reporter_ci_out.json b/...rter/gold/statistics_reporter_ci_out.json → ...asic/gold/statistics_reporter_ci_out.json
diff --git a/...eporter/gold/statistics_reporter_out.json → ...s/basic/gold/statistics_reporter_out.json b/...eporter/gold/statistics_reporter_out.json → ...s/basic/gold/statistics_reporter_out.json
diff --git a/..._reporter/gold/statistics_vpp_ci_out.json → ...ers/basic/gold/statistics_vpp_ci_out.json b/..._reporter/gold/statistics_vpp_ci_out.json → ...ers/basic/gold/statistics_vpp_ci_out.json
diff --git a/...ics_reporter/gold/statistics_vpp_out.json → ...orters/basic/gold/statistics_vpp_out.json b/...ics_reporter/gold/statistics_vpp_out.json → ...orters/basic/gold/statistics_vpp_out.json
diff --git a/...statistics_reporter/statistics_reporter.i → ...sts/reporters/basic/statistics_reporter.i b/...statistics_reporter/statistics_reporter.i → ...sts/reporters/basic/statistics_reporter.i
diff --git a/...tistics_reporter/statistics_reporter_ci.i → .../reporters/basic/statistics_reporter_ci.i b/...tistics_reporter/statistics_reporter_ci.i → .../reporters/basic/statistics_reporter_ci.i
diff --git a/...tics_reporter/statistics_reporter_error.i → ...porters/basic/statistics_reporter_error.i b/...tics_reporter/statistics_reporter_error.i → ...porters/basic/statistics_reporter_error.i
diff --git a/...ters/statistics_reporter/statistics_vpp.i → ...st/tests/reporters/basic/statistics_vpp.i b/...ters/statistics_reporter/statistics_vpp.i → ...st/tests/reporters/basic/statistics_vpp.i
diff --git a/...s/statistics_reporter/statistics_vpp_ci.i → ...tests/reporters/basic/statistics_vpp_ci.i b/...s/statistics_reporter/statistics_vpp_ci.i → ...tests/reporters/basic/statistics_vpp_ci.i
diff --git a/...tests/reporters/statistics_reporter/tests → ...ic_tools/test/tests/reporters/basic/tests b/...tests/reporters/statistics_reporter/tests → ...ic_tools/test/tests/reporters/basic/tests
diff --git a/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/bca/bca.i b/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/bca/bca.i
@@ -0,0 +1,37 @@
+[StochasticTools]
+[]
+
+[VectorPostprocessors]
+  # Table 14.1 in Efron and Tibshirani, 1993
+  [A]
+    type = ConstantVectorPostprocessor
+    value = '48 36 20 29 42 42 20 42 22 41 45 14 6 0 33 28 34 4 32 24 47 41 24 26 30 41'
+    outputs = none
+  []
+  [B]
+    type = ConstantVectorPostprocessor
+    value = '42 33 16 39 38 36 15 33 20 43 34 22 7 15 34 29 41 13 38 25 27 41 28 14 28 40'
+    outputs = none
+  []
+[]
+
+[Reporters]
+  # Reproduce Table 13.1 in Efron and Tibshirani, 1993
+  [stats]
+    type = StatisticsReporter
+    vectorpostprocessors = 'A B'
+    compute = 'mean'
+    ci_method = 'bca'
+    ci_levels = '0.025 0.05 0.1 0.16 0.5 0.84 0.9 0.95 0.975'
+    ci_replicates = 1000
+    ci_seed = 1980
+  []
+[]
+
+[Outputs]
+  execute_on = FINAL
+  [out]
+    type = JSON
+    execute_system_information_on = NONE
+  []
+[]
diff --git a/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/bca/bca_distributed.i b/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/bca/bca_distributed.i
@@ -0,0 +1,29 @@
+[StochasticTools]
+[]
+
+[VectorPostprocessors]
+  [data]
+    type = TestDistributedVectorPostprocessor
+    parallel_type = distributed
+    outputs = none
+  []
+[]
+
+[Reporters]
+  [stats]
+    type = StatisticsReporter
+    vectorpostprocessors = 'data'
+    compute = 'mean'
+    ci_method = 'bca'
+    ci_levels = '0.025 0.05 0.1 0.16 0.5 0.84 0.9 0.95 0.99'
+    ci_replicates = 10
+  []
+[]
+
+[Outputs]
+  execute_on = FINAL
+  [out]
+    type = JSON
+    execute_system_information_on = NONE
+  []
+[]
diff --git a/...chastic_tools/test/tests/reporters/bootstrap_statistics/bca/gold/bca_distributed_out.json b/...chastic_tools/test/tests/reporters/bootstrap_statistics/bca/gold/bca_distributed_out.json
@@ -0,0 +1,53 @@
+{
+    "time_steps": [
+        {
+            "reporters": {
+                "stats": {
+                    "info": {
+                        "confidence_intervals": {
+                            "levels": [
+                                0.025,
+                                0.05,
+                                0.1,
+                                0.16,
+                                0.5,
+                                0.84,
+                                0.9,
+                                0.95,
+                                0.99
+                            ],
+                            "method": "bca",
+                            "replicates": 10,
+                            "seed": 1
+                        },
+                        "name": "stats",
+                        "type": "StatisticsReporter"
+                    },
+                    "values": [
+                        {
+                            "name": "data_data_MEAN",
+                            "stat": "MEAN",
+                            "type": "std::pair<double, std::vector<double> >",
+                            "value": [
+                                204.5,
+                                [
+                                    180.5,
+                                    180.5,
+                                    180.5,
+                                    180.5,
+                                    195.0,
+                                    214.53333333333333,
+                                    217.83333333333334,
+                                    218.9,
+                                    221.13333333333333
+                                ]
+                            ]
+                        }
+                    ]
+                }
+            },
+            "time": 2.0,
+            "time_step": 2
+        }
+    ]
+}
diff --git a/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/bca/gold/bca_out.json b/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/bca/gold/bca_out.json
@@ -0,0 +1,72 @@
+{
+    "time_steps": [
+        {
+            "reporters": {
+                "stats": {
+                    "info": {
+                        "confidence_intervals": {
+                            "levels": [
+                                0.025,
+                                0.05,
+                                0.1,
+                                0.16,
+                                0.5,
+                                0.84,
+                                0.9,
+                                0.95,
+                                0.975
+                            ],
+                            "method": "bca",
+                            "replicates": 1000,
+                            "seed": 1980
+                        },
+                        "name": "stats",
+                        "type": "StatisticsReporter"
+                    },
+                    "values": [
+                        {
+                            "name": "A_value_MEAN",
+                            "stat": "MEAN",
+                            "type": "std::pair<double, std::vector<double> >",
+                            "value": [
+                                29.653846153846153,
+                                [
+                                    24.153846153846153,
+                                    25.076923076923077,
+                                    25.846153846153847,
+                                    26.576923076923077,
+                                    29.423076923076923,
+                                    31.884615384615383,
+                                    32.5,
+                                    33.23076923076923,
+                                    34.07692307692308
+                                ]
+                            ]
+                        },
+                        {
+                            "name": "B_value_MEAN",
+                            "stat": "MEAN",
+                            "type": "std::pair<double, std::vector<double> >",
+                            "value": [
+                                28.884615384615383,
+                                [
+                                    24.423076923076923,
+                                    25.192307692307693,
+                                    26.0,
+                                    26.615384615384617,
+                                    28.692307692307693,
+                                    30.692307692307693,
+                                    31.26923076923077,
+                                    31.923076923076923,
+                                    32.38461538461539
+                                ]
+                            ]
+                        }
+                    ]
+                }
+            },
+            "time": 2.0,
+            "time_step": 2
+        }
+    ]
+}
diff --git a/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/bca/tests b/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/bca/tests
@@ -0,0 +1,39 @@
+[Tests]
+  design = StatisticsReporter.md
+  issues = '#14409 #16584'
+
+  [bca]
+    requirement = "The system shall support computing bias corrected and accelerated confidence level intervals of statics"
+
+    [basic]
+      type = JSONDiff
+      input = bca.i
+      jsondiff = bca_out.json
+
+      detail = "of a vector of data"
+    []
+
+    [replicated]
+      type = JSONDiff
+      input = bca_distributed.i
+      jsondiff = bca_distributed_out.json
+      cli_args = "VectorPostprocessors/data/parallel_type=replicated"
+      min_parallel = 3
+      max_parallel = 3
+      allow_test_objects = true
+
+      detail = "using data that is replicated and"
+    []
+
+    [distributed]
+      type = RunException
+      input = bca_distributed.i
+      expect_err = "Due to the computational demands, the BiasCorrectedAccelerated does not work with distributed data"
+      min_parallel = 3
+      max_parallel = 3
+      allow_test_objects = true
+
+      detail = "errors when data distributed across processors."
+    []
+  []
+[]
diff --git a/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/errors/error.i b/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/errors/error.i
@@ -0,0 +1,29 @@
+[StochasticTools]
+[]
+
+[VectorPostprocessors]
+  [treatment]
+    type = ConstantVectorPostprocessor
+    value = '94 197 16 38 99 141 23'
+    outputs = none
+  []
+  [control]
+    type = ConstantVectorPostprocessor
+    value = '52 104 146 10 51 30 40 27 46'
+    outputs = none
+  []
+[]
+
+[Reporters]
+  [stats]
+    type = StatisticsReporter
+    compute = 'mean stderr'
+    ci_method = 'percentile'
+    ci_levels = '0.025 0.05 0.1 0.16 0.5'
+    ci_replicates = 1000
+    ci_seed = 1980
+  []
+[]
+
+[Outputs]
+[]
diff --git a/...ests/reporters/bootstrap_statistics/errors/gold/percentile_distributed_out_stats_0001.csv b/...ests/reporters/bootstrap_statistics/errors/gold/percentile_distributed_out_stats_0001.csv
@@ -0,0 +1,11 @@
+stat_type,data_data
+3,204.5
+3.025,180.5
+3.05,180.5
+3.1,193.63333333333
+3.16,193.63333333333
+3.5,204.56666666667
+3.84,221.13333333333
+3.9,221.13333333333
+3.95,227.93333333333
+3.975,227.93333333333
diff --git a/...tools/test/tests/reporters/bootstrap_statistics/errors/gold/percentile_out_stats_0001.csv b/...tools/test/tests/reporters/bootstrap_statistics/errors/gold/percentile_out_stats_0001.csv
@@ -0,0 +1,21 @@
+stat_type,treatment_value,control_value
+3,86.857142857143,56.222222222222
+3.025,42.428571428571,33.111111111111
+3.05,48.285714285714,36.222222222222
+3.1,55.714285714286,40.555555555556
+3.16,62.714285714286,43.555555555556
+3.5,85.571428571429,55.444444444444
+3.84,109.71428571429,68.777777777778
+3.9,116.57142857143,72.444444444444
+3.95,125.42857142857,78.222222222222
+3.975,132.42857142857,82.555555555556
+7,25.235489533052,14.158603369779
+7.025,11.389217958854,4.2233185711105
+7.05,13.197093062721,4.801941479788
+7.1,14.377088727281,6.780281868936
+7.16,16.512415448552,8.6449157255137
+7.5,23.595658542215,13.498399539333
+7.84,27.979462827868,16.66814808231
+7.9,29.364470178151,17.411770532787
+7.95,30.838067296084,18.320786952746
+7.975,31.892997120154,18.990819744495
diff --git a/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/errors/tests b/modules/stochastic_tools/test/tests/reporters/bootstrap_statistics/errors/tests
@@ -0,0 +1,44 @@
+[Tests]
+  design = StatisticsReporter.md
+  issues = '#14409 #16584'
+
+  [errors]
+    requirement = "The system shall error when computing confidence level intervals when"
+
+    [no_ci_levels]
+      type = RunException
+      input = error.i
+      cli_args = "Reporters/stats/ci_levels=''"
+      expect_err = "If the 'ci_method' parameter is supplied then the 'ci_levels' must also be supplied with values in \(0, 1\)"
+
+      detail = "the confidence level intervals are omitted;"
+    []
+
+    [ci_levels_zero]
+      type = RunException
+      input = error.i
+      cli_args = "Reporters/stats/ci_levels='0'"
+      expect_err = "The supplied levels must be greater than zero"
+
+      detail = "the confidence level intervals are omitted;"
+    []
+
+    [ci_levels_one]
+      type = RunException
+      input = error.i
+      cli_args = "Reporters/stats/ci_levels='1'"
+      expect_err = "The supplied levels must be less than 1\.0"
+
+      detail = "the confidence level intervals are out of range;"
+    []
+
+    [no_input]
+      type = RunException
+      input = error.i
+      cli_args = "Reporters/stats/vectorpostprocessors=''"
+      expect_err = "The 'vectorpostprocessors' and/or 'reporters' parameters must be defined and non-empty"
+
+      detail = "input is not provided."
+    []
+  []
+[]