feat: Allow failing toys

src/pyhf/infer/calculators.py

@@ -675,6 +675,7 @@
         test_stat="qtilde",
         ntoys=2000,
         track_progress=True,
+        skip_failing_toys=False,
     ):
         r"""
         Toy-based Calculator.
@@ -705,6 +706,7 @@
             ~pyhf.infer.calculators.ToyCalculator: The calculator for toy-based quantities.
 
         """
+        self.skip_failing_toys = skip_failing_toys
         self.ntoys = ntoys
         self.data = data
         self.pdf = pdf
@@ -754,6 +756,9 @@
             Tuple (~pyhf.infer.calculators.EmpiricalDistribution): The distributions under the hypotheses.
 
         """
+
+        print('skip?', self.skip_failing_toys)
+
         tensorlib, _ = get_backend()
         sample_shape = (self.ntoys,)
 
@@ -792,29 +797,45 @@
 
         signal_teststat = []
         for sample in tqdm.tqdm(signal_sample, **tqdm_options, desc='Signal-like'):
-            signal_teststat.append(
-                teststat_func(
+            try:
+                value = teststat_func(
                     poi_test,
                     sample,
                     self.pdf,
                     self.init_pars,
                     self.par_bounds,
                     self.fixed_params,
                 )
-            )
+            except:
+                if self.skip_failing_toys:
+                    value = None
+                    print(f'signal failed for: {sample}')
+                else:
+                    raise
+
+            if (value is not None) and (tensorlib.isfinite(value)):
+                signal_teststat.append(value)
 
         bkg_teststat = []
         for sample in tqdm.tqdm(bkg_sample, **tqdm_options, desc='Background-like'):
-            bkg_teststat.append(
-                teststat_func(
+            try:
+                value = teststat_func(
                     poi_test,
                     sample,
                     self.pdf,
                     self.init_pars,
                     self.par_bounds,
                     self.fixed_params,
                 )
-            )
+            except:
+                if self.skip_failing_toys:
+                    value = None
+                    print(f'background failed for: {sample}')
+                else:
+                    raise
+
+            if (value is not None) and (tensorlib.isfinite(value)):
+                bkg_teststat.append(value)
 
         s_plus_b = EmpiricalDistribution(tensorlib.astensor(signal_teststat))
         b_only = EmpiricalDistribution(tensorlib.astensor(bkg_teststat))