Do not zero out one-tailed z-statistics for p-values > 0.5 (neurostuf…

…f#693)

* Do not zero out one-tailed z-statistics for p-values > 0.5

* add comment

* Replace negative values in z with values estimated by CDF

* Add test for p to z conversion

* @yifan0330 Apply suggestions from code review

* Random tests failing. Run black

nimare/decode/continuous.py

@@ -157,7 +157,6 @@ def __init__(
         target_image="z_desc-specificity",
         n_cores=1,
     ):
-
         if meta_estimator is None:
             meta_estimator = MKDAChi2()
         else:

nimare/extract/utils.py

@@ -126,7 +126,7 @@ def _get_dataset_dir(dataset_name, data_dir=None, default_paths=None):
 
     # If not, create a folder in the first writeable directory
     errors = []
-    for (path, is_pre_dir) in paths:
+    for path, is_pre_dir in paths:
         if not is_pre_dir:
             path = os.path.join(path, dataset_name)
 

nimare/io.py

@@ -487,7 +487,6 @@ def convert_neurovault_to_dataset(
 
     dataset_dict = {}
     for coll_name, nv_coll in collection_ids.items():
-
         nv_url = f"https://neurovault.org/api/collections/{nv_coll}/images/?format=json"
         images = requests.get(nv_url).json()
         if "Not found" in images.get("detail", ""):

nimare/meta/cbma/ale.py

@@ -235,7 +235,6 @@ def _compute_null_approximate(self, ma_maps):
         ale_hist = ma_hists[0, :].copy()
 
         for i_exp in range(1, ma_hists.shape[0]):
-
             exp_hist = ma_hists[i_exp, :]
 
             # Find histogram bins with nonzero values for each histogram.

nimare/meta/cbma/mkda.py

@@ -1211,7 +1211,6 @@ def _compute_null_approximate(self, ma_maps):
         stat_hist = ma_hists[0, :].copy()
 
         for i_exp in range(1, ma_hists.shape[0]):
-
             exp_hist = ma_hists[i_exp, :]
 
             # Find histogram bins with nonzero values for each histogram.

nimare/meta/kernel.py

@@ -304,7 +304,6 @@ def __init__(self, r=10, value=1):
         self.value = value
 
     def _transform(self, mask, coordinates):
-
         ijks = coordinates[["i", "j", "k"]].values
         exp_idx = coordinates["id"].values
         transformed = compute_kda_ma(

nimare/meta/utils.py

@@ -223,7 +223,6 @@ def compute_ale_ma(mask, ijks, kernel=None, exp_idx=None, sample_sizes=None, use
     all_coords = []
     all_data = []
     for i_exp, _ in enumerate(exp_idx_uniq):
-
         # Index peaks by experiment
         curr_exp_idx = exp_idx == i_exp
         ijk = ijks[curr_exp_idx]
@@ -271,7 +270,6 @@ def compute_ale_ma(mask, ijks, kernel=None, exp_idx=None, sample_sizes=None, use
                 & (zlk >= 0)
                 & (zhk >= 0)
             ):
-
                 ma_values[xl:xh, yl:yh, zl:zh] = np.maximum(
                     ma_values[xl:xh, yl:yh, zl:zh], kernel[xlk:xhk, ylk:yhk, zlk:zhk]
                 )

nimare/tests/test_estimator_performance.py

@@ -24,6 +24,7 @@
 # PRECOMPUTED FIXTURES
 # --------------------
 
+
 ##########################################
 # random state
 ##########################################

nimare/tests/test_transforms.py

@@ -247,7 +247,7 @@ def test_ddimages_to_coordinates_merge_strategy(testdata_ibma):
 
 
 @pytest.mark.parametrize(
-    "z,tail,expected_p",
+    "expected_z,tail,expected_p",
     [
         (0.0, "two", 1.0),
         (0.0, "one", 0.5),
@@ -256,10 +256,13 @@ def test_ddimages_to_coordinates_merge_strategy(testdata_ibma):
         (-1.959963, "one", 0.975),
         (-1.959963, "two", 0.05),
         ([0.0, 1.959963, -1.959963], "two", [1.0, 0.05, 0.05]),
+        ([0.0, 1.959963, -1.959963], "one", [0.5, 0.025, 0.975]),
     ],
 )
-def test_z_to_p(z, tail, expected_p):
-    """Test z to p conversion."""
-    p = transforms.z_to_p(z, tail)
+def test_z_to_p(expected_z, tail, expected_p):
+    """Test z to p, and p to z conversion."""
+    p = transforms.z_to_p(expected_z, tail)
+    z = transforms.p_to_z(expected_p, tail) * np.sign(expected_z)
 
     assert np.all(np.isclose(p, expected_p))
+    assert np.all(np.isclose(z, expected_z))

nimare/transforms.py

@@ -368,7 +368,6 @@ def transform(self, dataset):
 
         coordinates_dict = {}
         for _, row in images_df.iterrows():
-
             if row["id"] in list(dataset.coordinates["id"]) and self.merge_strategy == "fill":
                 continue
 
@@ -679,12 +678,13 @@ def p_to_z(p, tail="two"):
         Z-statistics (unsigned)
     """
     p = np.array(p)
+
+    # Ensure that no p-values are converted to Inf/NaNs
+    p = np.clip(p, 1.0e-300, 1.0 - 1.0e-16)
     if tail == "two":
         z = stats.norm.isf(p / 2)
     elif tail == "one":
-        z = stats.norm.isf(p)
-        z = np.array(z)
-        z[z < 0] = 0
+        z = np.abs(stats.norm.isf(p))
     else:
         raise ValueError('Argument "tail" must be one of ["one", "two"]')