Repository navigation
Expand file tree
/
Copy path_mantel.py
More file actions
1103 lines (927 loc) · 42 KB
/
Copy path_mantel.py
File metadata and controls
1103 lines (927 loc) · 42 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file LICENSE.txt, distributed with this software.
# ----------------------------------------------------------------------------
from __future__ import annotations
from itertools import combinations
from warnings import warn
from typing import TYPE_CHECKING
import numpy as np
import pandas as pd
import scipy.special
from scipy.stats import kendalltau, ConstantInputWarning, NearConstantInputWarning
from ._cutils import mantel_perm_pearsonr_cy, mantel_perm_pearsonr_condensed_cy
from ._gpu import _numba_gpu_module_for, _mark_gpu_unavailable
from ._mantel_gpu import _run_mantel_gpu
from skbio.stats.distance import DistanceMatrix
from skbio.util import get_rng
from skbio.util._array import ingest_array, _get_array
from skbio.util._decorator import array_api_doc
from skbio._config import _resolve_engine
import array_api_compat as _aac
try:
from numba import njit, prange
NUMBA_AVAILABLE = True
except ImportError:
NUMBA_AVAILABLE = False
if TYPE_CHECKING: # pragma: no cover
from collections.abc import Sequence
from ._base import DistanceMatrix
from numpy.typing import ArrayLike
from skbio.util._typing import SeedLike
if NUMBA_AVAILABLE:
@njit(parallel=True)
def _mantel_perm_pearsonr_nb(
x_data, perm_order, xmean, normxm, ym_normalized, permuted_stats
):
"""Fused permute, normalize, and Pearson correlation for Mantel test.
Replaces the following Python code::
def _mantel_perm_pearsonr_one(x_flat, xmean, normxm, ym_normalized):
xm_normalized = (x_flat - xmean) / normxm
one_stat = np.dot(xm_normalized, ym_normalized)
one_stat = max(min(one_stat, 1.0), -1.0)
return one_stat
permuted_stats = np.array([
_mantel_perm_pearsonr_one(
distmat_reorder_condensed(x._data, perm_order[p]), ...
) for p in range(permutations)
])
Takes a pre-computed permutation order matrix and normalizes values on
the fly using mul and add (derived from xmean and normxm) instead of
materializing the full permuted matrix. Each permutation is independent
and computed in parallel via prange.
Parameters
----------
x_data : 2D array_like
Full distance matrix (n x n).
perm_order : 2D array_like
Permutation index array (n_perm x n). Each row is a permutation
of range(n) specifying the reordering of rows and columns.
xmean : float
Mean of the condensed form of x.
normxm : float
Norm of (x_flat - xmean).
ym_normalized : 1D array_like
(y_flat - ymean) / normym, pre-normalized.
permuted_stats : 1D array_like
Output array (n_perm,) to write correlation coefficients into.
"""
n = x_data.shape[0]
n_perm = perm_order.shape[0]
mul = 1.0 / normxm
add = -xmean / normxm
for p in prange(n_perm):
my_ps = 0.0
for row in range(n - 1):
vrow = perm_order[p, row]
row_start = row * (n - 1) - ((row - 1) * row) // 2
for icol in range(n - row - 1):
col = icol + row + 1
yval = ym_normalized[row_start + icol]
xval = x_data[vrow, perm_order[p, col]] * mul + add
my_ps = yval * xval + my_ps
if my_ps > 1.0:
my_ps = 1.0
elif my_ps < -1.0:
my_ps = -1.0
permuted_stats[p] = my_ps
@njit(parallel=True)
def _mantel_perm_pearsonr_condensed_nb(
x_data, perm_order, xmean, normxm, ym_normalized, permuted_stats
):
"""Fused permute, normalize, and Pearson for condensed Mantel test.
Same as _mantel_perm_pearsonr_nb but accepts x_data in condensed
form (1D array of length n*(n-1)/2) instead of the full 2D matrix.
Uses condensed_index formula to map (vrow, vcol) pairs from the
permuted ordering back to the 1D condensed array.
Parameters
----------
x_data : 1D array_like
Condensed distance matrix (lower triangle only).
perm_order : 2D array_like
Permutation index array (n_perm x n).
xmean : float
Mean of the condensed form of x.
normxm : float
Norm of (x_flat - xmean).
ym_normalized : 1D array_like
(y_flat - ymean) / normym, pre-normalized.
permuted_stats : 1D array_like
Output array (n_perm,) to write correlation coefficients into.
"""
n_perm = perm_order.shape[0]
n = perm_order.shape[1]
mul = 1.0 / normxm
add = -xmean / normxm
for p in prange(n_perm):
my_ps = 0.0
for row in range(n - 1):
vrow = perm_order[p, row]
row_start = row * (n - 1) - ((row - 1) * row) // 2
for icol in range(n - row - 1):
col = icol + row + 1
vcol = perm_order[p, col]
if vrow < vcol:
x_idx = vrow * n + vcol - ((vrow + 2) * (vrow + 1)) // 2
else:
x_idx = vcol * n + vrow - ((vcol + 2) * (vcol + 1)) // 2
yval = ym_normalized[row_start + icol]
xval = x_data[x_idx] * mul + add
my_ps = yval * xval + my_ps
if my_ps > 1.0:
my_ps = 1.0
elif my_ps < -1.0:
my_ps = -1.0
permuted_stats[p] = my_ps
@array_api_doc(backends=["numpy", "jax", "torch", "cupy"])
def mantel(
x: DistanceMatrix | ArrayLike,
y: DistanceMatrix | ArrayLike,
method: str = "pearson",
permutations: int = 999,
alternative: str = "two-sided",
strict: bool = True,
lookup: dict[str, str] | None = None,
seed: SeedLike | None = None,
engine: str | None = None,
) -> tuple[float, float, int]:
r"""Compute correlation between distance matrices using the Mantel test.
The Mantel test compares two distance matrices by computing the correlation
between the distances in the lower (or upper) triangular portions of the
symmetric distance matrices. Correlation can be computed using Pearson's
product-moment correlation coefficient or Spearman's rank correlation
coefficient.
As defined in [1]_, the Mantel test computes a test statistic :math:`r_M`
given two symmetric distance matrices :math:`D_X` and :math:`D_Y`.
:math:`r_M` is defined as
.. math::
r_M=\frac{1}{d-1}\sum_{i=1}^{n-1}\sum_{j=i+1}^{n}
stand(D_X)_{ij}stand(D_Y)_{ij}
where
.. math::
d=\frac{n(n-1)}{2}
and :math:`n` is the number of rows/columns in each of the distance
matrices. :math:`stand(D_X)` and :math:`stand(D_Y)` are distance matrices
with their upper triangles containing standardized distances. Note that
since :math:`D_X` and :math:`D_Y` are symmetric, the lower triangular
portions of the matrices could equivalently have been used instead of the
upper triangular portions (the current function behaves in this manner).
If ``method='spearman'``, the above equation operates on ranked distances
instead of the original distances.
Statistical significance is assessed via a permutation test. The rows and
columns of the first distance matrix (`x`) are randomly permuted a
number of times (controlled via `permutations`). A correlation coefficient
is computed for each permutation and the p-value is the proportion of
permuted correlation coefficients that are equal to or more extreme
than the original (unpermuted) correlation coefficient. Whether a permuted
correlation coefficient is 'more extreme' than the original correlation
coefficient depends on the alternative hypothesis (controlled via
`alternative`).
Parameters
----------
x, y : DistanceMatrix or array_like
Input distance matrices to compare. If `x` and `y` are both
``DistanceMatrix`` instances, they will be reordered based on matching
IDs (see `strict` and `lookup` below for handling matching/mismatching
IDs); thus they are not required to be in the same ID order. If `x` and
`y` are ``array_like``, no reordering is applied and both matrices must
have the same shape. In either case, `x` and `y` must be at least 3x3
in size *after* reordering and matching of IDs.
method : {'pearson', 'spearman','kendalltau'}
Method used to compute the correlation between distance matrices.
permutations : int, optional
Number of times to randomly permute `x` when assessing statistical
significance. Must be greater than or equal to zero. If zero,
statistical significance calculations will be skipped and the p-value
will be ``np.nan``.
alternative : {'two-sided', 'greater', 'less'}
Alternative hypothesis to use when calculating statistical
significance. The default ``'two-sided'`` alternative hypothesis
calculates the proportion of permuted correlation coefficients whose
magnitude (i.e. after taking the absolute value) is greater than or
equal to the absolute value of the original correlation coefficient.
``'greater'`` calculates the proportion of permuted coefficients that
are greater than or equal to the original coefficient. ``'less'``
calculates the proportion of permuted coefficients that are less than
or equal to the original coefficient.
strict : bool, optional
If ``True``, raises a ``ValueError`` if IDs are found that do not exist
in both distance matrices. If ``False``, any nonmatching IDs are
discarded before running the test. See `n` (in Returns section below)
for the number of matching IDs that were used in the test. This
parameter is ignored if `x` and `y` are ``array_like``.
lookup : dict, optional
Maps each ID in the distance matrices to a new ID. Used to match up IDs
across distance matrices prior to running the Mantel test. If the IDs
already match between the distance matrices, this parameter is not
necessary. This parameter is disallowed if `x` and `y` are
``array_like``.
seed : int, Generator or RandomState, optional
A user-provided random seed or random generator instance. See
:func:`details <skbio.util.get_rng>`.
.. versionadded:: 0.6.3
engine : {'cython', 'numba', 'fast'}, optional
Compute engine for permutation calculations with the 'pearson' and 'spearman'
methods. Other methods are not impacted. If None (default), the global
``compute_engine`` setting will be used. 'fast' selects Numba if installed,
otherwise Cython. See :ref:`compute_engines` for details and requirements.
.. versionadded:: 0.7.4
Returns
-------
corr_coeff : float
Correlation coefficient of the test (depends on `method`).
p_value : float
p-value of the test.
n : int
Number of rows/columns in each of the distance matrices, after any
reordering/matching of IDs. If ``strict=False``, nonmatching IDs may
have been discarded from one or both of the distance matrices prior to
running the Mantel test, so this value may be important as it indicates
the *actual* size of the matrices that were compared.
Raises
------
ValueError
If `x` and `y` are not at least 3x3 in size after reordering/matching
of IDs, or an invalid `method`, number of `permutations`, or
`alternative` are provided.
TypeError
If `x` and `y` are not both ``DistanceMatrix`` instances or
``array_like``.
See Also
--------
DistanceMatrix
scipy.stats.pearsonr
scipy.stats.spearmanr
pwmantel
Notes
-----
The Mantel test was first described in [2]_. The general algorithm and
interface are similar to ``vegan::mantel``, available in R's vegan
package [3]_.
This function uses parallel computation for improved performance. See the
:ref:`parallelization guide <parallelization>` for information on controlling the
number of threads used.
On GPU-resident distance matrices with ``engine='numba'``, a fused GPU kernel
runs on CuPy or PyTorch matrices, on both CUDA and ROCm devices. The exception
is ROCm PyTorch on stacks where a Numba HIP kernel cannot be compiled after
ROCm PyTorch has been imported in the same process; those matrices fall back
to the array-API path, which runs on the device regardless. The result is
identical across all paths.
With ``engine='numba'``, GPU buffers must belong to the default device. On a
system with several devices, the default must be changed to match the buffer
ownership before this function is invoked, through ``select_device`` of the
Numba GPU module in use: ``numba_cuda_mlir.cuda`` or ``numba.cuda`` on CUDA,
``numba.hip`` on ROCm.
A mismatch is not reported when the kernel is launched, and on ROCm it has
been observed to leave the GPU context unusable for the rest of the process.
The array-API path, taken when ``engine='numba'`` is not requested, honors
whichever device the input is on.
``np.nan`` will be returned for the p-value if `permutations` is zero or if
the correlation coefficient is ``np.nan``. The correlation coefficient will
be ``np.nan`` if one or both of the inputs does not have any variation
(i.e. the distances are all constant) and ``method='spearman'``.
References
----------
.. [1] Legendre, P. and Legendre, L. (2012) Numerical Ecology. 3rd English
Edition. Elsevier.
.. [2] Mantel, N. (1967). "The detection of disease clustering and a
generalized regression approach". Cancer Research 27 (2): 209-220. PMID
6018555.
.. [3] http://cran.r-project.org/web/packages/vegan/index.html
Examples
--------
Import the functionality we'll use in the following examples:
>>> from skbio import DistanceMatrix
>>> from skbio.stats.distance import mantel
Define two 3x3 distance matrices:
>>> x = DistanceMatrix([[0, 1, 2],
... [1, 0, 3],
... [2, 3, 0]])
>>> y = DistanceMatrix([[0, 2, 7],
... [2, 0, 6],
... [7, 6, 0]])
Compute the Pearson correlation between them and assess significance using
a two-sided test with 999 permutations:
>>> coeff, p_value, n = mantel(x, y)
>>> print(round(coeff, 4))
0.7559
Thus, we see a moderate-to-strong positive correlation (:math:`r_M=0.7559`)
between the two matrices.
In the previous example, the distance matrices (``x`` and ``y``) have the
same IDs, in the same order:
>>> x.ids
('0', '1', '2')
>>> y.ids
('0', '1', '2')
If necessary, ``mantel`` will reorder the distance matrices prior to
running the test. The function also supports a ``lookup`` dictionary that
maps distance matrix IDs to new IDs, providing a way to match IDs between
distance matrices prior to running the Mantel test.
For example, let's reassign the distance matrices' IDs so that there are no
matching IDs between them:
>>> x.ids = ('a', 'b', 'c')
>>> y.ids = ('d', 'e', 'f')
If we rerun ``mantel``, we get the following error notifying us that there
are nonmatching IDs (this is the default behavior with ``strict=True``):
>>> mantel(x, y)
Traceback (most recent call last):
...
ValueError: IDs exist that are not in both distance matrices.
If we pass ``strict=False`` to ignore/discard nonmatching IDs, we see that
no matches exist between `x` and `y`, so the Mantel test still cannot be
run:
>>> mantel(x, y, strict=False)
Traceback (most recent call last):
...
ValueError: No matching IDs exist between the distance matrices.
To work around this, we can define a ``lookup`` dictionary to specify how
the IDs should be matched between distance matrices:
>>> lookup = {'a': 'A', 'b': 'B', 'c': 'C',
... 'd': 'A', 'e': 'B', 'f': 'C'}
``lookup`` maps each ID to ``'A'``, ``'B'``, or ``'C'``. If we rerun
``mantel`` with ``lookup``, we get the same results as the original
example where all distance matrix IDs matched:
>>> coeff, p_value, n = mantel(x, y, lookup=lookup)
>>> print(round(coeff, 4))
0.7559
``mantel`` also accepts input that is ``array_like``. For example, if we
redefine `x` and `y` as nested Python lists instead of ``DistanceMatrix``
instances, we obtain the same result:
>>> x = [[0, 1, 2],
... [1, 0, 3],
... [2, 3, 0]]
>>> y = [[0, 2, 7],
... [2, 0, 6],
... [7, 6, 0]]
>>> coeff, p_value, n = mantel(x, y)
>>> print(round(coeff, 4))
0.7559
It is import to note that reordering/matching of IDs (and hence the
``strict`` and ``lookup`` parameters) do not apply when input is
``array_like`` because there is no notion of IDs.
"""
if method in ("pearson", "spearman"):
special = True
elif method == "kendalltau":
corr_func = kendalltau
special = False # set to true, if we have a dedicated implementation
else:
raise ValueError("Invalid correlation method '%s'." % method)
engine = _resolve_engine(
engine,
("cython", "numba"),
fast="numba" if NUMBA_AVAILABLE else "cython",
)
if permutations < 0:
raise ValueError(
"Number of permutations must be greater than or equal to zero."
)
if alternative not in ("two-sided", "greater", "less"):
raise ValueError("Invalid alternative hypothesis '%s'." % alternative)
# A DistanceMatrix backed by a non-NumPy (e.g. GPU-resident) buffer is
# dispatched to the backend-agnostic xp path for pearson/spearman so the
# matrices stay on their device. kendalltau has no xp path and falls through
# to the host/scipy route below, where _order_dms materializes on host.
x_is_xp = isinstance(x, DistanceMatrix) and not _aac.is_numpy_array(x.data)
y_is_xp = isinstance(y, DistanceMatrix) and not _aac.is_numpy_array(y.data)
if special and (x_is_xp or y_is_xp):
if not (x_is_xp and y_is_xp) or (
_aac.array_namespace(x.data) is not _aac.array_namespace(y.data)
):
# mixed or mismatched backends: fall back to the NumPy path below with
# a warning rather than a hard failure.
warn(
"x and y are not both DistanceMatrix objects on the same array "
"backend; falling back to the NumPy path.",
UserWarning,
)
else:
# both are non-NumPy DistanceMatrix objects on the same backend: align
# ids exactly like the NumPy path; the reorder/filter stays on the
# input's device (distmat_reorder is array-API aware).
x, y = _order_dms(x, y, strict=strict, lookup=lookup)
# With engine="numba" and a matching Numba GPU backend, run the fused
# GPU kernel on the device-resident matrices; otherwise take the
# backend-agnostic array-API path.
gpu = _numba_gpu_module_for(x.data) if engine == "numba" else None
if gpu is not None:
try:
return _run_mantel_gpu(
gpu,
x.data,
y.data,
permutations,
seed,
alternative,
spearman=(method == "spearman"),
)
except Exception:
# The fused kernel could not build/run on this stack; record
# it and fall back to the array-API path (correct anywhere).
_mark_gpu_unavailable(x.data)
return _mantel_stats_pearson_xp(
x.data,
y.data,
permutations,
seed,
alternative,
spearman=(method == "spearman"),
)
# Default NumPy path: reached for kendalltau (no xp path), when neither input
# is a non-NumPy array, or when the inputs were not all on the same backend
# (the fallback above). A non-NumPy DistanceMatrix is materialized on the host
# here (SciPy's kendalltau is host-only, and _order_dms runs on host).
if x_is_xp:
x = DistanceMatrix(_get_array(x.data, to_numpy=True), x.ids)
if y_is_xp:
y = DistanceMatrix(_get_array(y.data, to_numpy=True), y.ids)
x, y = _order_dms(x, y, strict=strict, lookup=lookup)
n = x.shape[0]
if n < 3:
raise ValueError(
"Distance matrices must have at least 3 matching IDs "
"between them (i.e., minimum 3x3 in size)."
)
if special:
if method == "pearson":
orig_stat, comp_stat, permuted_stats = _mantel_stats_pearson(
x, y, permutations, seed, engine
)
else:
orig_stat, comp_stat, permuted_stats = _mantel_stats_spearman(
x, y, permutations, seed, engine
)
else:
x_flat = x.condensed_form()
y_flat = y.condensed_form()
orig_stat = comp_stat = corr_func(x_flat, y_flat)[0]
del x_flat
permuted_stats = []
if not (permutations == 0 or np.isnan(orig_stat)):
# This path permutes inside the loop, so it needs one generator that
# advances across iterations; a seed would repeat the same draw.
rng = get_rng(seed)
perm_gen = (
corr_func(x.permute(condensed=True, seed=rng), y_flat)[0]
for _ in range(permutations)
)
permuted_stats = np.fromiter(perm_gen, float, count=permutations)
del y_flat
if permutations == 0 or np.isnan(orig_stat):
p_value = np.nan
else:
if alternative == "two-sided":
count_better = (np.absolute(permuted_stats) >= np.absolute(comp_stat)).sum()
elif alternative == "greater":
count_better = (permuted_stats >= comp_stat).sum()
else:
count_better = (permuted_stats <= comp_stat).sum()
p_value = (count_better + 1) / (permutations + 1)
return orig_stat, p_value, n
def _upper_tri_xp(xp, n, device=None):
"""Row-major upper-triangle (i < j) index vectors, matching condensed order."""
idx = xp.arange(n, device=device)
ones = xp.ones(n, dtype=idx.dtype, device=device)
ii = idx[:, None] * ones[None, :]
jj = ones[:, None] * idx[None, :]
mask = idx[:, None] < idx[None, :]
return xp.astype(ii[mask], idx.dtype), xp.astype(jj[mask], idx.dtype)
def _xp_rank_average(xp, a):
"""Average-tie ranks, matching ``scipy.stats.rankdata`` (method='average')."""
n = a.shape[0]
dev = _device(a)
sorter = xp.argsort(a, stable=True)
inv = xp.argsort(sorter) # inverse permutation without item assignment
a_sorted = a[sorter]
obs = xp.concat([xp.asarray([True], device=dev), a_sorted[1:] != a_sorted[:-1]])
dense = xp.cumulative_sum(xp.astype(obs, xp.int64), axis=0)[inv]
count_nz = xp.nonzero(obs)[0]
count = xp.concat([count_nz, xp.asarray([n], dtype=count_nz.dtype, device=dev)])
return 0.5 * (
xp.astype(count[dense], xp.float64)
+ xp.astype(count[dense - 1], xp.float64)
+ 1.0
)
def _mantel_stats_pearson_xp(x, y, permutations, seed, alternative, spearman=False):
"""Mantel pearson/spearman result in the ``xp`` namespace.
Backend-agnostic equivalent of ``_mantel_stats_pearson``/``_spearman`` for
full (non-condensed) array-API matrices. Matches the RNG usage of the NumPy
path (identity permutation first, then ``permutations`` calls to
``rng.permutation(n)``) so p-values are reproducible and identical.
Returns ``(orig_stat, p_value, n)``, matching ``mantel``'s return.
"""
rng = get_rng(seed)
xp, X, Y = ingest_array(x, y)
if X.shape != Y.shape:
raise ValueError("Distance matrices must have the same shape.")
if (X.ndim != 2) or (X.shape[0] != X.shape[1]):
raise ValueError("Distance matrix must be a square 2-D array.")
n = X.shape[0]
if n < 3:
raise ValueError(
"Distance matrices must have at least 3 matching IDs "
"between them (i.e., minimum 3x3 in size)."
)
iu0, iu1 = _upper_tri_xp(xp, n, device=_device(X))
x_flat = X[iu0, iu1]
y_flat = Y[iu0, iu1]
if spearman:
x_flat = _xp_rank_average(xp, x_flat)
y_flat = _xp_rank_average(xp, y_flat)
# rebuild the ranked symmetric matrix so permuted pairs can be gathered
# (one-time host assembly; on-device gather is a possible follow-up).
xr = _get_array(x_flat, to_numpy=True)
i0, j0 = _get_array(iu0, to_numpy=True), _get_array(iu1, to_numpy=True)
full = np.zeros((n, n), dtype=np.float64)
full[i0, j0] = xr
full[j0, i0] = xr
Xsrc = xp.asarray(full, device=_device(X))
else:
Xsrc = X
# constant input -> correlation undefined (matches scipy/skbio behavior)
if bool(xp.all(x_flat == x_flat[0])) or bool(xp.all(y_flat == y_flat[0])):
warn(ConstantInputWarning())
return np.nan, np.nan, n
xmean = xp.mean(x_flat)
normxm = xp.sqrt(xp.sum((x_flat - xmean) ** 2))
ymean = xp.mean(y_flat)
normym = xp.sqrt(xp.sum((y_flat - ymean) ** 2))
ym_norm = (y_flat - ymean) / normym
# near-constant input -> loss of precision in r (matches the NumPy path)
threshold = 1e-13
if (float(normxm) < threshold * abs(float(xmean))) or (
float(normym) < threshold * abs(float(ymean))
):
warn(NearConstantInputWarning())
def _corr(cond):
s = float(xp.sum(((cond - xmean) / normxm) * ym_norm))
return max(min(s, 1.0), -1.0)
orig_stat = _corr(x_flat)
comp_stat = orig_stat
permuted_stats = []
if not (permutations == 0 or np.isnan(orig_stat)):
# identity permutation gives comp_stat (matches perm_order[0] in the
# NumPy path); the RNG is then consumed exactly `permutations` times.
idx = xp.arange(n, device=_device(X))
comp_stat = _corr(Xsrc[idx[iu0], idx[iu1]])
permuted_stats = np.empty(permutations, dtype=np.float64)
for k in range(permutations):
pp = xp.asarray(rng.permutation(n), device=_device(X))
permuted_stats[k] = _corr(Xsrc[pp[iu0], pp[iu1]])
# p-value from the permutation distribution (matches the NumPy path exactly)
if permutations == 0 or np.isnan(orig_stat):
p_value = np.nan
else:
if alternative == "two-sided":
count_better = (np.absolute(permuted_stats) >= np.absolute(comp_stat)).sum()
elif alternative == "greater":
count_better = (permuted_stats >= comp_stat).sum()
else:
count_better = (permuted_stats <= comp_stat).sum()
p_value = (count_better + 1) / (permutations + 1)
return orig_stat, p_value, n
def _device(arr):
"""Best-effort device of an array-API array (None if not exposed)."""
return getattr(arr, "device", None)
def _mantel_stats_pearson_flat(x, y_flat, permutations, seed=None, engine=None):
"""Compute original and permuted stats using pearsonr.
Parameters
----------
x : DistanceMatrix
Input distance matrix.
y_flat: 1D array
Compact representation of a distance matrix.
permutations : int
Number of times to randomly permute `x` when assessing statistical
significance. Must be greater than or equal to zero. If zero,
statistical significance calculations will be skipped and
permuted_stats will be an empty array.
seed : int, Generator or RandomState, optional
A user-provided random seed or random generator instance.
Returns
-------
orig_stat : 1D array_like
Correlation coefficient of the test.
comp_stat : 1D array_like
Correlation coefficient to compare against permuted_stats, usually
the same as orig_stat, but on certain architectures it will differ.
This should be used for any p-value calculation as it will match the
values for any "self-permutations" in the permuted_stats.
permuted_stats : 1D array_like
Permuted correlation coefficients of the test.
"""
rng = get_rng(seed)
x_flat = x.condensed_form()
# If an input is constant, the correlation coefficient is not defined.
if (x_flat == x_flat[0]).all() or (y_flat == y_flat[0]).all():
warn(ConstantInputWarning())
return np.nan, np.nan, []
# inline pearsonr, condensed from scipy.stats.pearsonr
xmean = x_flat.mean()
xm = x_flat - xmean
normxm = np.linalg.norm(xm)
xm_normalized = xm / normxm
del xm
del x_flat
ymean = y_flat.mean()
ym = y_flat - ymean
normym = np.linalg.norm(ym)
ym_normalized = ym / normym
del ym
threshold = 1e-13
if (normxm < threshold * abs(xmean)) or (normym < threshold * abs(ymean)):
# If all the values in x (likewise y) are very close to the mean,
# the loss of precision that occurs in the subtraction xm = x - xmean
# might result in large errors in r.
warn(NearConstantInputWarning())
orig_stat = np.dot(xm_normalized, ym_normalized)
# Presumably, if abs(orig_stat) > 1, then it is only some small artifact of
# floating point arithmetic.
orig_stat = max(min(orig_stat, 1.0), -1.0)
mat_n = x.shape[0]
# note: xmean and normxm do not change with permutations
permuted_stats = []
comp_stat = orig_stat
if not (permutations == 0 or np.isnan(orig_stat)):
# inline DistanceMatrix.permute, grouping them together
x_data = x._data
if not x_data.flags.c_contiguous:
x_data = np.asarray(x_data, order="C")
# compute all pearsonr permutations at once
# create first the list of permutations
perm_order = np.empty((permutations + 1, mat_n), dtype=np.intp)
# first row/statistic will be comp_stat
perm_order[0, :] = np.arange(mat_n)
for row in range(1, permutations + 1):
perm_order[row, :] = rng.permutation(mat_n)
permuted_stats = np.empty(permutations + 1, dtype=x_data.dtype)
if x._flags["CONDENSED"]:
if engine == "numba":
_mantel_perm_pearsonr_condensed_nb(
x_data, perm_order, xmean, normxm, ym_normalized, permuted_stats
)
else:
mantel_perm_pearsonr_condensed_cy(
x_data, perm_order, xmean, normxm, ym_normalized, permuted_stats
)
else:
if engine == "numba":
_mantel_perm_pearsonr_nb(
x_data, perm_order, xmean, normxm, ym_normalized, permuted_stats
)
else:
mantel_perm_pearsonr_cy(
x_data, perm_order, xmean, normxm, ym_normalized, permuted_stats
)
comp_stat = permuted_stats[0]
permuted_stats = permuted_stats[1:]
return orig_stat, comp_stat, permuted_stats
def _mantel_stats_pearson(x, y, permutations, seed=None, engine=None):
"""Compute original and permuted stats using pearsonr.
Parameters
----------
x, y : DistanceMatrix
Input distance matrices to compare.
permutations : int
Number of times to randomly permute `x` when assessing statistical
significance. Must be greater than or equal to zero. If zero,
statistical significance calculations will be skipped and
permuted_stats will be an empty array.
seed : int, Generator or RandomState, optional
A user-provided random seed or random generator instance.
Returns
-------
orig_stat : 1D array_like
Correlation coefficient of the test.
comp_stat : 1D array_like
Correlation coefficient to compare against permuted_stats, usually
the same as orig_stat, but on certain architectures it will differ.
This should be used for any p-value calculation as it will match the
values for any "self-permutations" in the permuted_stats.
permuted_stats : 1D array_like
Permuted correlation coefficients of the test.
"""
y_flat = y.condensed_form()
return _mantel_stats_pearson_flat(x, y_flat, permutations, seed, engine)
def _mantel_stats_spearman(x, y, permutations, seed=None, engine=None):
"""Compute original and permuted stats using spearmanr.
Parameters
----------
x, y : DistanceMatrix
Input distance matrices to compare.
permutations : int
Number of times to randomly permute `x` when assessing statistical
significance. Must be greater than or equal to zero. If zero,
statistical significance calculations will be skipped and
permuted_stats will be an empty array.
seed : int, Generator or RandomState, optional
A user-provided random seed or random generator instance.
Returns
-------
orig_stat : 1D array_like
Correlation coefficient of the test.
comp_stat : 1D array_like
Correlation coefficient to compare against permuted_stats, usually
the same as orig_stat, but on certain architectures it will differ.
This should be used for any p-value calculation as it will match the
values for any "self-permutations" in the permuted_stats.
permuted_stats : 1D array_like
Permuted correlation coefficients of the test.
"""
x_flat = x.condensed_form()
y_flat = y.condensed_form()
# If an input is constant, the correlation coefficient is not defined.
if (x_flat == x_flat[0]).all() or (y_flat == y_flat[0]).all():
warn(ConstantInputWarning())
return np.nan, np.nan, []
y_rank = scipy.stats.rankdata(y_flat)
del y_flat
x_rank = scipy.stats.rankdata(x_flat)
del x_flat
x_rank_matrix = DistanceMatrix(x_rank, x.ids)
del x_rank
# for our purposes, spearman is just pearson on rankdata
return _mantel_stats_pearson_flat(x_rank_matrix, y_rank, permutations, seed, engine)
def pwmantel(
dms: Sequence[DistanceMatrix | ArrayLike],
labels: Sequence[str | int] | None = None,
method: str = "pearson",
permutations: int = 999,
alternative: str = "two-sided",
strict: bool = True,
lookup: dict[str, str] | None = None,
seed: SeedLike | None = None,
) -> pd.DataFrame:
"""Run Mantel tests for every pair of given distance matrices.
Runs a Mantel test for each pair of distance matrices and collates the
results in a ``DataFrame``. Distance matrices do not need to be in the same
ID order if they are ``DistanceMatrix`` instances. Distance matrices will
be re-ordered prior to running each pairwise test, and if ``strict=False``,
IDs that don't match between a pair of distance matrices will be dropped
prior to running the test (otherwise a ``ValueError`` will be raised if
there are nonmatching IDs between any pair of distance matrices).
Parameters
----------
dms : iterable of DistanceMatrix objects, array_like objects, or filepaths
to distance matrices. If they are ``array_like``, no reordering or
matching of IDs will be performed.
labels : iterable of str or int, optional
Labels for each distance matrix in `dms`. These are used in the results
``DataFrame`` to identify the pair of distance matrices used in a
pairwise Mantel test. If ``None``, defaults to monotonically-increasing
integers starting at zero.
method : {'pearson', 'spearman'}
Correlation method. See ``mantel`` function for more details.
permutations : int, optional
Number of permutations. See ``mantel`` function for more details.
alternative : {'two-sided', 'greater', 'less'}
Alternative hypothesis. See ``mantel`` function for more details.
strict : bool, optional
Handling of nonmatching IDs. See ``mantel`` function for more details.
lookup : dict, optional
Map existing IDs to new IDs. See ``mantel`` function for more details.
seed : int, Generator or RandomState, optional
A user-provided random seed or random generator instance. See
:func:`details <skbio.util.get_rng>`.
.. versionadded:: 0.6.3
Returns
-------
pandas.DataFrame
``DataFrame`` containing the results of each pairwise test (one per
row). Includes the number of objects considered in each test as column
``n`` (after applying `lookup` and filtering nonmatching IDs if
``strict=False``). Column ``p-value`` will display p-values as ``NaN``
if p-values could not be computed (they are stored as ``np.nan`` within
the ``DataFrame``; see ``mantel`` for more details).
See Also
--------
mantel
DistanceMatrix.read
Notes
-----
This function uses parallel computation for improved performance.
See the :ref:`parallelization guide <parallelization>` for information on
controlling the number of threads used.
Passing a list of filepaths can be useful as it allows for a smaller amount
of memory consumption as it only loads two matrices at a time as opposed to
loading all distance matrices into memory.
Examples
--------
Import the functionality we'll use in the following examples:
>>> from skbio import DistanceMatrix
>>> from skbio.stats.distance import pwmantel
Define three 3x3 distance matrices:
>>> x = DistanceMatrix([[0, 1, 2],
... [1, 0, 3],
... [2, 3, 0]])
>>> y = DistanceMatrix([[0, 2, 7],
... [2, 0, 6],
... [7, 6, 0]])
>>> z = DistanceMatrix([[0, 5, 6],
... [5, 0, 1],
... [6, 1, 0]])
Run Mantel tests for each pair of distance matrices (there are 3 possible
pairs):
>>> pwmantel((x, y, z), labels=('x', 'y', 'z'),
... permutations=0) # doctest: +NORMALIZE_WHITESPACE
statistic p-value n method permutations alternative
dm1 dm2
x y 0.755929 NaN 3 pearson 0 two-sided
z -0.755929 NaN 3 pearson 0 two-sided
y z -0.142857 NaN 3 pearson 0 two-sided
Note that we passed ``permutations=0`` to suppress significance tests; the
p-values in the output are labelled ``NaN``.
"""
rng = get_rng(seed)
num_dms = len(dms)
if num_dms < 2:
raise ValueError("Must provide at least two distance matrices.")
if labels is None:
labels = range(num_dms)