Skip to content

[NET-225] Result P-values #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pollaro merged 3 commits into from
Nov 4, 2024
Merged

[NET-225] Result P-values #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d1cc8aa
rename p-values to be more consistent.
pollaro Oct 30, 2024
70d3cbd
changed names to better names, created static method
pollaro Oct 31, 2024
807e1c9
lots of changes.
pollaro Nov 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions +nla/+net/+result/+plot/NetworkTestPlot.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ function drawFigure(obj, plot_type)
end

function [width, height] = drawTriMatrixPlot(obj)

import nla.net.result.NetworkTestResult

if ~isequal(obj.matrix_plot, false)
obj.matrix_plot.plot_title.String = {};
obj.parameters.updated_test_options.prob_max = obj.current_settings.p_threshold;
Expand All @@ -145,8 +146,10 @@ function drawFigure(obj, plot_type)
mcc = obj.current_settings.mcc;
end

probability = NetworkTestResult().getPValueNames(obj.test_method, obj.network_test_result.test_name);

probability_parameters = obj.parameters.plotProbabilityParameters(obj.edge_test_options, obj.edge_test_result,...
obj.test_method, "statistic_p_value", sprintf(obj.title), mcc, obj.createSignificanceFilter(),...
obj.test_method, probability, sprintf(obj.title), mcc, obj.createSignificanceFilter(),...
obj.current_settings.ranking);

if obj.current_settings.upper_limit ~= 0.3 && obj.current_settings.lower_limit ~= -0.3
Expand All @@ -168,8 +171,11 @@ function drawChord(obj, ~, ~, plot_type)

obj.getPlotTitle();

probability = NetworkTestResult().getPValueNames(obj.test_method, obj.network_test_result.test_name);
p_value = strcat("uncorrected_", probability);

probability_parameters = obj.parameters.plotProbabilityParameters(obj.edge_test_options, obj.edge_test_result,...
obj.test_method, "p_value", sprintf(obj.title), obj.current_settings.mcc, obj.createSignificanceFilter(),...
obj.test_method, p_value, sprintf(obj.title), obj.current_settings.mcc, obj.createSignificanceFilter(),...
obj.current_settings.ranking);

chord_plotter = nla.net.result.chord.ChordPlotter(obj.network_atlas, obj.edge_test_result);
Expand Down
31 changes: 12 additions & 19 deletions +nla/+net/+result/NetworkResultPlotParameter.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ function brainFigureButtonCallback(network1, network2)
end

function statistic = getStatsFromMethodAndName(obj, method, plot_statistic, ranking_method)
import nla.RankingMethod nla.NetworkLevelMethod
import nla.RankingMethod nla.NetworkLevelMethod nla.net.result.NetworkTestResult

switch method
case NetworkLevelMethod.NO_PERMUTATIONS
Expand All @@ -178,26 +178,19 @@ function brainFigureButtonCallback(network1, network2)
case NetworkLevelMethod.WITHIN_NETWORK_PAIR
test_method = "within_network_pair";
end

% combining the method and stat name to get the data. With a fail safe for forgetting 'single_sample'
if isequal(test_method, "within_network_pair")...
&& ~startsWith(plot_statistic, "single_sample")...
&& ~any(ismember(obj.noncorrelation_input_tests, obj.network_test_results.test_name))
plot_statistic = strcat("single_sample_", plot_statistic);

switch ranking_method
case RankingMethod.WINKLER
ranking = "winkler_";
case RankingMethod.WESTFALL_YOUNG
ranking = "westfall_young_";
otherwise
ranking = "uncorrected_";
end

statistic = obj.network_test_results.(test_method).(plot_statistic);
if nargin > 2 && ~isequal(ranking_method, RankingMethod.EGGEBRECHT)
plot_statistic = "p_value";
if isequal(ranking_method, RankingMethod.WINKLER)
ranking_method = "winkler";
elseif isequal(ranking_method, RankingMethod.WESTFALL_YOUNG)
ranking_method = "westfall_young";
else
ranking_method = "statistic";
end
statistic = obj.network_test_results.(test_method).(strcat((ranking_method), "_", (plot_statistic)));
end
probability = NetworkTestResult().getPValueNames(method, obj.network_test_results.test_name);

statistic = obj.network_test_results.(test_method).(strcat(ranking, probability));
end
end

Expand Down
79 changes: 56 additions & 23 deletions +nla/+net/+result/NetworkTestResult.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
end

properties (Constant)
% TODO: replace wtih enums
test_methods = ["no_permutations", "full_connectome", "within_network_pair"]
noncorrelation_input_tests = ["chi_squared", "hypergeometric"] % These are tests that do not use correlation coefficients as inputs
end
Expand Down Expand Up @@ -162,17 +163,17 @@ function runDiagnosticPlots(obj, edge_test_options, updated_test_options, edge_t

if isfield(flags, "show_nonpermuted") && flags.show_nonpermuted
title = "Non-Permuted";
p_values = obj.no_permutations.p_value;
p_values = obj.no_permutations.uncorrected_two_sample_p_value;
fdr_method = network_test_options.fdr_correction;
end
if isfield(flags, "show_full_conn") && flags.show_full_conn
title = "Full Connectome";
p_values = obj.full_connectome.p_value;
p_values = obj.full_connectome.uncorrected_two_sample_p_value;
fdr_method = nla.net.mcc.None;
end
if isfield(flags, "show_within_net_pair") && flags.show_within_net_pair
title = "Within Network Pair";
p_values = obj.within_network_pair.single_sample_p_value;
p_values = obj.within_network_pair.uncorrected_single_sample_p_value;
fdr_method = network_test_options.fdr_correction;
end
[significance, name] = obj.singleSigMat(network_atlas, network_test_options, p_values, fdr_method, title);
Expand All @@ -189,9 +190,9 @@ function runDiagnosticPlots(obj, edge_test_options, updated_test_options, edge_t
% getters for dependent properties
function value = get.permutation_count(obj)
% Convenience method to carry permutation from data through here
if isfield(obj.permutation_results, "p_value_permutations") &&...
~isequal(obj.permutation_results.p_value_permutations, false)
value = size(obj.permutation_results.p_value_permutations.v, 2);
if isfield(obj.permutation_results, "two_sample_p_value_permutations") &&...
~isequal(obj.permutation_results.two_sample_p_value_permutations, false)
value = size(obj.permutation_results.two_sample_p_value_permutations.v, 2);
elseif isfield(obj.permutation_results, "single_sample_p_value_permutations") &&...
~isequal(obj.permutation_results.single_sample_p_value_permutations, false)
value = size(obj.permutation_results.single_sample_p_value_permutations.v, 2);
Expand Down Expand Up @@ -225,15 +226,8 @@ function createResultsStorage(obj, test_options, number_of_networks, test_specif

function createTestSpecificResultsStorage(obj, number_of_networks, test_specific_statistics)
%CREATETESTSPECIFICRESULTSSTORAGE Create the substructures for the specific statistical tests

import nla.TriMatrix nla.TriMatrixDiag

obj.permutation_results.p_value_permutations = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
if ~any(strcmp(obj.test_name, obj.noncorrelation_input_tests))
obj.permutation_results.single_sample_p_value_permutations = TriMatrix(number_of_networks,...
TriMatrixDiag.KEEP_DIAGONAL);
end

for statistic_index = 1:numel(test_specific_statistics)
test_statistic = test_specific_statistics(statistic_index);
obj.no_permutations.(test_statistic) = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
Expand All @@ -244,19 +238,46 @@ function createTestSpecificResultsStorage(obj, number_of_networks, test_specific

function createPValueTriMatrices(obj, number_of_networks, test_method)
%CREATEPVALUETRIMATRICES Creates the p-value substructure for the test method

import nla.TriMatrix nla.TriMatrixDiag

obj.(test_method).p_value = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL); % p-value rank
obj.(test_method).statistic_p_value = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL); % p-value by statistic rank
obj.(test_method).winkler_p_value = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL); % p-value by winkler's method
obj.(test_method).westfall_young_p_value = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL); % p-value by westfall-young
if ~isequal(test_method, "full_connectome") && ~any(strcmp(obj.test_name, obj.noncorrelation_input_tests))
obj.(test_method).single_sample_p_value = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
obj.(test_method).statistic_single_sample_p_value = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
obj.(test_method).winkler_single_sample_p_value = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
obj.(test_method).westfall_young_single_sample_p_value = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
non_correlation_test = any(strcmp(obj.test_name, obj.noncorrelation_input_tests));
uncorrected_names = ["uncorrected_", "legacy_"];
corrected_names = ["winkler_", "westfall_young_"];

switch test_method
case "no_permutations"
for uncorrected_name = uncorrected_names
p_value = "two_sample_p_value"
obj.(test_method).(strcat(uncorrected_name, p_value)) = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
if isequal(non_correlation_test, false)
p_value = "single_sample_p_value";
obj.(test_method).(strcat(uncorrected_name, p_value)) = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
end
end
case "full_connectome"
p_value = "two_sample_p_value";
for name = [corrected_names uncorrected_names]
obj.(test_method).(strcat(name, p_value)) = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
end
case "within_network_pair"
% This is so hacky, but Matlab doesn't play well with logical order-of-operations
if isequal(non_correlation_test, true)
p_value = "two_sample_p_value";
else
p_value = "single_sample_p_value";
end
for name = [corrected_names uncorrected_names]
obj.(test_method).(strcat(name, p_value)) = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
end
end

% We need the permutation fields for all results. We need the two-sample ones for everything
obj.permutation_results.two_sample_p_value_permutations = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
if isequal(non_correlation_test, false)
obj.permutation_results.single_sample_p_value_permutations = TriMatrix(number_of_networks,...
TriMatrixDiag.KEEP_DIAGONAL);
end

%Cohen's D results
obj.(test_method).d = TriMatrix(number_of_networks, TriMatrixDiag.KEEP_DIAGONAL);
end
Expand Down Expand Up @@ -292,5 +313,17 @@ function createPValueTriMatrices(obj, number_of_networks, test_method)
Number('d_max', "Cohen's D threshold >", 0, 0.5, 1),...
};
end

function probability = getPValueNames(test_method, test_name)
noncorrelation_input_tests = ["chi_squared", "hypergeometric"];
non_correlation_test = any(strcmp(test_name, noncorrelation_input_tests));

probability = "two_sample_p_value";
if isequal(non_correlation_test, false)
if isequal(test_method, "no_permutations") || isequal(test_method, "within_network_pair")
probability = "single_sample_p_value";
end
end
end
end
end
4 changes: 2 additions & 2 deletions +nla/+net/+test/ChiSquaredTest.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,9 @@
% Matlab function for chi-squared cdf to get p-value. "Upper" calculates the upper tail instead of
% using 1 - lower tail
if permutations
result.permutation_results.p_value_permutations.v = chi2cdf(result.permutation_results.chi2_statistic_permutations.v, 1, "upper");
result.permutation_results.two_sample_p_value_permutations.v = chi2cdf(result.permutation_results.chi2_statistic_permutations.v, 1, "upper");
else
result.no_permutations.p_value.v = chi2cdf(result.no_permutations.chi2_statistic.v, 1, "upper");
result.no_permutations.uncorrected_two_sample_p_value.v = chi2cdf(result.no_permutations.chi2_statistic.v, 1, "upper");
end
end
end
Expand Down
8 changes: 4 additions & 4 deletions +nla/+net/+test/HyperGeometricTest.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@
% Store results in the 'no_permutations' structure if this is the no-permutation test
permutation_results = "no_permutations";
greater_than_expected = "greater_than_expected";
p_value = "p_value";
p_value = "uncorrected_two_sample_p_value";
if isequal(permutations, true)
% Otherwise, add it on to the back of the 'permutation_results' structure
permutation_results = "permutation_results";
greater_than_expected = strcat(greater_than_expected, "_permutations");
p_value = strcat(p_value, "_permutations");
p_value = "two_sample_p_value_permutations";
end

% Container to hold results
Expand Down Expand Up @@ -57,9 +57,9 @@
% This just results in a p-value of 1. Which means no difference between chance and null
% hypothesis.
if permutations
result.permutation_results.p_value_permutations.v(~result.permutation_results.greater_than_expected_permutations.v) = 1;
result.permutation_results.two_sample_p_value_permutations.v(~result.permutation_results.greater_than_expected_permutations.v) = 1;
else
result.no_permutations.p_value.v(~result.no_permutations.greater_than_expected.v) = 1;
result.no_permutations.uncorrected_two_sample_p_value.v(~result.no_permutations.greater_than_expected.v) = 1;
end
end
end
Expand Down
17 changes: 8 additions & 9 deletions +nla/+net/+test/KolmogorovSmirnovTest.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,15 @@

% Store results in the 'no_permutations' structure if this is the no-permutation test
permutation_results = "no_permutations";
p_value = "p_value";
ks_statistic = "ks_statistic";
single_sample_p_value = "single_sample_p_value";
single_sample_p_value = "uncorrected_single_sample_p_value";
single_sample_ks_statistic = "single_sample_ks_statistic";
if isequal(permutations, true)
% Otherwise, add it on to the back of the 'permutation_results' structure
permutation_results = "permutation_results";
p_value = strcat(p_value, "_permutations");
p_value = "two_sample_p_value_permutations";
ks_statistic = strcat(ks_statistic, "_permutations");
single_sample_p_value = strcat(single_sample_p_value, "_permutations");
single_sample_p_value = "single_sample_p_value_permutations";
single_sample_ks_statistic = strcat(single_sample_ks_statistic, "_permutations");
end

Expand All @@ -44,11 +43,11 @@
for network2 = 1:network
network_rho = edge_test_results.coeff.get(network_atlas.nets(network).indexes,...
network_atlas.nets(network2).indexes);

[~, p, ks] = kstest2(network_rho, edge_test_results.coeff.v);
result.(permutation_results).(p_value).set(network, network2, p);
result.(permutation_results).(ks_statistic).set(network, network2, ks);

if ~isequal(permutation_results, "no_permutations")
[~, p, ks] = kstest2(network_rho, edge_test_results.coeff.v);
result.(permutation_results).(p_value).set(network, network2, p);
result.(permutation_results).(ks_statistic).set(network, network2, ks);
end
[~, single_sample_p, single_sample_ks] = kstest(network_rho);
result.(permutation_results).(single_sample_p_value).set(network, network2, single_sample_p);
result.(permutation_results).(single_sample_ks_statistic).set(network, network2, single_sample_ks);
Expand Down
15 changes: 8 additions & 7 deletions +nla/+net/+test/StudentTTest.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,16 @@

% Store results in the 'no_permutations' structure if this is the no-permutation test
permutation_results = "no_permutations";
p_value = "p_value";
p_value = "uncorrected_two_sample_p_value";
t_statistic = "t_statistic";
single_sample_p_value = "single_sample_p_value";
single_sample_p_value = "uncorrected_single_sample_p_value";
single_sample_t_statistic = "single_sample_t_statistic";
if isequal(permutations, true)
% Otherwise, add it on to the back of the 'permutation_results' structure
permutation_results = "permutation_results";
p_value = strcat(p_value, "_permutations");
p_value = "two_sample_p_value_permutations";
t_statistic = strcat(t_statistic, "_permutations");
single_sample_p_value = strcat(single_sample_p_value, "_permutations");
single_sample_p_value = "single_sample_p_value_permutations";
single_sample_t_statistic = strcat(single_sample_t_statistic, "_permutations");
end

Expand All @@ -47,9 +47,10 @@
[~, p, ~, stats] = ttest2(network_rho, edge_test_results.coeff.v);

[~, single_sample_p, ~, single_sample_stats] = ttest(network_rho);

result.(permutation_results).(p_value).set(network, network2, p);
result.(permutation_results).(t_statistic).set(network, network2, stats.tstat);
if ~isequal(permutation_results, "no_permutations")
result.(permutation_results).(p_value).set(network, network2, p);
result.(permutation_results).(t_statistic).set(network, network2, stats.tstat);
end
result.(permutation_results).(single_sample_p_value).set(network, network2, single_sample_p);
result.(permutation_results).(single_sample_t_statistic).set(network, network2, single_sample_stats.tstat);
end
Expand Down
Loading