diff --git a/+nla/+gfx/+chord/ChordPlot.m b/+nla/+gfx/+chord/ChordPlot.m index 8ec6e12a..ca1486d5 100644 --- a/+nla/+gfx/+chord/ChordPlot.m +++ b/+nla/+gfx/+chord/ChordPlot.m @@ -295,7 +295,7 @@ function connectNetworks(obj) % boolean array used to determine if networks connected networks_connected = false(obj.number_of_networks, obj.number_of_networks + 1); - + if obj.chord_type == "nla.PlotType.CHORD" % These two arrays are the networks individucally numbered. Taking the same index of both % (in vector, network_array.v(idx)) gives the two networks we're testing @@ -312,7 +312,7 @@ function connectNetworks(obj) ROI_center_radians = []; ROI_centers = []; end - + for network = 1:obj.number_of_networks if obj.chord_type == "nla.PlotType.CHORD" % These fill in the four networks above. diff --git a/+nla/+net/+result/+chord/ChordPlotter.m b/+nla/+net/+result/+chord/ChordPlotter.m index c2a056f8..c1017361 100644 --- a/+nla/+net/+result/+chord/ChordPlotter.m +++ b/+nla/+net/+result/+chord/ChordPlotter.m @@ -68,7 +68,6 @@ function generateChordFigure(obj, parameters, chord_type) % thresholding below the "insignificance" value statistic_matrix = copy(parameters.statistic_plot_matrix); statistic_matrix.v(~parameters.significance_plot.v) = insignificance; - chord_plotter = nla.gfx.chord.ChordPlot(obj.network_atlas, figure_axis, 500, statistic_matrix,... 'color_map', parameters.color_map, 'direction', parameters.significance_type, 'upper_limit',... coefficient_bounds(2), 'lower_limit', coefficient_bounds(1), 'chord_type', chord_type); @@ -78,8 +77,6 @@ function generateChordFigure(obj, parameters, chord_type) % Plot edge chord obj.generateEdgeChordFigure(plot_figure, parameters, chord_type) end - - end end @@ -137,7 +134,6 @@ function generateEdgeChordFigure(obj, plot_figure, parameters, chord_type) clipped_values.v = obj.edge_test_result.prob.v; significance_type = "nla.gfx.SigType.DECREASING"; - coefficient_min = 0; coefficient_max = obj.edge_test_result.prob_max; diff --git a/+nla/+net/+result/NetworkTestResult.m b/+nla/+net/+result/NetworkTestResult.m index f060c89a..224439a4 100644 --- a/+nla/+net/+result/NetworkTestResult.m +++ b/+nla/+net/+result/NetworkTestResult.m @@ -1,27 +1,16 @@ classdef NetworkTestResult < matlab.mixin.Copyable - %NETWORKTESTRESULT Network Test Results - % This is the super class that all network test results will be in - % When a result is created the three repositories (within_network_pair, full_connectome, no_permutations) are set - % to false. This makes it easier to do an if/else check on them. - % The three private methods create the structures and trimatrices to keep the data. - % Notation: - % Test Methods: The method used for ranking the statistics (within net pair, full connectome, no permutation) - % Statistics: The statistical results from a specific network test (chi-squared, t-tests) - % - % Output object: - % test_name: The name of the network test run (chi_squared, hypergeometric, etc) - % test_options: The options passed in. Method, plotting methods (formerly input_struct) - % within_network_pair: Results from within_network_pair tests - % full_connectome: Results from full_connectome tests - % no_permutations: Results from the no permutation test - % permutation_results: Results from all the permutations of the network tests. These are used in the ranking to create - % the results for the test methods - % - % Within each of the three results structures will be properties containing the tri-matrices. Each test is different, - % but all contain: - % p_value: TriMatrix with p-values - % single_sample_p_value: TriMatrix with the single sample p-value (if available) + % Network level test results + % Class to store all relevant results for a network level test. Each test will create an instance of this to store results % + % :param test_name: The name of the network test run + % :param test_display_name: The name of the network test for display + % :param test_options: Options and inputs for tests to be run (also called input_struct) + % :param ranking_statistic: The statistic to be used in ranking to determine p-value + % :param within_network_pair: Results of the within network pair test. Single sample p-values (except :math:`\chi^2`\ and hypergeometric tests). "legacy_" results use the individual test p-values to rank and determine the final p-value. Multiple ranking strategies available + % :param full_connectome: Results of the full_connectome test. Same format as above. + % :param no_permutations: Results for the non-permuted test. Same format as above. + % :param permutation_results: Results of each permutation. Statistics and p-values. Note: The p-values are for each individual permutation test, not the overall p-value. + properties test_name = "" % Name of the network test run test_display_name = "" % Name of the network test for the front-end to display @@ -43,7 +32,6 @@ end properties (Constant) - % TODO: replace wtih enums test_methods = ["no_permutations", "full_connectome", "within_network_pair"] noncorrelation_input_tests = ["chi_squared", "hypergeometric"] % These are tests that do not use correlation coefficients as inputs end @@ -51,15 +39,6 @@ methods function obj = NetworkTestResult(test_options, number_of_networks, test_name, test_display_name,... test_specific_statistics, ranking_statistic) - %CONSTRUCTOR Used for creating results. - % - % Arguments: - % test_options [Struct]: Options for the test. Formerly 'input_struct' - % number_of_networks [Int]: The number of networks in the data being analyzed - % test_name [String]: The name of the network test being run - % test_specific_statistics [Array[String]]: Test statistics for a test. (Example: t_statistic for a t-Test) - % ranking_statistic [String]: Test statistic that will be used in ranking - import nla.TriMatrix nla.TriMatrixDiag if nargin == 0 return @@ -77,7 +56,15 @@ end end + % Used for plotting function output(obj, edge_test_options, updated_test_options, network_atlas, edge_test_result, flags) + % Outputs data to be plotted using nla.net.result.plot.NetworkTestPlot + % + % :param edge_test_options: The test_options used to instantiate the class. Contains the functional connectivity and network atlas among other options + % :param updated_test_options: The network test options. These can also include the options for plotting. + % :param network_atlas: The network atlas + % :param edge_test_result: Results of the edge level test. + % :param flags: More options that are used after the tests have run. One of them is which test method to plot. import nla.NetworkLevelMethod if isfield(flags, "show_nonpermuted") && flags.show_nonpermuted @@ -93,8 +80,12 @@ function output(obj, edge_test_options, updated_test_options, network_atlas, edg network_result_plot.drawFigure(nla.PlotType.FIGURE) end + % Use for merging multiple results together into one function merge(obj, other_objects) - %MERGE Merge two groups of results together. Not guaranteed to be ordered + % Used to merge multiple results together into one + % + % :param other_objects: The other result objects to merge into this result object + if ~iscell(other_objects) other_objects = {other_objects}; end @@ -111,7 +102,9 @@ function merge(obj, other_objects) end function concatenateResult(obj, other_object) - %CONCATENATERESULT Add a result to the back of a TriMatrix. Used to keep permutation data. + % Concatenate results together. This is used to preserve the individual permutation results. + % + % :param other_object: The object to append to the end of the current result statistics = fieldnames(obj.permutation_results); for statistic_index = 1:numel(statistics) @@ -124,74 +117,7 @@ function concatenateResult(obj, other_object) obj.last_index = obj.last_index + 1; end - - function histogram = createHistogram(obj, statistic) - if ~endsWith(statistic, "_permutations") - statistic = strcat(statistic, "_permutations"); - end - permutation_data = obj.permutation_results.(statistic); - histogram = zeros(nla.HistBin.SIZE, "uint32"); - - for permutation = 1:obj.permutation_count - histogram = histogram + uint32(histcounts(permutation_data.v(:, permutation), nla.HistBin.EDGES)'); - end - end - - function runDiagnosticPlots(obj, edge_test_options, updated_test_options, edge_test_result, network_atlas, flags) - import nla.NetworkLevelMethod - - diagnostics_plot = nla.gfx.plots.DiagnosticPlot(edge_test_options, updated_test_options,... - edge_test_result, network_atlas, obj); - - if isfield(flags, "show_nonpermuted") && flags.show_nonpermuted - test_method = "no_permutations"; - elseif isfield(flags, "show_full_conn") && flags.show_full_conn - test_method = "full_connectome"; - elseif isfield(flags, "show_within_net_pair") && flags.show_within_net_pair - test_method = "within_network_pair"; - end - - diagnostics_plot.displayPlots(test_method); - end - - % I'm assuming this is Get Significance Matrix. It's used for the convergence plots button, but the naming makes zero sense - % Any help on renaming would be great. - function [test_number, significance_count_matrix, names] = getSigMat(obj, network_test_options, network_atlas, flags) - - import nla.TriMatrix nla.TriMatrixDiag - - test_number = 0; - significance_count_matrix = TriMatrix(network_atlas.numNets(), 'double', TriMatrixDiag.KEEP_DIAGONAL); - names = []; - - if isfield(flags, "show_nonpermuted") && flags.show_nonpermuted - title = "Non-Permuted"; - p_values = obj.no_permutations.uncorrected_two_sample_p_value; - fdr_method = network_test_options.fdr_correction; - end - if isfield(flags, "show_full_conn") && flags.show_full_conn - title = "Full Connectome"; - p_values = obj.full_connectome.uncorrected_two_sample_p_value; - fdr_method = network_test_options.fdr_correction; - end - if isfield(flags, "show_within_net_pair") && flags.show_within_net_pair - title = "Within Network Pair"; - p_values = obj.within_network_pair.uncorrected_single_sample_p_value; - fdr_method = network_test_options.fdr_correction; - end - [significance, name] = obj.singleSigMat(network_atlas, network_test_options, p_values, fdr_method, title); - [test_number, significance_count_matrix, names] = obj.appendSignificanceMatrix(test_number, significance_count_matrix,... - names, significance, name); - end - - function table_new = generateSummaryTable(obj, table_old) - table_new = [table_old, table(... - obj.full_connectome.uncorrected_two_sample_p_value.v, 'VariableNames', [obj.test_display_name + "Full Connectome Two Sample p-value"]... - )]; - end - - %% - % getters for dependent properties + function value = get.permutation_count(obj) % Convenience method to carry permutation from data through here if isfield(obj.permutation_results, "two_sample_p_value_permutations") &&... @@ -205,8 +131,8 @@ function runDiagnosticPlots(obj, edge_test_options, updated_test_options, edge_t end end + % Convenience method to determine if inputs were correlation coefficients, or "significance" values function value = get.is_noncorrelation_input(obj) - % Convenience method to determine if inputs were correlation coefficients, or "significance" values value = any(strcmp(obj.noncorrelation_input_tests, obj.test_name)); end @@ -220,9 +146,12 @@ function runDiagnosticPlots(obj, edge_test_options, updated_test_options, edge_t methods (Access = private) function createResultsStorage(obj, test_options, number_of_networks, test_specific_statistics) - %CREATERESULTSSTORAGE Create the substructures for the methods chosen + % Creates the objects to hold results. Uses statistic names from test objects. + % + % :param test_options: The test options + % :param number_of_networks: The number of networks. Used to determine the size of the TriMatrix result. A property of the Network Atlas + % :param test_specific_statistics: The statistics used in each test. A property of each test. - % create the results containers. This replaces the false boolean with a struct of TriMatrices for test_method_index = 1:numel(obj.test_methods) if isequal(obj.(obj.test_methods(test_method_index)), false) &&... isequal(test_options.(obj.test_methods(test_method_index)), true) @@ -235,7 +164,6 @@ function createResultsStorage(obj, test_options, number_of_networks, test_specif end function createTestSpecificResultsStorage(obj, number_of_networks, test_specific_statistics) - %CREATETESTSPECIFICRESULTSSTORAGE Create the substructures for the specific statistical tests import nla.TriMatrix nla.TriMatrixDiag for statistic_index = 1:numel(test_specific_statistics) @@ -247,7 +175,6 @@ function createTestSpecificResultsStorage(obj, number_of_networks, test_specific end function createPValueTriMatrices(obj, number_of_networks, test_method) - %CREATEPVALUETRIMATRICES Creates the p-value substructure for the test method import nla.TriMatrix nla.TriMatrixDiag non_correlation_test = any(strcmp(obj.test_name, obj.noncorrelation_input_tests)); @@ -313,13 +240,24 @@ function createPValueTriMatrices(obj, number_of_networks, test_method) sig_count_mat.v = sig_count_mat.v + sig.v; names = [names name]; end + + function p_value = choosePlottingMethod(obj, test_options, plot_test_type) + p_value = "p_value"; + if test_options == nla.gfx.ProbPlotMethod.STATISTIC + p_value = strcat("statistic_", p_value); + end + if ~obj.is_noncorrelation_input && plot_test_type == "within_network_pair" + p_value = strcat("single_sample_", p_value); + end + end end methods (Static) function options = editableOptions() - % options that can be edited post-run (ie: are simple - % thresholds etc. for summary statistics, or generally can be - % modified without requiring re-permutation) + % Static method to return options that can be adjusted afterwards. + % + % :return: Options. Defaults to behavior_count, prob_max (The threshold for p-values), d_max (The threshold for Cohen's D values) + import nla.inputField.Integer nla.inputField.Number options = {... Integer('behavior_count', 'Test count:', 1, 1, Inf),... @@ -329,6 +267,12 @@ function createPValueTriMatrices(obj, number_of_networks, test_method) end function probability = getPValueNames(test_method, test_name) + % Static method to determine prefixes of p-values for test results + % + % :param test_method: No permutations, full connectome, or within network pair + % :param test_name: The name of the test run + % :return: The full name of the p-value. (example: "single_sammple_p_value") + import nla.NetworkLevelMethod noncorrelation_input_tests = ["chi_squared", "hypergeometric"]; non_correlation_test = any(strcmp(test_name, noncorrelation_input_tests)); diff --git a/+nla/+net/+test/StudentTTest.m b/+nla/+net/+test/StudentTTest.m index d55d2209..e56f93ff 100644 --- a/+nla/+net/+test/StudentTTest.m +++ b/+nla/+net/+test/StudentTTest.m @@ -45,7 +45,6 @@ network_rho = edge_test_results.coeff.get(network_atlas.nets(network).indexes,... network_atlas.nets(network2).indexes); [~, p, ~, stats] = ttest2(network_rho, edge_test_results.coeff.v); - [~, single_sample_p, ~, single_sample_stats] = ttest(network_rho); result.(permutation_results).(p_value).set(network, network2, p); result.(permutation_results).(t_statistic).set(network, network2, stats.tstat); @@ -53,8 +52,7 @@ result.(permutation_results).(single_sample_p_value).set(network, network2, single_sample_p); result.(permutation_results).(single_sample_t_statistic).set(network, network2, single_sample_stats.tstat); end - end - + end end end diff --git a/+nla/+net/ResultRank.m b/+nla/+net/ResultRank.m index 8df963e7..1a046a7f 100644 --- a/+nla/+net/ResultRank.m +++ b/+nla/+net/ResultRank.m @@ -84,9 +84,9 @@ end end + function ranking = winklerMethodRank(obj, test_method, permutation_results, no_permutation_results, ranking_statistic,... probability, ranking) - winkler_probability = strcat("winkler_", probability); max_statistic_array = max(abs(permutation_results.(strcat(ranking_statistic, "_permutations")).v)); for index = 1:numel(no_permutation_results.(strcat("uncorrected_", probability)).v) @@ -100,7 +100,6 @@ ); end end - ranking.(test_method).(winkler_probability).v = ranking.(test_method).(winkler_probability).v ./ obj.permutations; end @@ -159,7 +158,7 @@ end function value = get.number_of_networks(obj) - value = obj.permuted_network_results.no_permutations.p_value.size; + value = obj.permuted_network_results.no_permutations.p_value.size; end %% end diff --git a/.github/workflows/run_matlab_tests.yml b/.github/workflows/run_matlab_tests.yml index 67d3fad7..a32217d9 100644 --- a/.github/workflows/run_matlab_tests.yml +++ b/.github/workflows/run_matlab_tests.yml @@ -27,4 +27,4 @@ jobs: uses: matlab-actions/run-command@v2 with: command: addpath(genpath(pwd)); results = runTests(); assertSuccess(results) - use-parallel: true + use-parallel: true \ No newline at end of file diff --git a/README.rst b/README.rst deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/source/_static/behavior_table.png b/docs/source/_static/behavior_table.png new file mode 100644 index 00000000..92522cb7 Binary files /dev/null and b/docs/source/_static/behavior_table.png differ diff --git a/docs/source/_static/main_gui.png b/docs/source/_static/main_gui.png new file mode 100644 index 00000000..fc9ae5b3 Binary files /dev/null and b/docs/source/_static/main_gui.png differ diff --git a/docs/source/_static/main_gui2.png b/docs/source/_static/main_gui2.png new file mode 100644 index 00000000..13d00078 Binary files /dev/null and b/docs/source/_static/main_gui2.png differ diff --git a/docs/source/_static/result_gui.png b/docs/source/_static/result_gui.png new file mode 100644 index 00000000..2a59deb8 Binary files /dev/null and b/docs/source/_static/result_gui.png differ diff --git a/docs/source/_static/test_results_gui.png b/docs/source/_static/test_results_gui.png new file mode 100644 index 00000000..a3637071 Binary files /dev/null and b/docs/source/_static/test_results_gui.png differ diff --git a/docs/source/_static/trimatrix_plot.png b/docs/source/_static/trimatrix_plot.png new file mode 100644 index 00000000..2c0456d1 Binary files /dev/null and b/docs/source/_static/trimatrix_plot.png differ diff --git a/docs/source/behavior_table.rst b/docs/source/behavior_table.rst new file mode 100644 index 00000000..50a459ce --- /dev/null +++ b/docs/source/behavior_table.rst @@ -0,0 +1,41 @@ +Behavior Table +============================= + +.. figure:: _static/behavior_table.png + + Behavior Table with example values + + +Behavior File +------------------------------------ + +The behavior file is a table of statistics, scores, or any other factor that should be used in the +tests. The file can be in any tabular format that is compatible with MATLAB (i.e. csv, mat, tsv). +The statistic that will be used in the tests is referred to as the "Behavior". + +Setting the Behavior +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Only one field can be set as the Behavior to test against. This is done by highlighting (clicking on) +the desired column and pressing the green button (marked "Set Behavior"). When clicked away from or another +column is clicked, the column should stay green. *Note: If a column is selected as a behavior or covariate(s), +it must have a nuermical value. No blanks or NaNs* + +Setting Covariate(s) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Setting covariates is the same as setting the Behavior except that more than column may be selected. +Covariates can also be unselected. These actions are done by highlighting a column and pressing either of the two pink buttons +(marked: "Add Covariate" and "Remove Covariate"). These columns should be pink afterwards. + +Setting Permutation Groups +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Permutation groups can also be set. This allows permutation of the data and tests to be run in user-set blocks instead of +the entire dataset. + +To do this a column should be added to the behavior file. This column should be filled with positive numbers that are the same +per group. For example, one group would be 0, all of the subjects in this group should have a 0 in this column. The next group +would be 1, each subject in this group should have a 1. Continue this for the number of groups desired. + +This column can then be selected by pushing the blue buttons (marked: "Add Permutation Group Level" and "Remove Permutation Group Level"). \ No newline at end of file diff --git a/docs/source/edge_level_tests.rst b/docs/source/edge_level_tests.rst index 2e2bf0f7..922891a9 100644 --- a/docs/source/edge_level_tests.rst +++ b/docs/source/edge_level_tests.rst @@ -12,9 +12,8 @@ Common Inputs -------------------------- :P: Edge-level p-value threshold -:Network Atlas: :doc:`/network_atlases` -:Functional Connectivity: Initial coorelation matrix if size N\ :sub:`ROIs`\ x N\ :sub:`ROIs`\ x N\ :sub:`scans`\. - r-values or Fisher z-transformed r-values. +:Network Atlas: :doc:`Network Atlas ` +:Functional Connectivity: Initial coorelation matrix (r-values or Fisher z-transformed r-values) of size N\ :sub:`ROIs`\ x N\ :sub:`ROIs`\ x N\ :sub:`scans`\ :Behavior: MATLAB table (``.mat``) or tab seperated text file (``.txt``) ============== =================== ================ @@ -27,7 +26,7 @@ Common Inputs ============== =================== ================ Each column header is a name of a variable. - Each column contains N\ :sub:`scans`\ entries. + Each column contains N\ :sub:`scans`\ entries. After loading this file, the table should display in the GUI. The user may mark one column as 'Behavior' for the score of interest. Other columns may be marked as 'Covariates' which are partialed prior to running statistics. @@ -55,82 +54,91 @@ Provided Tests * Faster implementation that stardard MATLAB code providing identical :math:`\tau`\ and p-values. * Run-time difference from *O*\ (*n*\ :sup:`2`) to *O*\ (*n* log *n*) * This is done with a red-black tree. -* **Welch's t-test`` +* **Welch's t-test** * Implements an optomized Welch's t-test comparing the functional connectivity of two groups. * Extra imports compared to other edge level tests + :Group name(s): Names associated with each group. (For example, 'Male' and 'Female') :Group val(s): Behavioral value associated with each group. If 'Female' is donated as '0', and 'Male' as '1', set the vals to the numerical values. +.. _precalculated: + * **Pre-calculated data loader** * Allows loading of observed and permuted edge-level data the user has pre-calculated outside the NLA. * Four ``.mat`` files needed as inputs * p-values should be thresholded + :Observed p: ``.mat`` file containing N\ :sub:`ROI_pairs`\ x 1 matrix of logical values, the observed, thresholded edge-level p-values. N\ :sub:`ROI_pairs`\ are the lower triangle values of a N\ :sub:`ROIs`\ x N\ :sub:`ROIs`\ matrix. :Observed coeff: ``.mat`` file containing N\ :sub:`ROI_pairs`\ x 1 matrix of observed edge-level coefficients. :Permuted p: ``.mat`` file containing N\ :sub:`ROI_pairs`\ x N\ :sub:`permutations`\ of logical values. Observed, thresholded, permuted p-values. :Permuted coeff: ``.mat`` file containing N\ :sub:`ROI_pairs`\ x N\ :sub:`permutations`\ of permuted edge-level coefficients. -Creating an edge-level test -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Creating additional edge-level tests +----------------------------------------------- -1. All test objects must inherit from the ``nla.edge.test.Base`` class. -2. All test objects must be saved into the ``+nla/+edge/+test`` directory -3. There are a few required properties and methods that are required: - - * **name** - * All tests must be given a name. Example:: - - properties (Constant) - name = "Kendall's tau" - end - - * **result** - * This is a method. The function handle is:: +To create an edge-level test, a test class must be added to the codebase. Refer to current tests in ``+nla/+edge/+test`` for examples. Guidelines are listed below - function result = run(obj, test_options) +* **Test objects** + All test objects must inherit from ``nla.edge.test.Base`` and be in the ``+nla/+edge/+test`` directory. There are also a few methods and + properties that must be included - * **test_options** - * This is a structure with options that are used either in the test or visualizing results + * A constant property ``name`` is required. - * **requiredInputs** - * This is a static function to define the inputs for the test:: - - methods (Static) - function inputs = requiredInputs() - inputs = {nla.inputField.Number('prob_max', 'P <', 0, 0.05, 1), nla.inputField.NetworkAtlas(), nla.inputField.Behavior()}; - end - end + :: + + properties (Constant) + name = "Pearson's r" + end - * This defines a number field ``prob_max`` from [0, 1] with a default of 0.05. It also specifies a network atlas (:ref:`NetworkAtlas() ` input field, and a behavior input field. - * These are all required. If the user does not supply them, the test not run in the GUI. + * A ``run`` method is also required. -4. If the test is located in the correct folder, after a GUI restart (not MATLAB GUI) the test will populate in the Edge Level test list. + :: -In addition to creating the test, a result object will also need to be created. + result = run(obj, input_struct) -Creating a result -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + :input_struct: Also called ``test_options`` in the codebase. Parameters needed for a test. The functional connectivity, network atlas, and other properties are stored here. + +.. _requiredInputs: + + * A ``requiredInputs`` method. + + :: + + methods (Static) + function inputs = requiredInputs() + inputs = { + nla.inputField.Number('prob_max', 'P <', 0, 0.05, 1), + nla.inputField.NetworkAtlas(), + nla.inputField.Behavior() + } + end + end + + This function creates 3 input fields in the GUI. A number ``prob_max`` with range [0, 1] and a default value of 0.05. + A network atlas file, and a behavior file. These are required, meaning that the GUI will not run without these inputs being + fulfilled. These values are all stored in the ``input_struct`` object. -1. ``nla.edge.BaseResult`` will work if custom data fields are not required. -2. The result must inherit from ``nla.edge.BaseResult`` -3. This result must be placed in ``+nla/+edge/+result/`` -4. Methods and properties +* **Result object** + A result object must be defined for the test edge-level results. If no custom data fields are needed, then the object in ``+nla/+edge/+test/Base.m`` + may be used and this step can be skipped. - * **output** - * This is the data that will be passed to create a figure of the data:: + * A ``output`` method must be included. + + :: function output(obj, network_atlas, flags) - * Network atlas :ref:`NetworkAtas() ` - * flags - a MATLAB structure that currently only has a field ``display_sig`` which is a boolean to determine if displayed p-values are thresholded + :network_atlas: An atlas of the form defined in ``nla.NetworkAtlas`` + :flags: Contains flags for the various types of figures to output. + + * (Optional) A ``merge`` method to merge blocks of permutation results together. An example can be found in + ``+nla/+edge/+result/PermBase.m`` file. - * **merge** - * This is an optional method - * It is used to merge blocks of results together (like in a parallel processing environment):: + :: - function merge(obj, results) + merge(obj, results) - * The ``results`` argument is a result to merge the object with. Afterwards, the current object will be the two merged blocks + :results: Cell array of result objects to merge. The object that calls the method will have the ``result`` merged with it. \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 4ee1a48e..2debd5fc 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,8 +15,11 @@ Welcome to NetworkLevelAnalysis's documentation! methodology setup getting_started + using_the_gui + behavior_table network_atlases edge_level_tests + network_level_tests bibliography diff --git a/docs/source/network_level_tests.rst b/docs/source/network_level_tests.rst index fcd08dca..007d485b 100644 --- a/docs/source/network_level_tests.rst +++ b/docs/source/network_level_tests.rst @@ -17,8 +17,8 @@ only the selected network. Common Inputs ------------------------ -:P: Network level p-value threshold -:Behavior Count: Number of different behavior vectors intended to test. +:P: Network-level p-value threshold +:Behavior Count: Number of different behavior vectors intended to test. P-values can be Bonferroni corrected by this number Provided Tests --------------------------- @@ -36,18 +36,92 @@ Provided Tests * **Kolmogorov-Smirnov** + * MATLAB `kstest2 `_ function. + +* **Wilcoxon rank-sum test** + + * MATLAB `ranksum `_ function. + * MATLAB's `kstest2 ` * **Wilcoxon** * MATLAB's `ranksum ` * **Welch's t-test** + + * Implements an optomized Welch's t-test comparing the functional connectivity of two groups. + * Extra imputs compared to other edge level tests - * Uses a modified version of MATLAB's t-test found in `+nla/WelchT` + :Group name(s): Names associated with each group. (For example, 'Male' and 'Female') + :Group val(s): Behavioral value associated with each group. If 'Female' is donated as '0', and 'Male' as '1', set the vals to the numerical values. -.. math:: +* **Student's t-test** + + * MATLAB `ttest2 `_ function. + +Creating additional network-level tests +----------------------------------------------------- + +To create a network-level test, a test class must be added to the codebase. Refer to the current tests in ``+nla/+net/+test`` + +* **Test objects** + All test objects must inherit from ``nla.net.test.Base`` and be in the ``+nla/+net/+test`` directory. There are also properties and methods + that must be included. + + * Constant properties required + :: + + properties (Constant) + name = "students_t" + display_name = "Student's T-test" + statistics = ["t_statistic", "single_sample_t_statistic"] + ranking_statistic = "t_statistic" + end + + + :name: The name of the test with no special characters (spaces, &, etc) + :display_name: A formal name that will be used for displaying in the GUI. Any string will work + :statistics: All statistics that will be generated by the test. No special characters + :ranking_statistic: The statistic used for ranking and calculating p-values. Note: if there is a single sample version of the statisticin addition to a two sample statistic, the GUI will automatically add "single_sample\_" during rankings for non-permuted and within network pair ranking. + + * ``run`` + :: + + result = run(obj, test_options, edge_test_results, network_atlas, permutations) + + + :test_options: Also called ``input_struct`` in edge-level tests. Parameters needed to run the test. + :edge_test_results: The output from the edge-level test. + :network_atlas: A network atlas of the form ``nla.NetworkAtlas`` + :permutations: Boolean to determine if the test is being run with permutations (``true``) or without (``false``) + + * ``requiredInputs`` See :ref:`Edge-level tests ` + + +Network Level results +--------------------------------------- +.. mat:module:: . + +All network level tests use ``nla.net.NetworkTestResult`` as the result object. This object uses the properties of the test +to name the results. + +.. mat:autoclass:: NetworkTestResult + + .. mat:automethod:: merge + + .. mat:automethod:: concatenateResult + + .. mat:automethod:: output + + .. mat:automethod:: createResultsStorage + + .. mat:automethod:: editableOptions + + .. mat:automethod:: getPValueNames - t = \frac{\overline{X_1} - \overline{X_2}}{\sqrt{s_{\overline{X_1}}^2 - s_{\overline{X_2}}^2}} - where - s_{\overline{X_i}} = \frac{s_i}{\sqrt{N_i}} +The three test methods (no_permutations, full_connectome, within_network_pair) will each contain multiple TriMatrices (lower half of a square matrix) of results. -* \ No newline at end of file +:d: This is the results of a Cohen's D test (effects size) using the results of the test method +:uncorrected__sample_p_value: The uncorrected p-value found by ranking the observed (non-permuted) result versus the test results of all the permutations. +:legacy__sample_p_value: The p-value found using the individual test p-values. Not verified for correctness. +:westfall__sample_p_value: The uncorrected p-value corrected for family-wise error rate using the method described by Westfall and Young :cite:p:`WestfallPH` +:winkler__sample_p_value: The uncorrected p-value corrected using the method described in :cite:p:`WinklerA` diff --git a/docs/source/refs.bib b/docs/source/refs.bib index 0f4e0d1c..a3a9beb6 100644 --- a/docs/source/refs.bib +++ b/docs/source/refs.bib @@ -446,5 +446,21 @@ @Article{SeitzmanB pages = {116290} } +@Book{WestfallP, + title = {Resampling-Based Multiple Testing: Examples and Methods for p-Value Adjustment.}, + author = {Westfall P.H. & Young S.S.}, + publisher = {Wiley}, + year = {1993} +} + +@Article{WinklerA, + title = {Permutation inference for the general linear model.}, + author = {Winkler A. Ridgway G. Webster M. Smith S. & Nichols T.}, + journal = {NeuroImage}, + year = {2014}, + volume = {92}, + pages = {381-397} +} + .. |oumlat| unicode:: U+00D6 .. O umlaut .. |alpha| unicode:: U+03B1 .. alpha \ No newline at end of file diff --git a/docs/source/using_the_gui.rst b/docs/source/using_the_gui.rst new file mode 100644 index 00000000..3981e025 --- /dev/null +++ b/docs/source/using_the_gui.rst @@ -0,0 +1,80 @@ +Using the GUI +============================== + +Main Window +--------------------------- + +To start the GUI, navigate to the folder that contains the files and type ``NLA_GUI`` in the command section of the MATLAB window. +Or, in the file browser section of the MATLAB window, right click on ``NLA_GUI.mlapp`` and select ``Run``. + +.. figure:: _static/main_gui2.png + + Main windows of Network Level Analysis program + +1. Edge-level test dropdown selector (See :doc:`Edge-level Statistical Tests <../edge_level_tests>`) +2. Edge-level test pane + This pane will list all of the options and inputs needed for each test that's currently selected. + Usually there are selectors for functional connectivity, network atlas, and behavior files. There may also be other options depending on the test. + If "Precalculated data" is selected, there will be selectors for data instead. (See: :ref:`Precalculated data loader `) +3. Behavior table + This will display the table when the behavior file is loaded. The table is used to select the behvaior to test, co-variates used (optional), and + permutation groupings (optional). (See: :doc:`Behavior Table <../behavior_table>`) +4. Network-level test pane + Selection of network-level test(s). One can be selected, or multiple with Ctrl/Shift + left click. + See :doc:`Network-level tests` +5. Run options + Checkboxes to select test method(s). If within network pair is selected, full connectome will also be selected. + Permutation count is how many permutations to run. More permutations will take more time, but will produce more precise results. + Run will run the edge level test and open the results window. + +.. _loading_results: + +Loading Results +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +If previous data was saved (See :ref:`saving results `) there is an option to load it here. Click the ``File`` menu in the upper left-hand corner and select "Load Previous Results." +Depending on the size of the saved data, this could take a bit of time. + +Results Window +---------------------------------- + +.. figure:: _static/test_results_gui.png + + Results windown with results for Network Level Analysis program + +After ``Run`` is pressed in the main window, the results window will open. Initially, most of it bill be bank except for a ``View`` button to view the result +of the (non-permuted) edge-level test along with another ``Run`` button. Pressing this run button will begin running all the permutations of the edge-level and network-level test(s). + +After all the permutations of the tests are run, the window will change. + +1. Edge test results display +2. Network level test results. Grouped by test method. The list can be changed to group by test by pushing the ``Flip Nesting`` button. +3. Results display options +4. Open TriMatrix Plot. + This opens an interactive plot of the statistics. (See :ref:`TriMatrix Plot `) +5. Open Diagnostic Plots. + These three plots + +.. _saving_results: + +Saving Results +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To save results for use later (See :ref:`loading results `), click the ``File`` menu in the upper left-hand corner and select save. This may take a bit of time depending on how large the dataset is and how many permutations were run. +The results will be saved as: + +#. a ResultPool object using models and classes from the NLA codebase. This can only be used if the NLA is in MATLAB's current path. +#. a nested structure of data that can be used without the NLA code. The structures are in the same ordering as the ResultPool, but there are no built-in classes and orderings. + +.. _trimatrix_plot: + +TriMatrix Plot +--------------------------------- + +.. figure:: _static/trimatrix_plot.png + + Trimatrix (lower triangular area) of p-values and display options + +1. TriMatrix plot of p-values for selected test. +2. Options. After changing options, the ``Apply`` button must be pushed to take effect. + There are also two buttons to display chord plots. One displays the network-level results, one displays the edge-level results. The options for these must be selected before the + chord plots are opened. The chord plots will not update after they are opened. \ No newline at end of file