ENH - allow for slower, but more memory efficient appending (#2510)

Author: schoffelen

ft_appendfreq.m

@@ -25,10 +25,14 @@
 % These mat files should contain only a single variable, corresponding with
 % the input/output structure.
 %
+% If you encounter difficulties with memory usage, you can use
+%   cfg.memory = 'low' or 'high', whether to be memory or computationally efficient, respectively (default = 'high')
+%
 % See also FT_FREQANALYSIS, FT_DATATYPE_FREQ, FT_APPENDDATA, FT_APPENDTIMELOCK,
 % FT_APPENDSENS
 
 % Copyright (C) 2011-2017, Robert Oostenveld
+% Copyright (C) 2018-, Jan-Mathijs Schoffelen and Robert Oostenveld
 %
 % This file is part of FieldTrip, see http://www.fieldtriptoolbox.org
 % for the documentation and details.
@@ -77,6 +81,7 @@
 cfg.appenddim  = ft_getopt(cfg, 'appenddim', []);
 cfg.tolerance  = ft_getopt(cfg, 'tolerance',  1e-5); % this is passed to append_common, which passes it to ft_selectdata
 cfg.appendsens = ft_getopt(cfg, 'appendsens', 'no');
+cfg.memory     = ft_getopt(cfg, 'memory', 'high');
 
 hastime = isfield(varargin{1}, 'time');
 hasfreq = isfield(varargin{1}, 'freq');
@@ -120,7 +125,14 @@
 assert(~isempty(cfg.parameter), 'cfg.parameter should be specified');
 
 % use a low-level function that is shared with the other ft_appendxxx functions
-freq = append_common(cfg, varargin{:});
+if strcmp(cfg.memory, 'high') || numel(varargin)<=2
+  freq = append_common(cfg, varargin{:});
+elseif strcmp(cfg.memory, 'low')
+  freq = varargin{1};
+  for i=2:numel(varargin)
+    freq = append_common(cfg, freq, varargin{i});
+  end
+end
 
 % do the general cleanup and bookkeeping at the end of the function
 ft_postamble debug

ft_appendtimelock.m

@@ -16,10 +16,22 @@
 %                         are allowed to still be considered compatible (default = 1e-5)
 %   cfg.keepsampleinfo  = 'yes', 'no', 'ifmakessense' (default = 'ifmakessense')
 %
+% To facilitate data-handling and distributed computing you can use
+%   cfg.inputfile   =  ...
+%   cfg.outputfile  =  ...
+% If you specify one of these (or both) the input data will be read from a
+% *.mat file on disk and/or the output data will be written to a *.mat file.
+% These mat files should contain only a single variable, corresponding with
+% the input/output structure.
+%
+% If you encounter difficulties with memory usage, you can use
+%   cfg.memory = 'low' or 'high', whether to be memory or computationally efficient, respectively (default = 'high')
+%
 % See also FT_TIMELOCKANALYSIS, FT_DATATYPE_TIMELOCK, FT_APPENDDATA, FT_APPENDFREQ,
 % FT_APPENDSOURCE, FT_APPENDSENS
 
 % Copyright (C) 2011-2018, Robert Oostenveld
+% Copyright (C) 2019-, Jan-Mathijs Schoffelen and Robert Oostenveld
 %
 % This file is part of FieldTrip, see http://www.fieldtriptoolbox.org
 % for the documentation and details.
@@ -63,6 +75,7 @@
 cfg.tolerance       = ft_getopt(cfg, 'tolerance',  1e-5); % this is passed to append_common, which passes it to ft_selectdata
 cfg.appendsens      = ft_getopt(cfg, 'appendsens', 'no');
 cfg.keepsampleinfo  = ft_getopt(cfg, 'keepsampleinfo', 'no');
+cfg.memory          = ft_getopt(cfg, 'memory', 'high');
 
 try
   % although not 100% robust, this could make some users becoming aware of the issue of overlapping trials
@@ -114,7 +127,14 @@
 end
 
 % use a low-level function that is shared with the other ft_appendxxx functions
-timelock = append_common(cfg, varargin{:});
+if strcmp(cfg.memory, 'high') || numel(varargin)<=2
+  timelock = append_common(cfg, varargin{:});
+elseif strcmp(cfg.memory, 'low')
+  timelock = varargin{1};
+  for i=2:numel(varargin)
+    timelock = append_common(cfg, timelock, varargin{i});
+  end
+end
 
 if isfield(timelock, 'avg') && ~isfield(timelock, 'trial')
   ft_warning('renaming the appended averages to "trial"');

test/test_ft_appendtimelock.m

@@ -180,4 +180,55 @@
 assert(isfield(tlockapp, 'sampleinfo')); % here it should exist
 assert(~isfield(tlockapp, 'fsample'));
 
+%% test with cfg.memory high/low
+tlock1 = [];
+tlock1.label = {'1';'2'};
+tlock1.time = 1:5;
+tlock1.dimord = 'rpt_chan_time';
+tlock1.trial = randn(10, 2, 5);
+
+tlock2 = [];
+tlock2.label = {'1';'2'};
+tlock2.time = 1:5;
+tlock2.dimord = 'rpt_chan_time';
+tlock2.trial = randn(8, 2, 5);
+
+tlock3 = [];
+tlock3.label = {'1';'2'};
+tlock3.time = 1:5;
+tlock3.dimord = 'rpt_chan_time';
+tlock3.trial = randn(7, 2, 5);
+
+cfg = [];
+cfg.memory = 'high';
+thigh = ft_appendtimelock(cfg, tlock1, tlock2, tlock3);
+cfg.memory = 'low';
+tlow = ft_appendtimelock(cfg, tlock1, tlock2, tlock3);
+assert(isequal(thigh.trial, tlow.trial));
+
+tlock1 = [];
+tlock1.label = {'1';'2'};
+tlock1.time = 1:5;
+tlock1.dimord = 'rpt_chan_time';
+tlock1.trial = randn(8, 2, 5);
+
+tlock2 = [];
+tlock2.label = {'3';'4'};
+tlock2.time = 1:5;
+tlock2.dimord = 'rpt_chan_time';
+tlock2.trial = randn(8, 2, 5);
+
+tlock3 = [];
+tlock3.label = {'5';'6'};
+tlock3.time = 1:5;
+tlock3.dimord = 'rpt_chan_time';
+tlock3.trial = randn(8, 2, 5);
+
+cfg = [];
+cfg.memory = 'high';
+thigh = ft_appendtimelock(cfg, tlock1, tlock2, tlock3);
+cfg.memory = 'low';
+tlow = ft_appendtimelock(cfg, tlock1, tlock2, tlock3);
+assert(isequal(thigh.trial, tlow.trial));
+