Merge pull request #2298 from robertoostenveld/contsili-update_dccnpath

update dccnpath and added test script

test/test_bug1254.m

@@ -2,12 +2,12 @@
 
 % MEM 2gb
 % WALLTIME 00:10:00
+% DEPENDENCY qsubcompile qsubcellfun
+% DATA public
 
 % the bug has not been fixed yet, so there is no point in automatically
 % executing this test
 return
-% DEPENDENCY qsubcompile qsubcellfun
-% DATA public
 
 dataset = dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/ctf/Subject01.ds');
 

test/test_bug168.m

@@ -3,7 +3,7 @@
 % MEM 2gb
 % WALLTIME 00:10:00
 % DEPENDENCY ft_realtime_topography
-% DATA public
+% DATA no
 
 [ftver, ftpath] = ft_version;
 cd(ftpath);

test/test_bug2590.m

@@ -5,7 +5,7 @@
 % DEPENDENCY ft_removetemplateartifact ft_definetrial ft_artifact_ecg ft_preprocessing ft_timelockanalysis
 % DATA public
 
-clear all
+% clear all
 close all
 
 cd(dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/ctf'));

test/test_dccnpath.m

@@ -0,0 +1,134 @@
+function test_dccnpath
+
+% MEM 2gb
+% WALLTIME 00:10:00
+% DEPENDENCY dccnpath
+% DATA public
+
+global ft_default
+ft_default.dccnpath = [];
+
+% remember the original directory
+origdir = pwd;
+
+%% Alternative0: Finds the right path for users that have access to the DCCN computer cluster. It will replace '/home' by 'H:' and will replace forward by backward slashes.
+
+try
+  filename = dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/edf/testAlphaIR20170321-0.edf');
+  if ispc
+    if ~strcmp(filename, 'H:\common\matlab\fieldtrip\data\ftp\test\edf\testAlphaIR20170321-0.edf')
+      error('Alternative0 does not work');
+    end
+  else
+    if ~strcmp(filename, '/home/common/matlab/fieldtrip/data/ftp/test/edf/testAlphaIR20170321-0.edf')
+      error('Alternative0 does not work');
+    end
+  end
+catch
+  hostname = gethostname();
+  if startsWith(hostname, 'DCCN') || startsWith(hostname, 'dccn')
+    error('dccnpath() does not find the right DCCN path')
+  end
+end
+
+%% Alternative1: Test with local files in the present working directory
+
+tmpdir = tempname;
+mkdir(fullfile(tmpdir, 'ftp', 'test', 'edf'));
+cd(fullfile(tmpdir, 'ftp', 'test', 'edf'));
+websave('testAlphaIR20170321-0.edf', 'https://download.fieldtriptoolbox.org/test/edf/testAlphaIR20170321-0.edf');
+
+filename = dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/edf/testAlphaIR20170321-0.edf');
+if ~startsWith(filename, tmpdir)
+  error('dccnpath() does not find the right working directory');
+end
+
+% return to the original directory
+cd(origdir);
+
+%% Alternative2: It downloads test data to the local computer, if test data is not already downloaded. It has 6 different tests that are listed below
+
+%% Test1: When ft_default.dccnpath is given by the user, dccnpath() should always select alternative2 (this is not done now)
+
+% ... continue with the temporary copy from the previous test
+ft_default.dccnpath = tmpdir;
+
+filename = dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/edf/testAlphaIR20170321-0.edf');
+if ~startsWith(filename, tmpdir)
+  error('dccnpath() does not find the right working directory');
+end
+
+rmdir(ft_default.dccnpath, 's')
+
+%% Test2: Downloading a file from the HTTPS download server
+
+tmpdir = tempname;
+ft_default.dccnpath = tmpdir;
+
+filename = dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/ctf/Subject01.ds/ClassFile.cls');
+if ~exist(filename, 'file')
+  error('File is not downloaded from the HTTPS FieldTrip server');
+end
+
+%% Test3: File exists in the local copy and it doesn't get downloaded again
+
+% ... continue with the temporary copy from the previous test
+
+filename = dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/ctf/Subject01.ds/ClassFile.cls'); %'/home/common/matlab/fieldtrip/data/ftp/test/ctf/Subject01.ds/ClassFile.cls' has already been downloaded by test2
+if ~exist(filename, 'file')
+  error('File exists in the local copy, but dccnpath() can not find it');
+end
+
+rmdir(ft_default.dccnpath, 's');
+
+%% Test4: Downloading a folder from the HTTPS download server
+
+tmpdir = tempname;
+ft_default.dccnpath = tmpdir;
+
+foldername = dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/ctf/Subject02.ds');
+if ~exist(foldername, 'dir') % Here I check only if the main dir exists and not if this dir has the necessary contents.
+  error('Folder is not downloaded from the HTTPS FieldTrip server');
+end
+
+%% Test5: Folder exists in the local copy and it doesn't get downloaded again
+
+% ... continue with the temporary copy from the previous test
+
+foldername = dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/ctf/Subject02.ds'); %'/home/common/matlab/fieldtrip/data/ftp/test/ctf/Subject02.ds/' has already been downloaded by test4
+if ~exist(foldername, 'dir')
+  error('Folder exists in the local copy, but dccnpath() can not find it');
+end
+
+rmdir(ft_default.dccnpath, 's');
+
+%% Test6: When ft_default.dccnpath is not specified, then data should be saved automatically to tempdir
+
+% Download public data
+if isfield(ft_default, 'dccnpath')
+  ft_default = rmfield(ft_default, 'dccnpath');
+end
+
+hostname = gethostname();
+if ~startsWith(hostname, 'DCCN') && ~startsWith(hostname, 'dccn') % this test applies to external contributors and not for users in the DCCN cluster
+  filename = dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/ctf/Subject01.ds/ClassFile.cls');
+  if ~exist(filename, 'file') || ~contains(filename, tempdir)
+    error('Data were not automatically downloaded to tempdir');
+  end
+end
+
+% Do not download private data
+if isfield(ft_default, 'dccnpath')
+  ft_default = rmfield(ft_default, 'dccnpath');
+end
+
+hostname = gethostname();
+if ~startsWith(hostname, 'DCCN') && ~startsWith(hostname, 'dccn') % this test applies to external contributors and not for users in the DCCN cluster
+  try
+    filename = dccnpath('/home/common/matlab/fieldtrip/data/ftp/test/bug1027.mat');
+    ok = true;
+  catch
+    ok = false;
+  end
+  assert(~ok, 'Private data should not have been downloaded')
+end

utilities/dccnpath.m

@@ -4,22 +4,32 @@
 % test file from Linux, Windows or macOS computers both inside and outside the DCCN.
 %
 % Use as
-%  filename = dccnpath(filename)
+%   filename = dccnpath(filename)
+% where the input filename corresponds to the test data on the DCCN cluster and the
+% output filename corresponds to the local file including the full path where the
+% test data is available.
 %
-% The default location for FieldTrip and its test data is '/home/common/matlab/fieldtrip'.
-% This function will search-and-replace this string by the location that applies to
-% your computer. It will replace '/home' by 'H:' and will replace forward by backward slashes.
+% The test data location on the DCCN cluster is '/home/common/matlab/fieldtrip/data'
+% and the specification of the input filename MUST start with this.
 %
-% In case you have a local copy, you can override the default location by
-%   global ft_default
-%   ft_default.dccnpath = '/your/copy';
+% This function will search-and-replace the location on the DCCN cluster by the
+% location that applies to your computer. If needed, it will replace '/home' by 'H:'
+% and will replace forward by backward slashes.
 %
-% Note that most test scripts expect data located at /home/common/matlab/fieldtrip/data/ftp
-% or /home/common/matlab/fieldtrip/data/test, hence you should organize your local
-% copy of the data under /your/copy/data/ftp and /your/copy/data/test.
+% In case you have a local copy of the data, or if you are inside the DCCN and have
+% mounted the '/home' drive on another letter than 'H:', you should override the
+% default location using
+%    global ft_default
+%    ft_default.dccnpath = '/your/copy';
 %
-% Copyright (C) 2012-2022, Donders Centre for Cognitive Neuroimaging, Nijmegen, NL
+% If you do not have a local copy and do not define ft_default.dccnpath manually,
+% then dccnpath will automatically use a temporary directory and try to download the
+% data.
+%
+% See also WHICH, WEBSAVE
 
+% Copyright (C) 2012-2023, Donders Centre for Cognitive Neuroimaging, Nijmegen, NL
+%
 % This file is part of FieldTrip, see http://www.fieldtriptoolbox.org
 % for the documentation and details.
 %
@@ -40,41 +50,136 @@
 
 global ft_default
 
-% switch between linux and windows paths
+% it must always start with this
+assert(startsWith(filename, '/home/common/matlab/fieldtrip/data'));
+
+% alternative0 applies when in the DCCN cluster
 if ispc
-  filename = strrep(filename,'/home','H:');
-  filename = strrep(filename,'/','\');
+  alternative0 = strrep(filename,'/home','H:');
+  alternative0 = strrep(alternative0,'/','\');
 else
-  filename = strrep(filename,'H:','/home');
-  filename = strrep(filename,'\','/');
+  alternative0 = strrep(filename,'H:','/home');
+  alternative0 = strrep(alternative0,'\','/');
 end
 
-% alternative1 allows to test with local files in the present working directory
+if exist(alternative0, 'file') || exist(alternative0, 'dir')
+  ft_notice('using default DCCN path %s', alternative0);
+  filename = alternative0;
+  return
+end
+
+% alternative1 applies with local files in the present working directory
 % this is often convenient when initially setting up a new test script while the data is not yet uploaded
-[p, f, x] = fileparts(filename);
+[p, f, x] = fileparts(alternative0);
 alternative1 = [f x];
 
 if strcmp(alternative1, 'test')
   % this should not be used when the filename is only "test", since that is too generic
+  warning('the name "test" is too generic')
   alternative1 = '';
-elseif endsWith(pwd, alternative1)
-  % exist(alternative1, 'file') also returns 7 in case the present working directory name matches alternative1
-  % this is not the match we want
+elseif strcmp(alternative1, 'ctf')
+  % this should not be used when the filename is only "ctf", since that is too generic
+  warning('the name "ctf" is too generic')
   alternative1 = '';
 end
 
-% alternative2 allows for the user to specify the path in the global ft_default variable
+if exist(alternative1, 'file') || exist(alternative1, 'dir')
+  if ~isempty(x) && ~isequal(x,'.ds')   % if alternative1 is a file
+    filenamepath = which(alternative1); % also output the path that alternative1 is located at
+    ft_notice('using present working directory %s', filenamepath);
+    filename = filenamepath;
+    return;
+  else                 % if alternative1 is a folder
+    searchPath = path; % get the MATLAB search path
+    directories = strsplit(searchPath, pathsep); % split the search path into individual directories
+
+    folderPath = '';
+    for i = 1:numel(directories)
+      currentFolder = directories{i};
+      if contains(currentFolder, alternative1)
+        folderPath = currentFolder;
+        break
+      end
+    end
+    warning('using present working directory %s', folderPath);
+    filename = folderPath;
+    return
+  end
+end
+
+% alternative2 applies when specified ft_default.dccnpath
 % see https://github.com/fieldtrip/fieldtrip/issues/1998
-if isfield(ft_default, 'dccnpath')
-  alternative2 = strrep(filename, '/home/common/matlab/fieldtrip', ft_default.dccnpath);
+
+if ~isfield(ft_default, 'dccnpath') || isempty(ft_default.dccnpath)
+  ft_notice('using a temporary directory')
+  ft_default.dccnpath = tempdir;
+end
+
+% we do not want it to end with a '/' or '\'
+strip(ft_default.dccnpath, 'right', '/');
+strip(ft_default.dccnpath, 'right', '\');
+
+if ~ispc
+  alternative2 = strrep(alternative0, '/home/common/matlab/fieldtrip/data', ft_default.dccnpath);
 else
-  alternative2 = '';
+  alternative2 = strrep(alternative0, 'H:\common\matlab\fieldtrip\data', ft_default.dccnpath);
 end
 
-if ~exist(filename, 'file') && exist(alternative1, 'file')
-  warning('using local copy %s instead of %s', alternative1, filename);
-  filename = alternative1;
-elseif ~exist(filename, 'file') && exist(alternative2, 'file')
-  warning('using local copy %s instead of %s', alternative2, filename);
+if exist(alternative2, 'file')
+  ft_notice('using local copy %s ', alternative2);
   filename = alternative2;
+  return
+
+elseif isfolder(alternative2) && ~isemptydir(alternative2)
+  ft_notice('using local copy %s ', alternative2);
+  filename = alternative2;
+  return
+
+else
+  % if the file doesn't exist or the folder is empty, then download test data
+  % see also UNTAR, UNZIP, GUNZIP, which can download on the fly
+
+  if contains(alternative2, 'data/test') || contains(alternative2, 'data\test')
+    error('the test data are private and can not be downloaded')
+  end
+
+  % public data are downloaded from https://download.fieldtriptoolbox.org
+  % so, we need to find the right path to the HTTPS download server
+  pattern = 'ftp(.*)';
+  datadir = regexp(alternative0, pattern, 'tokens', 'once');
+  weblocation = strcat("https://download.fieldtriptoolbox.org", datadir);
+  weblocation = strrep(weblocation, '\', '/');
+
+  urlContent = webread(weblocation, weboptions('ContentType', 'text'));
+
+  if contains(urlContent, '<html')
+    % the URL corresponds to a folder
+    ft_notice('downloading recursively from %s', weblocation);
+    recursive_download(weblocation, alternative2);
+    filename = alternative2;
+
+  else
+    % the URL corresponds to a file
+    % create the necessary directories if they do not exist
+    [p, f, x] = fileparts(alternative2);
+    if ~isfolder(p)
+      mkdir(p);
+    end
+
+    if isfolder(alternative2)
+      [p, f, x] = fileparts(alternative0);
+      alternative2 = fullfile(alternative2, [f x]);
+    end
+
+    ft_notice('downloading recursively from %s', weblocation);
+    websave(alternative2, weblocation);
+    filename = alternative2;
+  end
 end
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% SUBFUNCTION
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+function bool = isemptydir(dirName)
+dirContents = dir(dirName);
+bool = ~isempty(dirContents(~ismember({dirContents.name}, {'.', '..'})));