Merge development into master

Date: 5/11/2023 3:06:37 PM
v1.79.3

-- Add more standalone startup function to setpaths.m to be able to independent of any startup code search paths except current folder
-- Chnage default value of config param Load Stim From TSV File from no to yes.

* DataTree, v1.14.2
-- Fix events TSV file naming bug. Add TSV file name method TreeNode.GetStimTsvFilename() to use for all events TSV file naming.
-- Fixes to setpaths for running DataTree standalone

* Utils, v1.6.0
-- Fix some bugs in events TSV stim loading error handling. Add config param "Replace TSV File Tabs with Spaces" to handle events TSV errors when file  is a mix of spaces and tabs.
-- Add method ConfigFileClass.GetValueOptions for getting all available value options for a param.

Date: 5/10/2023 10:52:19 AM
v1.79.1 -- Fix Homer3 displaying probe in 3D rather than 2D. (#177)

* DataTree, v1.14.0
-- Fix Homer3 displaying probe in 3D rather than 2D. Back out changes that default to 3D in ProbeClass and NirsClass GetSrcPos and GetDetPos for ALL cases. The way it's supposed to work is probe is supposed to use 2D coordinates ONLY when being displayed in Homer3. Rest of the time default to 3D if it exists. Reason for original change is misunderstanding of what the purpose of 2D coord was which is ONLY for display

* Utils, v1.5.1
-- Add ability to simulate full dataset (generateSimData.m, simulateDataTimeSeries.m)  to be able to eventually do workflow unit and system tests

Date: 5/3/2023 12:38:30 PM
v1.78.1  -- Fix error when displaying data, remove unnecessary call to get file descriptor: obj.fid = HDF5_GetFileDescriptor(fileobj); (#174)

Date: 5/3/2023 10:52:24 AM
-- Forgot to include HDF5_GetFileDescriptor.m in v1.78.0 commit. (#173)

Date: 5/3/2023 10:13:49 AM
v1.78.0 -- Improve saving performance of SNIRF files by using low level HDF5 library. (#172)
* DataTree, v1.13.0
-- Comprehensive solution to improving the save performance of SNIRF files by using low-level HDF5 calls and limiting the number of HDF5 open and create calls that are made for file, group and dataset.

Date: 4/14/2023 1:59:38 AM
1.77.0 -- Fix stim events TSV issue with conditions names with spaces - we should be able to handle that.  (#170)

Utils, v1.5.0
-- Fix stim events TSV issue with conditions names with spaces - we should be able to handle that. Change rule to handle spaces and how error reporting is done in readTsv.

DataTree, v1.12.1
-- Don't load events TSV files twice - once in SnirfClass and once in AcqDataClass its superclass.

Date: 4/11/2023 2:49:33 PM
-- Sync setpaths in DataTree with DataTree library

DataTree/AcquiredData/AcqDataClass.m

@@ -92,7 +92,6 @@
             if ~ischar(fileobj)
                 fileobj = '';
             end
-            obj.LoadBids(fileobj);           
         end
 
 
@@ -162,7 +161,11 @@ function Initialize(obj)
             if isempty(file)
                 return
             end
-            obj.bids.stim = readTsv([filesepStandard(p), file(1).name],'numstr2num');
+            [obj.bids.stim, err] = readTsv([filesepStandard(p), file(1).name],'numstr2num');
+%             if err < 0
+%                 obj.SetError(-8);
+%                 return;
+%             end
             if isempty(obj.bids.stim)
                 return
             end

DataTree/AcquiredData/DataFiles/FileLoadSaveClass.m

@@ -62,6 +62,11 @@ function Save(obj, filename, params, format)
             if ~exist('format','var')
                 format = obj.fileformat;
             end
+
+            p = fileparts(filename);
+            if isempty(p)
+                filename = ['./', filename];
+            end
 
             switch(lower(format))
                 case obj.supportedFomats.matlab

DataTree/AcquiredData/DataFiles/Hdf5/HDF5_GetFileDescriptor.m

@@ -0,0 +1,15 @@
+function fid = HDF5_GetFileDescriptor(fileobj)
+
+% Either fileobj is a name of a file or a HDF5 file descriptor
+if ischar(fileobj)
+    if ispathvalid(fileobj,'file')
+        fid = H5F.open(fileobj, 'H5F_ACC_RDWR', 'H5P_DEFAULT');
+    else
+        fid = H5F.create(fileobj, 'H5F_ACC_TRUNC', 'H5P_DEFAULT', 'H5P_DEFAULT');
+    end
+    if fid < 0
+        return;
+    end
+else
+    fid = fileobj;
+end

DataTree/AcquiredData/DataFiles/Hdf5/hdf5write_safe.m

@@ -1,122 +1,158 @@
-function err = hdf5write_safe(fname, name, val, options)
-
+function err = hdf5write_safe(fileobj, name, val, options)
+err = -1;
+if isempty(val)
+    return;
+end
+if ~exist('options','var')
+    options = '';
+end
 
+force_scalar = false;
+force_array = false;
+if any(strcmp(options, 'array'))
+    force_array = true;
+elseif any(strcmp(options, 'scalar'))
+    force_scalar = true;
+end
+
+% Identify type of val and use SNIRF v1.1-compliant write function
+fid = HDF5_GetFileDescriptor(fileobj);
+if fid < 0
     err = -1;
-    if isempty(val)
-        return;
-    end
-    if ~exist('options','var')
-        options = '';
-    end
-
-    force_scalar = false;
-    force_array = false;
-    if any(strcmp(options, 'array'))
-       force_array = true;
-    elseif any(strcmp(options, 'scalar'))
-        force_scalar = true;
-    end
-    % Identify type of val and use SNIRF v1.1-compliant write function
-
-    if exist(fname,'file')
-        fid = H5F.open(fname, 'H5F_ACC_RDWR', 'H5P_DEFAULT');
+    return;
+end
+
+% Create group where dataset is located
+group = filesepStandard(fileparts(name), 'nameonly');
+gid = HDF5_CreateGroup(fid, group);
+if gid < 0
+    err = -1;
+    return;
+end
+
+% Create dataset
+if iscell(val) || isstring(val)
+    if length(val) > 1 && ~force_scalar || force_array  % Returns true for single strings, believe it or not
+        write_string_array(fid, name, val);
     else
-        fid = H5F.create(fname, 'H5F_ACC_TRUNC', 'H5P_DEFAULT', 'H5P_DEFAULT');
-    end
-    if fid < 0
-        err = -1;
-        return;
+        write_string(fid, name, val);
     end
-
-    % Create group where dataset is located
-    group = filesepStandard(fileparts(name), 'nameonly');
-    gid = HDF5_CreateGroup(fid, group);
-    if gid < 0
-        err = -1;
-        return;
+elseif ischar(val)
+    write_string(fid, name, val);
+elseif isfloat(val)
+    if length(val) > 1 && ~force_scalar || force_array
+        write_numeric_array(fid, name, val);
+    else
+        write_numeric(fid, name, val);
     end
-
-    if iscell(val) || isstring(val)
-        if length(val) > 1 && ~force_scalar || force_array  % Returns true for single strings, believe it or not
-            write_string_array(fid, fname, name, val);
-        else
-            write_string(fid, fname, name, val);
-        end
-        return
-    elseif ischar(val)
-        write_string(fid, fname, name, val);
-        return
-    elseif isfloat(val)
-        if length(val) > 1 && ~force_scalar || force_array
-            write_numeric_array(fname, name, val);
-        else
-            write_numeric(fid, fname, name, val);
-        end
-        return
-    elseif isinteger(val)
-        if length(val) > 1 && ~force_scalar || force_array
-            write_numeric_array(fid, fname, name, val);  % As of now, no integer arrays exist
-        else
-            write_integer(fid, fname, name, val);
-        end
-        return
+elseif isinteger(val)
+    if length(val) > 1 && ~force_scalar || force_array
+        write_numeric_array(fid, name, val);  % As of now, no integer arrays exist
     else
-        warning(['An unrecognized variable was saved to ', name, ' in ', fname])
+        write_integer(fid, name, val);
     end
-
-    H5F.close(fid);
-
+else
+    warning(['An unrecognized variable was saved to ', name])
 end
 
-function err = write_string(fid, fname, name, val)
-    sid = H5S.create('H5S_SCALAR');
-    tid = H5T.copy('H5T_C_S1');
-    H5T.set_size(tid, 'H5T_VARIABLE');
-    did = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');
-    H5D.write(did, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', {val});
-    err = 0;
-end
 
-function err = write_string_array(fid, fname, name, val)
-    val = HDF5_Transpose(val);
-    sid = H5S.create_simple(1, numel(val), H5ML.get_constant_value('H5S_UNLIMITED'));
-    tid = H5T.copy('H5T_C_S1');
-    H5T.set_size(tid, 'H5T_VARIABLE');
-    pid = H5P.create('H5P_DATASET_CREATE');
-    H5P.set_chunk(pid, 2);
-    did = H5D.create(fid, name, tid, sid, pid);
-    H5D.write(did, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', val);
-    err = 0;
-end
 
-function err = write_numeric(fid, fname, name, val)
-    fid = H5F.open(fname, 'H5F_ACC_RDWR', 'H5P_DEFAULT');
-    tid = H5T.copy('H5T_NATIVE_DOUBLE');
-    sid = H5S.create('H5S_SCALAR');
-    H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');
-    h5write(fname, name, val);
-    err = 0;
+% -----------------------------------------------------------------
+function err = write_string(fid, name, val)
+sid = H5S.create('H5S_SCALAR');
+tid = H5T.copy('H5T_C_S1');
+H5T.set_size(tid, 'H5T_VARIABLE');
+did = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');
+H5D.write(did, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', {val});
+err = 0;
+
+
+
+% -----------------------------------------------------------------
+function err = write_string_array(fid, name, val)
+val = HDF5_Transpose(val);
+sid = H5S.create_simple(1, numel(val), H5ML.get_constant_value('H5S_UNLIMITED'));
+tid = H5T.copy('H5T_C_S1');
+H5T.set_size(tid, 'H5T_VARIABLE');
+pid = H5P.create('H5P_DATASET_CREATE');
+H5P.set_chunk(pid, 2);
+dsid = H5D.create(fid, name, tid, sid, pid);
+H5D.write(dsid, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', val);
+err = 0;
+
+
+
+% -----------------------------------------------------------------
+function err = write_numeric(fid, name, val)
+tid = H5T.copy('H5T_NATIVE_DOUBLE');
+sid = H5S.create('H5S_SCALAR');
+dsid = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');
+H5D.write(dsid, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', val);
+err = 0;
+
+
+
+% -----------------------------------------------------------------
+function err = write_integer(fid, name, val)
+warning off;  % Suppress the int truncation warning
+tid = H5T.copy('H5T_NATIVE_ULONG');
+sid = H5S.create('H5S_SCALAR');
+dsid = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');
+H5D.write(dsid, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', int32(val));
+err = 0;
+warning on;
+
+
+
+% -----------------------------------------------------------------
+function err = write_numeric_array(fid, name, data)
+err = 0;
+data = HDF5_Transpose(data);
+sizedata = size(data);
+if sizedata(1) == 1 || sizedata(2) == 1
+    n = length(data);
+else
+    n = sizedata;
 end
 
-function err = write_numeric_array(fname, name, val)
-    val = HDF5_Transpose(val);
-    sizeval = size(val);
-    if sizeval(1) == 1 || sizeval(2) == 1
-        n = length(val);
-    else
-        n = sizeval;
-    end
-    h5create(fname, name, n, 'Datatype', 'double');
-    h5write(fname, name, val);
-    err = 0;
+tid = -1;
+sid = -1;
+gid = -1;
+dsid = -1;
+
+maxdims = n;
+try
+
+    sid = H5S.create_simple(numel(n), fliplr(n), fliplr(maxdims));   
+    gid = HDF5_CreateGroup(fid, fileparts(name));
+    dsid = H5D.create(gid, name, 'H5T_NATIVE_DOUBLE', sid, 'H5P_DEFAULT');
+    H5D.write(dsid, 'H5ML_DEFAULT', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', data);
+
+catch
+
+    % Clean up; Close everything
+    cleanUp(tid, sid, gid, dsid);
+    err = -1;
+    return;
+
 end
+cleanUp(tid, sid, gid, dsid);
+
 
-function err = write_integer(fid, fname, name, val)
-    warning off;  % Suppress the int truncation warning
-    tid = H5T.copy('H5T_NATIVE_INT');
-    sid = H5S.create('H5S_SCALAR');
-    H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');
-    h5write(fname, name, val);
-    err = 0;
-    warning on;
+
+% ------------------------------------------------------
+function cleanUp(tid, sid, gid, dsid)
+if ~isnumeric(tid)
+    H5T.close(tid);
+end
+if ~isnumeric(sid)
+    H5S.close(sid);
+end
+if ~isnumeric(gid)
+    H5G.close(gid);
 end
+if ~isnumeric(dsid)
+    H5D.close(dsid);
+end
+
+

DataTree/AcquiredData/DataFiles/SnirfFile2Tsv.m

@@ -48,11 +48,4 @@
     writeTsv(dst, tsv);
 end
 
-% Remove stim from
-% if optionExists(options, 'removeStim')
-%     s.Load();
-%     s.stim = StimClass().empty();
-%     s.Save();
-% end
-