This repository has been archived by the owner on Jun 10, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1
/
ACFileLoaderDat.m
361 lines (287 loc) · 16.1 KB
/
ACFileLoaderDat.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
%ACFileLoaderDat - Opens an AC data file of type "dat" (as opposed to .bin
%file). Iterates through a list of files specified in the
%constructor and uses the specified import method to create new ACData
%objects. This is the file where the logic of specifying how
%the output of the import method gets stored in the ACData object.
%
% Syntax: loader = ACFileLoaderDat(fileNameIn, importMethodIn, deviceFileIn, outputLocationIn, unitsIn);
%
% Inputs:
% fileNameListIn - a list of filenames to load
% importMethodIn - a string specifying the name of the import function
% to use to read the fileNameIn
% deviceFileIn - the device file object for this data
% outputLocationIn -
% unitsIn -
%
% Example:
% acfl = ACFileLoaderDat( acFiles, params.AC_IMPORT_METHOD_NAME, params.DEVICE_FILE_LOCATION, prepacsOut, params.AC_UNITS);
%
% Other m-files required: ACData
% Subfunctions: none
% MAT-files required: none
%
% See also: ACDeviceFile, IngestManager, ACData
% Author: Wendy Neary
% MISCLab, University of Maine
% email address: wendy.neary@maine.edu
% Website: http://misclab.umeoce.maine.edu/index.php
% May 2015; Last revision: 24-Nov-15
% Fixed problem with skipping data files with totally overlapped data:
% If there was a data file whose timestamps completely overlapped the
% previous data file's timestamps, that file was being skipped.
% The next file's checking of timestamps was happening against the skipped
% file, however, and should have been happening against the last added
% file, not the last iterated file. I added a second iterator to keep
% track of which files had been added as good,vs. which files had been
% interated through. -WN
% June 2016 - Changed to fix discover that Compass Software was not naming
% - as stated in documentation (i.e. filename reflects END time
% - of software, not START time.
%------------- BEGIN CODE --------------
%%
classdef ACFileLoaderDat
properties
FileNameList
ImportMethodName
DeviceFileName
OutputLocation
Units
numWL
end
properties (SetAccess = private, GetAccess = private)
L; % Logger
end
methods
%%# ACFileLoaderDat
function obj = ACFileLoaderDat(fileNameListIn, importmethodIn, ...
deviceFileIn, outputLocationIn, unitsIn, numWLIn)
%#ACFileLoaderDat creates the data objects for the imported data
%#
%# SYNOPSIS ACFileLoaderDat(fileNameListIn, importmethodIn, deviceFileIn, outputLocationIn, unitsIn)
%# INPUT fileNameListIn - a list of filenames to load
%# importMethodIn - a string specifying the name of the import function
%# to use to read the fileNameIn
%# deviceFileIn - the device file object for this data
%# outputLocationIn -
%# unitsIn -
%# OUTPUT obj
%#
% get copy of logger
obj.L = log4m.getLogger();
% check input arguments
if nargin > 0
% check there is a file list
if ~isempty( fileNameListIn )
obj.FileNameList = fileNameListIn;
else
error('Need file list')
end
% check there is a import method specified
if ischar( importmethodIn )
obj.ImportMethodName = importmethodIn;
else
error('Need import method')
end
% check there is a device file specified
if ischar( deviceFileIn )
obj.DeviceFileName = deviceFileIn;
else
error('Need device file')
end
if ischar( outputLocationIn )
obj.OutputLocation = outputLocationIn;
else
error('Need output location for prepacs.tmp')
end
if ischar( unitsIn )
obj.Units = unitsIn;
else
error('Need units')
end
obj.numWL = str2num(numWLIn);
else
error('Supply an input argument')
end % end if nargin > 0
end % end constructor
%%# loadData
function dataOut = loadData(obj)
%loadData returns an array of AncillaryData objects. It gets
%the list of files set when the ACFileLoader was created. It
%reads the .dat files with the specified import method.
%
% SYNOPSIS dataOut = loadData(obj)
% INPUT obj - this object
% OUTPUT dataOut - cell array of adata and cdata data objects
nFiles = length(obj.FileNameList);
obj.L.info('ACFileLoaderDat', ...
sprintf('Number of files to iterate through: %u', nFiles));
% iterate through files, load from original data files and put
% in temporary structure to hold for looping & checking
for iFiles = 1:nFiles;
fileName = obj.FileNameList{iFiles};
obj.L.info('ACFileLoaderDat', ...
sprintf('Iteration %u. Filename: %s', iFiles, fileName'));
% -------------------------------------------------------
% open the data file and retrieve data from it
fh = str2func(obj.ImportMethodName);
disp(obj.ImportMethodName)
%changed 11/23
% get rid of any lines with a negative timestamp
origDataMatrix = fh(fileName);
index = origDataMatrix(:,1) >= 0;
dataMatrix = origDataMatrix(index, :);
% end change
% -------------------------------------------------------
% Data Processing Section:
% Logic is here because this is the object that knows about
% the specific FILE types --
timestampsMS = dataMatrix(:,1);
obj.L.debug('ACFileLoaderDat', ...
sprintf('dataMatrix size: %u x %u', size(dataMatrix)));
% calculate dimensions of matrix
endCat = obj.numWL+1;
cRawDataMatrix = dataMatrix(:,2:endCat);
startAat = endCat+1;
endAat = startAat + (obj.numWL - 1);
aRawDataMatrix = dataMatrix(:,startAat:endAat);
% create the correct timestamp array, using the dates from the file name
% (date not stored in timestamp)
% and adding the timestamps from the acdata file
[~, name, ext] = fileparts(fileName);
shortfilename = strcat(name, ext);
variablesFromFileName = textscan(shortfilename, 'acs%03s_%14s.dat');
dateFromFileName = variablesFromFileName{1,2};
fileDate = datenum(dateFromFileName, 'yyyymmddHHMMSS');
%------------------------------------------------------
%22Jun16:
% changed code to reflect issue with compass software
% realized filenames are not named according to the
% beginning time of the datafile name, but rather,
% the END of times in data file
% OLD CODE:
%timestamps = fileDate + datenum(0,0,0,0,0, (timestampsMS - timestampsMS(1))/1000);
% NEW CODE:
timestamps = fileDate - datenum(0,0,0,0,0, (timestampsMS(end) - timestampsMS)/1000);
%------------------------------------------------------
temp(iFiles).timestamps = timestamps;
temp(iFiles).fileName = fileName;
temp(iFiles).cRawDataMatrix = cRawDataMatrix;
temp(iFiles).aRawDataMatrix = aRawDataMatrix;
obj.L.debug('ACFileLoaderDat', ...
sprintf('timestamps start %s', datestr(timestamps(1), 'HH:MM:SS:FFF')));
obj.L.debug('ACFileLoaderDat', ...
sprintf('timestampsMS end %s', datestr(timestamps(end), 'HH:MM:SS:FFF')));
end; %# for iFiles = 1:nFiles;
nFiles = length(temp);
obj.L.info('ACFileLoaderDat', ...
sprintf('checking timestamps. Number of files to iterate through: %u', nFiles));
% create an index for the last good file. skipped files will
% not iterate this index
iLastAddedFile = 0;
for iFiles = 1:nFiles
currFile = temp(iFiles);
obj.L.debug('ACFileLoaderDat', ...
sprintf('iteration %u: filename: %s', iFiles, currFile.fileName));
if iLastAddedFile > 0
obj.L.debug('ACFileLoaderDat', ...
sprintf('last good added filename: %s', temp(iLastAddedFile).fileName));
end
[~, name, ext] = fileparts(currFile.fileName);
shortfilename = strcat(name, ext);
% if not last file
if iFiles ~= nFiles
nextFile = temp(iFiles +1);
obj.L.debug('ACFileLoaderDat', 'not last file');
%if the difference between the first timestamp of the
%next file and the last timestamp of this file is more
%than 2 hours, don't append (assume next day file?)
if ( nextFile.timestamps(1) - currFile.timestamps(end) ) > datenum(0,0,0,2,0,0);
obj.L.debug('ACFileLoaderDat', ...
'greater than 2 hour gap between this and next file; SHOULD SKIP TO NEXT FILE');
continue;
obj.L.error('ACFileLoaderDat', ...
'DID NOT SKIP TO NEXT FILE IN LOOP');
end
else
obj.L.debug('ACFileLoaderDat', 'last file');
end %# if not last file;
if iFiles > 1
% if not first file, check this file ok
prevAddedFile = temp(iLastAddedFile);
%if (prevFile.timestamps(end) > currFile.timestamps(1))
if (currFile.timestamps(1) < prevAddedFile.timestamps(end))
obj.L.debug('ACFileLoaderDat', ...
'this file''s first TS is BEFORE the end of the previous file''s last TS');
obj.L.debug('ACFileLoaderDat', ...
sprintf('this files first ts: %s', ...
datestr((currFile.timestamps(1)), 'HH:MM:SS:FFF')));
obj.L.debug('ACFileLoaderDat', ...
sprintf('prev files last ts: %s', ...
datestr((prevAddedFile.timestamps(end)), 'HH:MM:SS:FFF')));
%if (prevFile.timestamps(end) > currFile.timestamps(end))
if (currFile.timestamps(end) < prevAddedFile.timestamps(end))
obj.L.error('ACFileLoaderDat', ...
'curFile last TS BEFORE prevFile last TS - SKIPPING');
continue;
obj.L.error('ACFileLoaderDat', ...
'DID NOT SKIP TO NEXT FILE IN LOOP');
else
obj.L.error('ACFileLoaderDat', ...
'currFile last TS AFTER the prevFile last TS - BUT OVERLAP AT BEGINNING')
% find nearest last prev ts in this file, skip
% overlapping records
prevEndTime = prevAddedFile.timestamps(end);
obj.L.debug('ACFileLoaderDat', ...
sprintf('prevEndTime: %s:', datestr(prevEndTime, 'HH:MM:SS:FFF')));
[~,ind1] = min(abs(datenum(currFile.timestamps)-datenum(prevEndTime)));
closestStartTime = currFile.timestamps(ind1,:);
obj.L.debug('ACFileLoaderDat', ...
sprintf('closest start time: %s', datestr(closestStartTime, 'HH:MM:SS:FFF')));
obj.L.debug('ACFileLoaderDat', ...
sprintf('length of timestamps: %u', length(currFile.timestamps)));
% let's try just blanking
currFile.timestamps = currFile.timestamps(ind1+1:end, :);
obj.L.debug('ACFileLoaderDat', ...
sprintf('adjusted timestamps length: %u', length(currFile.timestamps)));
% BLANK C DATA
currFile.cRawDataMatrix = currFile.cRawDataMatrix(ind1+1:end, :);
obj.L.debug('ACFileLoaderDat', ...
sprintf('original A data size: %u x %u', size(currFile.aRawDataMatrix)));
% BLANK A DATA
currFile.aRawDataMatrix = currFile.aRawDataMatrix(ind1+1:end, :);
obj.L.debug('ACFileLoaderDat', ...
sprintf('adjusted A data size %u x %u', size(currFile.aRawDataMatrix)));
obj.L.debug('ACFileLoaderDat', 'finished adjusting for time');
end; %#(prevFile.timestamps(end) > currFile.timestamps(end))
else
obj.L.debug('ACFileLoaderDat', 'currFile 1st ts NOT before prevFile last ts');
end; %#(prevFile.timestamps(end) > currFile.timestamps(1))
else
obj.L.debug('ACFileLoaderDat', 'First file');
end; % end check if not first file
obj.L.debug('ACFileLoaderDat', 'outside of check date is greater than 2 hours');
obj.L.debug('ACFileLoaderDat','should be ok to add');
% create new ACData, constructor: ACData(nameIn, dataValuesIn, timestampsIn
% acdata = ACData('testAC', [cRawData, aRawData], timestamps);
% ------------------------------------------------------
% if a or c Data already exists
if ismember({'aRawData'}, who)
obj.L.info('ACFileLoaderDat', sprintf('adding %s', shortfilename));
%append data
aRawData.addData(['a: ' shortfilename], currFile.aRawDataMatrix, currFile.timestamps);
cRawData.addData(['c: ' shortfilename], currFile.cRawDataMatrix, currFile.timestamps);
disp('****')
iLastAddedFile = iFiles;
else % a or c Data doesn't already exist
obj.L.info('ACFileLoaderDat', sprintf('creating AC data with %s', shortfilename));
aRawData = ACData(['a: ' shortfilename], currFile.aRawDataMatrix, currFile.timestamps, obj.Units);
cRawData = ACData(['c: ' shortfilename], currFile.cRawDataMatrix, currFile.timestamps, obj.Units);
iLastAddedFile = iFiles;
end % end check for existing data object
end % end for loop
% create a cell array to hold both
dataOut = {aRawData, cRawData};
end % end loadData function
end % end methods block
end % end classDef