-
Notifications
You must be signed in to change notification settings - Fork 0
/
riv2xls.m
286 lines (264 loc) · 10.9 KB
/
riv2xls.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
% Riv2xls
% By Ethan Kyzivat and Ted Langhorst, with suggestions from Wayana Dolan
% and Lincoln Pitcher
% August 2018
% Written on a moving boat in the Peace-Athabasca Delta
% A script to pull data and metadata from a RiverSurveyorLive Matlab
% export and write them to an excel file to be used for manual inspection.
% This file contains more info than the default summary file output from
% River Surveryor Live (RSL). Used to semi-automatically perform
% quality control on bathymetry and discharge data. Prompts user input to select
% directory (typically named by the day's date, or following day's date
% if duration exceeds midnight UTC). It is necessary to export the
% day's files from RSL (ctrl + t, Matlab export all). If you set
% usesummfile equal to 1, then you must export the summary file from
% RS. (ctrl + s), make sure all are highlighed red (default), and save
% as ascii. The file name doesn't matter, but the extension must be
% .dis. Output QC file appears in this directory. One user parameter:
% usesummfile. Set to 0 zero if the script is having problems parsing
% the .dis file (the output spreadsheet will have less pre-populated
% fields).
%
% Additional, detailed instructions can be found in the attached
% spreadsheet "HowToUse.xlsx" and a sample of a filled out QA/QC
% spreadsheet can be found in "Example_filled_out.xlsx." Sample .mat
% and .riv files are in the sample_data folder.
% Version History
% Version 10 intelligently decides whethr or not to correct for the
% time firmware glitch, based on whether the actual date is August 12
% or earlier. Also has an error message if number of lines in .dis
% file are different from number of .mat files
% Version 9 ensures that each line of the .dis files is matched to
% proper file name AND can parse more thabn one .dis files, so no
% manual splicing is needed! Also fixes bug in end date (introduced in
% version 8).
% Version 8 corrects for firmware glitch that gives improper date.
% Also doesn't report mean veloc unless using the .dis file
% Version 7 saves lat/long, adds error message for improper loading.
% Version 6 fixes boat:water ratio inversion; only looks at HDOP and
% GPS_quality within transect, not at edges; stops auto-populating
% Track reference field (since these values are not in .mat file)
clear
close all
%% USER PARAMS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
usesummfile=1; % switch to 0 if no summ file
use_time_offset=0; % for old Sontek glitch that applied an erronous date/time offset
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
startDir=pwd;
FieldNames= {'Filename', 'Measurement_type', 'Location', 'Measurement_number',...
'Comments', 'Party', 'Boat_motor', 'Date', 'Start_time',...
'End_time', 'Time_zone', 'Transducer_depth', 'Mag_declination',...
'Track_ref', 'Start_bank', 'GPS_quality', 'HDOP', 'Voltage',...
'L_edge_dist', 'R_edge_dist', 'L_edge_Qual','R_edge_Qual','Edge_notes', 'Depth_ref_invalid',...
'Depth_notes', 'Depth_reference', 'Track_ref_invalid', 'Veloc_vector',...
'Veloc_SNR', 'Width', 'Area', 'Veloc_avg', 'Boat_veloc_avg','Boat_water_ratio',...
'Per_Measured', 'Keep_or_remove',...
'Final_notes', 'Quality', 'Latitude', 'Longitude', 'Q',...
'Mat_export'};
% fieldNames = {"File_Name"}
%%
disp('Good day! Select working directory.')
workingFolder=uigetdir;
cd(workingFolder)
k = dir('*.mat');
if isempty(k)
disp('You need to export the .mat files from RiverSurveyorLive, you idiot.')
return
end
% filter out ########r.mat files from search- they seem to be
% duplicates
k_keep=~contains({k.name}, 'r');
k=k(k_keep);
for i = 1:numel(k)
load(k(i).folder + "\"+ k(i).name);
sample=1:length(System.Sample);
val{i,1} = [k(i).name(1:end-4)];
val{i,2} ='';
val{i,3} = SiteInfo.Site_Name;
val{i,4} = SiteInfo.Meas_Number'';
val{i,5} = SiteInfo.Comments;
val{i,6} = SiteInfo.Party;
val{i,7} = SiteInfo.Boat_Motor;
val{i,8} = datetime(k(i).name(1:8), 'InputFormat', 'yyyyMMdd');%date
% val{i,6} =datetime(GPS.Utc(1),'ConvertFrom','excel',...
% 'TimeZone', 'America/Chicago', 'Format','HH:mm:ss') ;
% val{i,7} =datetime(GPS.Utc(end),'ConvertFrom','excel',...
% 'TimeZone', 'America/Chicago', 'Format','HH:mm:ss') ;
if str2double(char(datetime(val{i,8}, 'Format', 'yyyyMMdd'))) < 20180803
time_offset=37.75;
use_time_offset=1;
else time_offset=0; % 20180812 was first day of time glitch
end
val{i,9} = char(datetime(RawGPSData.GgaTimeStamp(1), 'ConvertFrom', 'epochtime', 'Epoch','2000-01-01',...
'Format','HH:mm:ss')+minutes(time_offset));
val{i,10} = char(datetime(RawGPSData.GgaTimeStamp(end,1), 'ConvertFrom', 'epochtime', 'Epoch','2000-01-01',...
'Format','HH:mm:ss')+minutes(time_offset));
val{i,11} = -6;
val{i,12} = round(Setup.sensorDepth,2);
val{i,13} = round(Setup.magneticDeclination, 2);
val{i,14} = mode(Summary.Track_Reference);
if val{i,14}==2
val{i,14}='GPS-GGA';
elseif val{i,14}==3
val{i,14}='GPS-VTG';
elseif val{i,14}==1
val{i,14}='BT';
else warning('Unexpected track reference code.')
end
val{i,15} = Setup.startEdge;
if val{i,15}==0
val{i,15}='Left';
elseif val{i,15}==1
val{i,15}='Right';
end
trans=sample(System.Step==3); %transect samples (not edges)
val{i,16} = min(GPS.GPS_Quality(trans));
if max(GPS.HDOP(trans))<2
val{i,17} = 1;
else
val{i,17} = 0;
end
val{i,18} = round(System.Voltage(end),2);
if Setup.startEdge==0
val{i,19} = Setup.Edges_0__DistanceToBank;
val{i,20} = Setup.Edges_1__DistanceToBank;
elseif Setup.startEdge==1
val{i,19} = Setup.Edges_1__DistanceToBank;
val{i,20} = Setup.Edges_0__DistanceToBank;
else
warning('Unexpected start edge code.')
end
% System.Sample=System.Sample(2:end);
edge1=sample(System.Step==2);
edge2=sample(System.Step==4);
edge1cells=Summary.Cells(edge1);
edge2cells=Summary.Cells(edge2);
if Setup.startEdge==0
if length(edge1)>=10 & min(edge1cells)>=2
val{i,21} = 1; %edge quality
else val{i,21} = 0;
end
if length(edge2)>=10 & min(edge2cells)>=2
val{i,22} = 1;
else val{i,22} = 0;
end
elseif Setup.startEdge==1
if length(edge1)>=10 & min(edge1cells)>=2
val{i,22} = 1;
else val{i,22} = 0;
end
if length(edge2)>=10 & min(edge2cells)>=2
val{i,21} = 1;
else val{i,21} = 0;
end
else
warning('Unexpected code.')
end
val{i,23} = ''; % edge notes
val{i,24} = ''; % depth-ref-invalid
val{i,25} = '';% depth notes
val{i,26} = Setup.depthReference;% depth ref
if val{i,26}==0
val{i,26}='VB';
elseif val{i,26}==1
val{i,26}='BT';
else warning('Unexpected depth reference code.')
end
val{i,27} = ''; %track ref invalid
val{i,28} = ''; %Veloc vector
val{i,29} = ''; %velocity_SNR
val{i,30} = ''; %width?
val{i,31} = Summary.Area;
val{i,32} = ''; %mean(Summary.Mean_Vel(:,1)); % need to be more precise
val{i,33} = '';
val{i,34} = '';
val{i,35} = ''; % % measured
val{i,36} = ''; %keep or remove
val{i,37} = ''; %remove notes
val{i,38} = ''; %quality
val{i,39} = mean(GPS.Latitude); %lat
val{i,40} = mean(GPS.Longitude); % long
val{i,41} = Summary.Total_Q(end);
val{i,42} = '';%Mat export
% val{i,33} =
% val{i,34} =
% val{i,35} =
% val{i,36} =
% val{i,37} =
% val{i,38} =
end
if use_time_offset==1
disp('Time offset applied to correct for firmware time glitch.')
end
%% load from .summ file
if usesummfile
summfile=cellstr(ls('*.dis'));
fprintf('%u .dis files found.\n', length(summfile))
if isempty(summfile{:})
disp('You need to export the .dis files from RiverSurveyorLive, you idiot.')
disp('ctrl-s and export to ascci. OR: make sure only one .dis in directory.')
disp('You can give it any name.')
disp('OR: set usesummfile=0 at beginning of script.')
return
end
for n=1:length(summfile) % number of .dis files
fid=fopen(summfile{n}, 'r');
summ=textscan(fid, '%s', 'Delimiter', '\n');
summheadings=textscan(summ{:}{54}, '%s', 'Delimiter', '\t');
summheadings=summheadings{:};
fclose(fid);
summ={summ{:}{55:end-19}};
j=1; % counter variable
for m=1:length(summ) % filter out lines not corresponding to data
if isempty(str2num(summ{m}(1))) % if this is the MEAN, STD, or COV line, delete it
else
summ2{j}=summ{m};
j=j+1;
end
end
summ=summ2;
fprintf('Number of entries in .dis file number %u: %u\n', n, length(summ))
fprintf('Number of .mat files: %u\n', numel(k))
if length(summ)~=numel(k)
disp('The length of your .dis file is not equal to the number of')
disp('.mat files...Try adding or removing .mat files or turn off ')
disp('the ''usesummfile'' parameter ')
fprintf('.Dis length: %u\n', length(summ))
fprintf('.mat files: %u\n', numel(k))
return
end
for i= 1:length(summ) % adding the -2 excludes the mean and std summary lines
summp{i}=textscan(summ{i}, '%s', 'Delimiter', '\t');
summp{i}=summp{i}{:};
end
for i=1:length(summ)
summfilename= summp{i}{2}(1:end-4); % name as reported in .dis file
j_index=contains({val{:,1}}, summfilename);
j=find(j_index);
val{j,30} =summp{i}{10}; % width
val{j,33} = summp{i}{12}; %boat speed
val{j,32} = summp{i}{13}; %updated mean speed
val{j,34} = str2double(val{j,33})./str2double(val{j,32}); %ratio
val{j,35} = summp{i}{21}; % % measured
end
end
end
%% make table for export
val_table=cell2table(val, 'VariableNames', FieldNames);
%xlswrite(fileName,[fieldNames; name])]
%% format and save excel file
% strings=textscan(k(1).folder, '%s', 'Delimiter', '\');
% datestr=strings{1}{end};
datestr=char(datetime(val{1,8}, 'Format', 'yyyyMMdd'));
fileName = [workingFolder,'\QC_', datestr,'.xlsx'];
Continue=1;
if exist(fileName)==2
Continue=input('QC file in this folder already found. Overwrite? (1/0) ');
end
if Continue==1
writetable(val_table, fileName);
fprintf('QC table written: %s\n', fileName);
else
disp('No files written.')
end
cd(startDir)