/
teaseisio.jl
2372 lines (2038 loc) · 86.7 KB
/
teaseisio.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
mutable struct JSeis{T<:NamedTuple,U<:NamedTuple,C<:Union{TraceCompressor{Float32}, TraceCompressor{Int16}}}
filename::String
mode::String
description::String
mapped::Bool
datatype::String
dataformat::DataType
dataorder::String
properties::T
axis_lengths::Array{Int,1}
axis_propdefs::U
axis_units::Array{String,1}
axis_domains::Array{String,1}
axis_lstarts::Array{Int,1}
axis_lincs::Array{Int,1}
axis_pstarts::Array{Float64,1}
axis_pincs::Array{Float64,1}
dataproperties::Array{DataProperty,1}
geom::Union{Geometry,Nothing}
hastraces::Bool
secondaries::Array{String, 1}
trcextents::Array{Extent,1}
hdrextents::Array{Extent,1}
currentvolume::Int
map::Array{Int32,1}
hdrlength::Int
compressor::C
function JSeis(properties::T, axis_propdefs::U, compressor::C) where {T,U,C}
io=new{T,U,C}()
io.properties=properties
io.axis_propdefs=axis_propdefs
io.compressor = compressor
io
end
end
# open/close
"""
jsopen(filename, mode, [parameters])
Open a new or existing JavaSeis dataset with name `filename::String` and in `mode::String`.
`mode` can be one of `"r"` (read), `"w"` (write/create) or `"r+"` (read and write).
It is convention for filename to havea ".js" extention.
If `"w"` is used for the value of `mode`, then the `axis_lengths` named parameter is required, and several optional
named function parameters are available:
# parameters
* `similarto::String` An existing JavaSeis dataset. If set, then all other named arguments can be used to modify the data context that belongs to the existing JavaSeis dataset.
* `description::String` Description of dataset, if not set, then a description is parsed from the filename.
* `mapped::Bool` If the dataset is full (no missing frames/traces), then it may be more efficient to set this to `false`. Defaults to `true`.
* `datatype::String` Examples are `CMP`, `SHOT`, etc. If not set, then `UNKNOWN` is used.
* `dataformat::Type` Choose from `Float32`, and `Int16`. If not set, then `Float32` is used.
* `dataorder::String` (not supported)
* `axis_lengths::Array{Int}` size of each dimension (sample/trace/frame/volume/hypercube) of the JavaSeis data context
* `axis_propdefs::Array{TracePropertyDef}` Trace properties corresponding to JavaSeis axes. If not set, then `SAMPLE`, `TRACE`, `FRAME`, `VOLUME` and `HYPRCUBE` are used.
* `axis_units::Array{String}` Units corresponding to JavaSeis axes. e.g. `SECONDS`, `METERS`, etc. If not set, then `UNKNOWN` is used.
* `axis_domains::Array{String}` Domains corresponding to JavaSeis axes. e.g. `SPACE`, `TIME`, etc. If not set, then `UNKNOWN` is used.
* `axis_lstarts::Array{Int}` Logical origins for each axis. If not set, then `1` is used for the logical origin of each axis.
* `axis_lincs::Array{Int}` Logical increments for each axis. If not set, then `1` is used for the logical increments of each axis.
* `axis_pstarts::Array{Float64}` Physical origins for each axis. If not set, then `0.0` is used for the physical origin of each axis.
* `axis_pincs::Array{Float64}` Physical increments for each axis. If not set, then `1.0` is used for the physical increments of each axis.
* `data_properties::Array{DataProperty}` An array of custom trace properties. These are in addition to the properties listed in `SSPROPS.md`.
* `properties::Array{TracePropertyDef}` An array of custom data properties. One property per data-set rather than one property per trace as in `properties` above.
* `geometry::Geometry` An optional three point geometry can be embedded in the JavaSeis file.
* `secondaries::Array{String}` An array of file-system locations used to store the file extents. If not set, then *primary* storage is used.
* `nextents::Int64` The number of file-extents used to store the data. If not set, then a heuristic is used to choose the number of extents. The heuristic is: min(256,10 + (FRAMEWORK_SIZE)/(2*1024^3)).
* `properties_add::Array{TracePropertyDef}` When `similarto` is specified, use this to add trace properties to those already existing in the `similarto` file.
* `properties_rm::Array{TracePropertyDef}` When `similarto` is specified, use this to remove trace properties to those already existing in the `similarto` file.
* `dataproperties_add::Array{DataProperty}` When `similarto` is specfied, use this to add dataset properties to those aloready existing in the `similarto` file.
* `dataproperties_rm::Array{DataProperty}` When `similarto` is specified, use this to remove dataset properties to those already existing in the `similarto` file.
"""
function jsopen(
filename::String, mode::String;
description = "", # JavaSeis file description
mapped = nothing, # ismapped, must be true for irregular (sparse) data
datatype = nothing, # datatype, stockdatatype[:CUSTOM], stockdatatype[:SOURCE], etc.
dataformat = nothing, # format stored on disk, Float32 or Float16
dataorder = "", # big ("BIG_ENDIAN") or little ("LITTLE_ENDIAN") endian stored on disk
axis_lengths = Array{Int}(undef, 0), # length (number of bins) along each axis
axis_propdefs = Array{TracePropertyDef}(undef, 0), # axis properties (use for bin header word along each dimension)
axis_units = Array{String}(undef, 0), # axis units (stockunit[:SECONDS], stockunit[:METERS], stockunit[:UNKNOWN] etc.)
axis_domains = Array{String}(undef, 0), # axis domains (stockdomain[:SPACE], stockdomain[:TIME], stockdomain[:UNKNOWN], etc.)
axis_lstarts = Array{Int}(undef, 0), # logical start index for each axis
axis_lincs = Array{Int}(undef, 0), # logical increment between bins for each axis
axis_pstarts = Array{Float64}(undef, 0), # physical start for each axis
axis_pincs = Array{Float64}(undef, 0), # physical increment between bins for each axis
dataproperties = Array{DataProperty}(undef, 0), # add global data properties
properties = Array{TracePropertyDef}(undef, 0), # add headers to the standard set
geometry = nothing, # 3 point geometry
secondaries = nothing, # secondary file-system locations for storing trace and header data, defaults to primary storage "."
nextents = 0, # number of file extents
similarto = "", # create a JavaSeis file similar to this one
properties_add = Array{TracePropertyDef}(undef, 0), # only used in conjunction with similarto, dis-allowed if properties is set
properties_rm = Array{TracePropertyDef}(undef, 0), # only used in conjunction with similarto, dis-allowed if properties is set
dataproperties_add = Array{DataProperty}(undef, 0), # only used in conjunction with similarto, dis-allowed if dataproperties is set
dataproperties_rm = Array{DataProperty}(undef, 0)) # only used in conjunction with similarto, dis-allowed if dataproperties is set
local traceproperties, _axis_propdefs, _axis_lengths, _dataformat, xml_fileproperties
if mode == "r" || mode == "r+"
xml_fileproperties = parse_file(joinpath(filename, "FileProperties.xml"))
traceproperties = get_trace_properties(xml_fileproperties)
_axis_propdefs = get_axis_propdefs(traceproperties, xml_fileproperties)
_axis_lengths = get_axis_lengths(xml_fileproperties)
_dataformat = get_dataformat(xml_fileproperties)
elseif mode == "w" && similarto == ""
traceproperties, _axis_propdefs = get_trace_properties(length(axis_lengths), properties, properties_add, properties_rm, axis_propdefs, similarto)
_axis_lengths = axis_lengths
_dataformat = dataformat == nothing ? Float32 : dataformat
elseif mode == "w" && similarto != ""
traceproperties, _axis_propdefs = get_trace_properties(length(axis_lengths), properties, properties_add, properties_rm, axis_propdefs, similarto)
iosim = jsopen(similarto)
_axis_lengths = length(axis_lengths) == 0 ? [size(iosim)...] : axis_lengths
_dataformat = dataformat == nothing ? iosim.dataformat : dataformat
end
compressor = TraceCompressor(_axis_lengths[1], _dataformat)
io = JSeis(traceproperties, _axis_propdefs, compressor)
io.filename = filename
io.mode = mode
io.currentvolume = -1
io.hdrlength = headerlength(io.properties)
if mode == "r" || mode == "r+"
io2 = open(joinpath(filename, "Name.properties"), "r")
io.description = get_description(io2)
close(io2)
io.mapped = get_mapped(xml_fileproperties)
io.datatype = get_datatype(xml_fileproperties)
io.dataformat = _dataformat
io.dataorder = get_dataorder(xml_fileproperties)
io.axis_lengths = _axis_lengths
io.axis_units = get_axis_units(xml_fileproperties)
io.axis_domains = get_axis_domains(xml_fileproperties)
io.axis_lstarts = get_axis_lstarts(xml_fileproperties)
io.axis_lincs = get_axis_lincs(xml_fileproperties)
io.axis_pstarts = get_axis_pstarts(xml_fileproperties)
io.axis_pincs = get_axis_pincs(xml_fileproperties)
io.dataproperties = get_dataproperties(xml_fileproperties)
io.geom = nothing
io.hastraces = false
if isfile(joinpath(filename, "Status.properties")) == true # Do not fail if Status.properties does not exist to maintain backwards compat
io2 = open(joinpath(filename, "Status.properties"))
io.hastraces = get_status(io2)
close(io2)
end
xml = parse_file(joinpath(filename, "VirtualFolders.xml"))
io.secondaries = get_secondaries(xml)
xml = parse_file(joinpath(filename, "TraceFile.xml"))
io.trcextents = get_extents(xml, io.secondaries, io.filename)
xml = parse_file(joinpath(filename, "TraceHeaders.xml"))
io.hdrextents = get_extents(xml, io.secondaries, io.filename)
io.currentvolume = -1
io.map = zeros(Int32, io.axis_lengths[3])
return io
end
if mode == "w" && isdir(filename) == true
teaseis_robust_rm(filename)
end
if mode == "w" && similarto == ""
io.mapped = mapped == nothing ? true : mapped
io.datatype = datatype == nothing ? stockdatatype[:CUSTOM] : datatype
io.dataformat = _dataformat
io.dataorder = dataorder == "" ? "LITTLE_ENDIAN" : dataorder
io.axis_lengths = axis_lengths
io.axis_units = axis_units
io.axis_domains = axis_domains
io.axis_lstarts = axis_lstarts
io.axis_lincs = axis_lincs
io.axis_pstarts = axis_pstarts
io.axis_pincs = axis_pincs
io.dataproperties = dataproperties
io.geom = geometry == nothing ? nothing : geometry
io.secondaries = secondaries == nothing ? ["."] : secondaries
elseif mode == "w" && similarto != ""
iosim = jsopen(similarto)
# special handling for data properties
if length(dataproperties) == 0
dataproperties = copy(iosim.dataproperties)
else
@assert length(dataproperties_add) == 0
@assert length(dataproperties_rm) == 0
end
if length(dataproperties_add) != 0
@assert length(dataproperties) == 0
for prop in dataproperties_add
if in(prop, dataproperties) == false
push!(dataproperties, prop)
end
end
end
io.mapped = mapped == nothing ? iosim.mapped : mapped
io.datatype = datatype == nothing ? iosim.datatype : datatype
io.dataformat = _dataformat
io.dataorder = dataorder == "" ? iosim.dataorder : dataorder
io.axis_lengths = _axis_lengths
io.axis_units = length(axis_units) == 0 ? copy(iosim.axis_units) : axis_units
io.axis_domains = length(axis_domains) == 0 ? copy(iosim.axis_domains) : axis_domains
io.axis_lstarts = length(axis_lstarts) == 0 ? copy(iosim.axis_lstarts) : axis_lstarts
io.axis_lincs = length(axis_lincs) == 0 ? copy(iosim.axis_lincs) : axis_lincs
io.axis_pstarts = length(axis_pstarts) == 0 ? copy(iosim.axis_pstarts) : axis_pstarts
io.axis_pincs = length(axis_pincs) == 0 ? copy(iosim.axis_pincs) : axis_pincs
io.dataproperties = dataproperties
io.geom = geometry == nothing ? iosim.geom : geometry
io.secondaries = secondaries == nothing ? copy(iosim.secondaries) : secondaries
nextents = nextents == 0 ? length(iosim.trcextents) : nextents
end
if mode == "w"
ndim = length(io.axis_lengths)
@assert ndim >= 3
@assert length(io.axis_propdefs) == ndim || length(io.axis_propdefs) == 0
@assert length(io.axis_units) == ndim || length(io.axis_units) == 0
@assert length(io.axis_domains) == ndim || length(io.axis_domains) == 0
@assert length(io.axis_lstarts) == ndim || length(io.axis_lstarts) == 0
@assert length(io.axis_lincs) == ndim || length(io.axis_lincs) == 0
@assert length(io.axis_pstarts) == ndim || length(io.axis_pstarts) == 0
@assert length(io.axis_pincs) == ndim || length(io.axis_pincs) == 0
return jsopen_write(io, nextents, ndim, description, properties, similarto == "" ? false : true)
end
error("mode \"$(mode)\" is not supported by jsopen.")
end
function teaseis_robust_rm(filename::AbstractString)
try
rm(jsopen(filename, "r"))
catch
rm(filename, force=true, recursive=true)
end
end
function jsopen_write(io::JSeis, nextents::Int, ndim::Int, description::String, properties::Array, issimilar::Bool)
# axes
if length(io.axis_units) == 0
io.axis_units = fill(stockunit[:UNKNOWN], ndim)
end
if length(io.axis_domains) == 0
io.axis_domains = fill(stockdomain[:UNKNOWN], ndim)
end
if length(io.axis_lstarts) == 0
io.axis_lstarts = ones(Int64, ndim)
end
if length(io.axis_lincs) == 0
io.axis_lincs = ones(Int64, ndim)
end
if length(io.axis_pstarts) == 0
io.axis_pstarts = zeros(Float64, ndim)
end
if length(io.axis_pincs) == 0
io.axis_pincs = ones(Float64, ndim)
end
# description, if not set by user, we grab it from the filename
if length(description) == 0
io.description = io.filename[end-2:end] == ".js" ? io.filename[1:end-3] : io.filename
if VERSION < v"1.1"
io.description = split(io.description, ['/','@'])[end]
else
io.description = split(splitpath(io.description)[end], '@')[end]
end
else
io.description = description
end
# data is initialized to empty
io.hastraces = false
# secondaries, if not set by user, we use primary storage ["."]
if io.secondaries == nothing
io.secondaries = ["."]
end
if length(io.secondaries) < 1
error("For primary storage use secondaries=nothing or secondaries=[\".\"], otherwise secondaries must an array of strings with length>0")
end
# choose default number of exents (heuristic)
nextents = nextents == 0 ? nextents_heuristic(io.axis_lengths, io.dataformat) : nextents
nextents = nextents > prod(io.axis_lengths[3:end]) ? prod(io.axis_lengths[3:end]) : nextents
# trace and header extents
io.trcextents = make_extents(nextents, io.secondaries, io.filename, io.axis_lengths, tracelength(io.compressor), "TraceFile")
io.hdrextents = make_extents(nextents, io.secondaries, io.filename, io.axis_lengths, io.hdrlength, "TraceHeaders")
# trace map
io.map = zeros(Int32, io.axis_lengths[3])
# create the various xml files and directories
make_primarydir(io)
make_extentdirs(io)
create_map(io)
write_fileproperties(io)
write_nameproperties(io)
write_statusproperties(io)
write_extentmanager(io)
write_virtualfolders(io)
return io
end
"""
jsopen(filename)
Equivalent to `jsopen(filename, "r")`
"""
jsopen(filename::String) = jsopen(filename, "r")
"""
jscreate(filename)
Create a JavaSeis dataset without opening it. This method has the same optional arguments as `jsopen`
"""
jscreate(filename::String; kwargs...) = close(jsopen(filename, "w"; kwargs...))
"""
close(io)
Close an open JavaSeis dataset where `io` is of type `JSeis` created using, for example, `jsopen`.
"""
function close(io::JSeis)
# nothing to do... stub in-case we decide to keep state for the various file-pointers
end
"""
rm(io)
Remove a JavaSeis dataset from disk. For example: `rm(jsopen("deleteme.js"))`
"""
function rm(io::JSeis)
rmsecondaries(io)
if isdir(io.filename) == true
rm(io.filename, recursive = true)
end
end
"""
empty!(io)
Empty a JavaSeis dataset from disk, retaining the meta-information. For example: `empty!(jsopen("emptyme.js"))`
"""
function empty!(io::JSeis)
emptysecondaries!(io)
if isdir(io.filename) == true
names = filter(s->(startswith(s, "TraceFile") == true || startswith(s, "TraceHeaders") == true) && endswith(s, ".xml") == false, readdir(io.filename))
for name in names
rm("$(io.filename)/$(name)")
end
end
io.hastraces = false
write_statusproperties(io)
end
function rmsecondaries(io::JSeis)
for secondary in io.secondaries
if secondary != "." && isdir(extentdir(secondary, io.filename)) == true
rm(extentdir(secondary, io.filename), recursive = true)
end
end
end
function emptysecondaries!(io::JSeis)
for secondary in io.secondaries
if secondary != "." && isdir(extentdir(secondary, io.filename)) == true
for name in readdir(extentdir(secondary, io.filename))
rm("$(extentdir(secondary, io.filename))/$(name)")
end
end
end
end
"""
cp(src, dst, [secondaries=nothing])
Copy a file from `src` (of type `JSeis`) to `dst` of type `String`. For example, `cp(jsopen("copyfrom.js"), "copyto.js")`.
Use the optional named argument `secondaries` to change the JavaSeis secondary location.
"""
function cp(src::JSeis, dst::AbstractString; secondaries=nothing)
iodst = jsopen(dst, "w", similarto=src.filename, secondaries=secondaries)
trcs, hdrs = allocframe(iodst)
for i = 1:length(src)
idx = ind2sub(src,i)
fld = readframe!(src, trcs, hdrs, idx...)
if fld > 0
writeframe(iodst, trcs, hdrs)
end
end
end
"""
mv(src, dst, [secondaries=nothing])
Move a file from `src` (of type `JSeis`) to `dst` of type `String`. For example, `cp(jsopen("movefrom.js"), "moveto.js")`.
Use the optional named argument `secondaries` to change the JavaSeis secondary location.
"""
function mv(src::JSeis, dst::AbstractString; secondaries=nothing)
cp(src, dst, secondaries=secondaries)
rm(src)
end
function show(io::IO, js::JSeis)
write(io, "JavaSeis file:\n");
write(io, "\tsize: $(size(js))\n");
write(io, "\ttype: $(js.datatype) ; format: $(asciidataformat(js.dataformat))\n");
write(io, "\taxis domains: $(domains(js))\n");
write(io, "\taxis units: $(units(js))\n");
write(io, "\taxis properties: $(labels(js))");
end
# reading from the various xml files:
get_mapped(xml::XMLDocument) = strip(content(get_file_property_element(xml, "Mapped"))) == "true" ? true : false
get_datatype(xml::XMLDocument) = strip(content(get_file_property_element(xml, "DataType")))
get_axis_lengths(xml::XMLDocument) = [parse(Int64, s) for s in split(content(get_file_property_element(xml, "AxisLengths")))]
get_axis_units(xml::XMLDocument) = split(content(get_file_property_element(xml, "AxisUnits")))
get_axis_domains(xml::XMLDocument) = split(content(get_file_property_element(xml, "AxisDomains")))
get_axis_lstarts(xml::XMLDocument) = [parse(Int64, s) for s in split(content(get_file_property_element(xml, "LogicalOrigins")))]
get_axis_lincs(xml::XMLDocument) = [parse(Int64, s) for s in split(content(get_file_property_element(xml, "LogicalDeltas")))]
get_axis_pstarts(xml::XMLDocument) = [parse(Float64, s) for s in split(content(get_file_property_element(xml, "PhysicalOrigins")))]
get_axis_pincs(xml::XMLDocument) = [parse(Float64, s) for s in split(content(get_file_property_element(xml, "PhysicalDeltas")))]
get_dataorder(xml::XMLDocument) = strip(content(get_file_property_element(xml, "ByteOrder")))
function get_axis_propdefs(properties, xml::XMLDocument)
labels = split(content(get_file_property_element(xml, "AxisLabels")))
propdefs = Array{TracePropertyDef}(undef, 0)
for (i,label) in enumerate(labels)
push!(propdefs, get_axis_propdef(properties, label, i))
end
names = ntuple(i->Symbol(propdefs[i].label), length(propdefs))
NamedTuple{names}(propdefs)
end
function get_axis_propdef(properties, label::AbstractString, dim::Int)
# map from JavaSeis axis name to ProMax property label
plabel = haskey(dictJStoPM, label) == true ? dictJStoPM[label] : label
for prop in properties
if prop.def.label == plabel
return prop.def
end
end
# The sample and trace labels do not need a corresponding trace property.
# Therefore, these should be considered valid datasets.
if dim == 1 || dim == 2
return TracePropertyDef(label, label, Int32, 1)
end
error("Malformed JavaSeis: axis props, axis label=$(label) has no corresponding trace property.")
end
function get_dataformat(xml::XMLDocument)
par = get_file_property_element(xml, "TraceFormat")
format = strip(content(par))
if format == "FLOAT"
return Float32
elseif format == "DOUBLE"
return Float64
elseif format == "COMPRESSED_INT32"
return Int32
elseif format == "COMPRESSED_INT16"
return Int16
end
error("Unrecognized JavaSeis data format")
end
function get_trace_properties(xml::XMLDocument)
props = Array{TraceProperty}(undef, 0)
for parset in child_elements(root(xml))
if attribute(parset, "name") == "TraceProperties"
for parset2 in child_elements(parset)
label, description, format, count, offset = "", "", "", 1, 0
for par in child_elements(parset2)
if attribute(par, "name") == "label"
label = strip(content(par))
elseif attribute(par, "name") == "description"
description = strip(content(par), [' ', '"'])
elseif attribute(par, "name") == "format"
format = strip(content(par))
elseif attribute(par, "name") == "elementCount"
count = parse(Int32, content(par))
elseif attribute(par, "name") == "byteOffset"
offset = parse(Int32, content(par))
end
end
push!(props, TraceProperty(TracePropertyDef(label, description, stringtype2type(format, count), count), offset))
end
break
end
end
names = ntuple(i->Symbol(props[i].def.label), length(props))
return NamedTuple{names}(props)
end
function get_trace_properties(ndim, propertydefs, propertydefs_add, propertydefs_rm, axis_propdefs, similarto)
local _propertydefs, _axis_propdefs
if similarto == ""
_propertydefs = propertydefs
_axis_propdefs = axis_propdefs
else
iosim = jsopen(similarto, "r")
# special handling for trace properties
if length(propertydefs) == 0
_propertydefs = propdef.(collect(iosim.properties))
else
@assert length(propertydefs_add) == 0
@assert length(propertydefs_rm) == 0
end
if length(propertydefs_add) != 0
for pdef in propertydefs_add
if in(pdef, _propertydefs) == false
push!(_propertydefs, pdef)
end
end
end
if length(propertydefs_rm) != 0
N = length(_propertydefs) - length(propertydefs_rm)
propertydefs_new = Array{TracePropertyDef}(N)
k = 1
for i = 1:length(_propertydefs), j = 1:length(propertydefs_rm)
if propertydefs_rm[j].lbl == _propertydefs[i].lbl
break
end
propertydefs_new[k] = _propertydefs[i]
k += 1
end
_propertydefs = propertydefs_new
end
_axis_propdefs = length(axis_propdefs) == 0 ? collect(iosim.axis_propdefs) : axis_propdefs
end
# initialize trace properties to an empty array
properties = Array{TraceProperty}(undef, 0)
# trace properties, minimal set (as defined by SeisSpace / ProMAX)
byteoffset = similarto == "" ? sspropset!(properties, 0) : 0
# trace properties, user defined
for pdef in _propertydefs
if in(pdef, properties) == false
push!(properties, TraceProperty(pdef, byteoffset))
byteoffset += sizeof(pdef)
end
end
# axis properties
if length(_axis_propdefs) == 0
_axis_propdefs = [stockprop[:SAMPLE], stockprop[:TRACE], stockprop[:FRAME], stockprop[:VOLUME], stockprop[:HYPRCUBE]][1:min(5,ndim)]
map(idim->push!(_axis_propdefs, TracePropertyDef("DIM$(idim)", "dimension $(idim)", Int32, 1)), 6:ndim)
end
for (i,pdef) in enumerate(_axis_propdefs)
@assert pdef.elementcount == 1
# map from JavaSeis axis name to ProMax property label
pdef = haskey(dictJStoPM, pdef.label) == true ? TracePropertyDef(dictJStoPM[pdef.label], pdef.description, pdef.format, pdef.elementcount) : pdef
_axis_propdefs[i] = pdef
if in(pdef, properties) == false
push!(properties, TraceProperty(pdef, byteoffset))
byteoffset += sizeof(pdef)
end
end
propsymbols = ntuple(i->Symbol(properties[i].def.label), length(properties))
axissymbols = ntuple(i->Symbol(_axis_propdefs[i].label), length(_axis_propdefs))
NamedTuple{propsymbols}(properties),NamedTuple{axissymbols}(_axis_propdefs)
end
function get_dataproperties(xml::XMLDocument)
dataprops = Array{DataProperty}(undef, 0)
for parset in child_elements(root(xml))
if attribute(parset, "name") == "CustomProperties"
for par in child_elements(parset)
name = attribute(par, "name")
# complicated custom props, see below
if name != "Geometry" && name != "FieldInstruments" && name != "extendedParmTable"
format = attribute(par, "type")
value = content(par)
push!(dataprops, DataProperty(name, format, value))
end
end
end
end
return dataprops
end
function get_geom(xml::XMLDocument)
for parset in child_elements(root(xml))
if attribute(parset, "name") == "CustomProperties"
for parset2 in child_elements(parset)
if attribute(parset2, "name") == "Geometry"
try
g = Dict()
for par in child_elements(parset2)
parname = attribute(par, "name")
if in(parname, ("u1","un","v1","vn","w1","wn"))
g[parname] = parse(Int,content(par))
else
g[parname] = parse(Float64,content(par))
end
end
return Geometry(
g["u1"],g["un"],g["v1"],g["vn"],g["w1"],g["wn"],
g["ox"],g["oy"],g["oz"],
g["ux"],g["uy"],g["uz"],
g["vx"],g["vy"],g["vz"],
g["wx"],g["wy"],g["wz"])
catch
@warn "Corrupt geometry information"
return nothing
end
end
end
end
end
return nothing
end
function get_description(io::IOStream)
for ln in eachline(io)
if ln[1] != '#'
x = split(ln, '=')
if x[1] == "DescriptiveName"
return strip(x[2])
end
end
end
return " "
end
function get_status(io::IOStream)
for ln in eachline(io)
if ln[1] != '#'
x = split(ln, '=')
if length(x) < 2
@warn "Corrupt status information. Status information, i.e. \"has traces\", may be incorrect."
return false
end
if chomp(x[2]) == "true"
return true
end
end
end
return false
end
function get_secondaries(xml::XMLDocument)
n = get_nsecondaries(xml)
secondaries = Array{String}(undef, n)
for i=1:n
secondaries[i] = get_secondary(xml, i)
end
return secondaries
end
function get_nsecondaries(xml::XMLDocument)
for par in child_elements(root(xml))
if attribute(par, "name") == "NDIR"
return parse(Int, content(par))
end
end
end
function get_secondary(xml, i)
for par in child_elements(root(xml))
if attribute(par, "name") == "FILESYSTEM-$(i-1)"
path = strip(split(content(par),',')[1])
return path
end
end
end
get_nextents(xml::XMLDocument) = parse(Int32, strip(content(get_extent_manager_element(xml, "VFIO_MAXFILE"))))
get_extentname(xml::XMLDocument) = strip(content(get_extent_manager_element(xml, "VFIO_EXTNAME")))
get_extentsize(xml::XMLDocument) = parse(Int64, strip(content(get_extent_manager_element(xml, "VFIO_EXTSIZE"))))
get_extentmaxpos(xml::XMLDocument) = parse(Int64, strip(content(get_extent_manager_element(xml, "VFIO_MAXPOS"))))
function get_file_property_element(xml::XMLDocument, name::String)
for parset in child_elements(root(xml))
if attribute(parset, "name") == "FileProperties"
for par in child_elements(parset)
if attribute(par, "name") == name
return par
end
end
end
end
error("Malformed JavaSeis.")
end
function get_extent_manager_element(xml::XMLDocument, name::String)
for par in child_elements(root(xml))
if attribute(par, "name") == name
return par
end
end
error("Malformed JavaSeis.")
end
# write xml files ...
function make_primarydir(io::JSeis)
if isdir(io.filename)
rm(io.filename, recursive = true)
end
mkpath(io.filename)
end
function make_extentdirs(io::JSeis)
for file in io.secondaries
if isdir(extentdir(file, io.filename))
rm(extentdir(file, io.filename), recursive = true)
end
mkpath(extentdir(file, io.filename))
end
end
function delete_first_line(filename::AbstractString)
io = open(filename)
lines = readlines(io,keep=true)
close(io)
io = open(filename, "w")
for i = 2:length(lines)
write(io, lines[i])
end
close(io)
end
function write_fileproperties(io::JSeis)
xdoc = XMLDocument()
# preamble
jsmetadata = create_root(xdoc, "parset")
set_attribute(jsmetadata, "name", "JavaSeis Metadata")
fileproperties = new_child(jsmetadata, "parset")
set_attribute(fileproperties, "name", "FileProperties")
# translate ProMax property labels to JavaSeis axis labels
axislabels = proplabel.(collect(io.axis_propdefs))
for (i,lbl) in enumerate(axislabels)
axislabels[i] = haskey(dictPMtoJS, lbl) == true ? dictPMtoJS[lbl] : lbl
end
# file properties
write_parproperty(fileproperties, "Comments", "string", " \"JavaSeis.jl - JavaSeis File Propertties 2006.3\" ")
write_parproperty(fileproperties, "JavaSeisVersion", "string", " 2006.3 ")
write_parproperty(fileproperties, "DataType", "string", " $(io.datatype) ")
write_parproperty(fileproperties, "TraceFormat", "string", " $(asciidataformat(io.dataformat)) ")
write_parproperty(fileproperties, "ByteOrder", "string", " $(io.dataorder) ")
write_parproperty(fileproperties, "Mapped", "boolean", " $(io.mapped == true ? "true" : "false") ")
write_parproperty(fileproperties, "DataDimensions", "int", " $(length(io.axis_lengths)) ")
write_parproperty(fileproperties, "AxisLabels", "string", formataxes(axislabels))
write_parproperty(fileproperties, "AxisUnits", "string", formataxes(io.axis_units))
write_parproperty(fileproperties, "AxisDomains", "string", formataxes(io.axis_domains))
write_parproperty(fileproperties, "AxisLengths", "long", formataxes(io.axis_lengths))
write_parproperty(fileproperties, "LogicalOrigins", "long", formataxes(io.axis_lstarts))
write_parproperty(fileproperties, "LogicalDeltas", "long", formataxes(io.axis_lincs))
write_parproperty(fileproperties, "PhysicalOrigins", "double", formataxes(io.axis_pstarts))
write_parproperty(fileproperties, "PhysicalDeltas", "double", formataxes(io.axis_pincs))
write_parproperty(fileproperties, "HeaderLengthBytes", "int", " $(headerlength(io)) ")
# trace properties
traceproperties = new_child(jsmetadata, "parset")
set_attribute(traceproperties, "name", "TraceProperties")
i = 0
for property in io.properties
write_traceproperty(traceproperties, i, property)
i += 1
end
# custom properties
customproperties = new_child(jsmetadata, "parset")
set_attribute(customproperties, "name", "CustomProperties")
for dataprop in io.dataproperties
write_parproperty(customproperties, dataprop.label, propertyformatstring(dataprop), " $(convert(dataprop.format, dataprop.value)) ")
end
# 3-point geometry
if io.geom != nothing
geometry = new_child(customproperties, "parset")
set_attribute(geometry, "name", "Geometry")
write_parproperty(geometry, "u1", "long", " $(io.geom.u1) ")
write_parproperty(geometry, "un", "long", " $(io.geom.un) ")
write_parproperty(geometry, "v1", "long", " $(io.geom.v1) ")
write_parproperty(geometry, "vn", "long", " $(io.geom.vn) ")
write_parproperty(geometry, "w1", "long", " $(io.geom.w1) ")
write_parproperty(geometry, "wn", "long", " $(io.geom.wn) ")
write_parproperty(geometry, "ox", "double", " $(io.geom.ox) ")
write_parproperty(geometry, "oy", "double", " $(io.geom.oy) ")
write_parproperty(geometry, "oz", "double", " $(io.geom.oz) ")
write_parproperty(geometry, "ux", "double", " $(io.geom.ux) ")
write_parproperty(geometry, "uy", "double", " $(io.geom.uy) ")
write_parproperty(geometry, "uz", "double", " $(io.geom.uz) ")
write_parproperty(geometry, "vx", "double", " $(io.geom.vx) ")
write_parproperty(geometry, "vy", "double", " $(io.geom.vy) ")
write_parproperty(geometry, "vz", "double", " $(io.geom.vz) ")
write_parproperty(geometry, "wx", "double", " $(io.geom.wx) ")
write_parproperty(geometry, "wy", "double", " $(io.geom.wy) ")
write_parproperty(geometry, "wz", "double", " $(io.geom.wz) ")
end
# Write data
save_file(xdoc, "$(io.filename)/FileProperties.xml")
# Delete the first line from the file <?xml verion="1.0...
delete_first_line("$(io.filename)/FileProperties.xml")
end
function write_traceproperty(parent::XMLElement, i::Int, property::TraceProperty)
header = new_child(parent, "parset")
set_attribute(header, "name", "entry_$(i)")
write_parproperty(header, "label", "string", " $(property.def.label) ")
write_parproperty(header, "description", "string", " \"$(property.def.description)\" ")
write_parproperty(header, "format", "string", " $(propertyformatstring(property)) ")
write_parproperty(header, "elementCount", "int", " $(property.def.elementcount) ")
write_parproperty(header, "byteOffset", "int", " $(property.byteoffset) ")
end
function formataxes(items::Array)
labels = "\n"
for label in items
labels = "$(labels) $(label)\n"
end
labels = "$(labels) "
end
function asciidataformat(dataformat::Type)
if dataformat == Float32
return "FLOAT"
elseif dataformat == Float64
return "DOUBLE"
elseif dataformat == Int16
return "COMPRESSED_INT16"
end
error("unsupported data format")
end
function write_nameproperties(io::JSeis)
ioname = open(joinpath(io.filename,"Name.properties"), "w")
write(ioname,
"#JavaSeis.jl - JavaSeis File Properties 2006.3
#$(datestamp())
DescriptiveName=$(io.description)")
close(ioname)
end
function write_statusproperties(io::JSeis)
iostatus = open(joinpath(io.filename, "Status.properties"), "w")
write(iostatus,
"#JavaSeis.jl - JavaSeis File Properties 2006.3
#$(datestamp())
HasTraces=$(io.hastraces == true ? "true" : "false")")
close(iostatus)
end
function datestamp()
date = Dates.now()
year = Dates.year(date)
mon = Dates.monthabbr(date)
wday = Dates.dayabbr(date)
day = Dates.dayofmonth(date)
hour = Dates.hour(date) < 10 ? "0$(Dates.hour(date))" : Dates.hour(date)
min = Dates.minute(date) < 10 ? "0$(Dates.minute(date))" : Dates.minute(date)
sec = Dates.second(date) < 10 ? "0$(Dates.second(date))" : Dates.second(date)
return "$(wday) $(mon) $(day) $(hour):$(min):$(sec) $(year)"
end
function write_extentmanager(io::JSeis)
xdoc = XMLDocument()
extentman = create_root(xdoc, "parset")
set_attribute(extentman, "name", "ExtentManager")
write_parproperty(extentman, "VFIO_VERSION", "string", " 2006.2 ")
write_parproperty(extentman, "VFIO_EXTSIZE", "long", " $(io.trcextents[1].size) ")
write_parproperty(extentman, "VFIO_MAXFILE", "int", " $(length(io.trcextents)) ")
write_parproperty(extentman, "VFIO_MAXPOS", "long", " $(prod(io.axis_lengths[2:end]) * tracelength(io) - 1) ")
write_parproperty(extentman, "VFIO_EXTNAME", "string", " TraceFile ")
write_parproperty(extentman, "VFIO_POLICY", "string", " RANDOM ")
save_file(xdoc, joinpath(io.filename, "TraceFile.xml"))
delete_first_line(joinpath(io.filename, "TraceFile.xml"))
xdoc = XMLDocument()
extentman = create_root(xdoc, "parset")
set_attribute(extentman, "name", "ExtentManager")
write_parproperty(extentman, "VFIO_VERSION", "string", " 2006.2 ")
write_parproperty(extentman, "VFIO_EXTSIZE", "long", " $(io.hdrextents[1].size) ")
write_parproperty(extentman, "VFIO_MAXFILE", "int", " $(length(io.hdrextents)) ")
write_parproperty(extentman, "VFIO_MAXPOS", "long", " $(prod(io.axis_lengths[2:end]) * headerlength(io) - 1) ")
write_parproperty(extentman, "VFIO_EXTNAME", "string", " TraceHeaders ")
write_parproperty(extentman, "VFIO_POLICY", "string", " RANDOM ")
save_file(xdoc, joinpath(io.filename, "TraceHeaders.xml"))
delete_first_line(joinpath(io.filename, "TraceHeaders.xml"))
end
function write_virtualfolders(io::JSeis)
xdoc = XMLDocument()
virtman = create_root(xdoc, "parset")
set_attribute(virtman, "name", "VirtualFolders")
write_parproperty(virtman, "NDIR", "int", " $(length(io.secondaries)) ")
for i = 1:length(io.secondaries)
write_parproperty(virtman, "FILESYSTEM-$(i-1)", "string", " $(io.secondaries[i]),READ_WRITE ")
end
write_parproperty(virtman, "Version", "string", " 2006.2 ")
write_parproperty(virtman, "Header", "string", " \"VFIO org.javaseis.VirtualFolder 2006.2\" ")
write_parproperty(virtman, "Type", "string", " SS ")
write_parproperty(virtman, "POLICY_ID", "string", " RANDOM ")
write_parproperty(virtman, "GLOBAL_REQUIRED_FREE_SPACE", "long", " $(prod(io.axis_lengths[2:end]) * (io.axis_lengths[1] * sizeof(io.dataformat) + headerlength(io))) ")
save_file(xdoc, joinpath(io.filename, "VirtualFolders.xml"))
delete_first_line(joinpath(io.filename, "VirtualFolders.xml"))
end
function write_parproperty(parent::XMLElement, name::String, format::String, value::String)
child = new_child(parent, "par")
set_attribute(child, "name", name)
set_attribute(child, "type", format)
add_text(child, value)
end
# trace map
volumeindex(io::JSeis, frm::Int64) = div(frm-1, io.axis_lengths[3]) + 1
mapposition(io::JSeis, frm::Int64) = frm - (volumeindex(io, frm) - 1) * io.axis_lengths[3]
function readmap(io::JSeis, frm::Int64)
vol = volumeindex(io, frm)
if vol == io.currentvolume
return
end
posn = (vol - 1) * io.axis_lengths[3] * sizeof(Int32)
iomap = open(joinpath(io.filename, "TraceMap"), "r")
seek(iomap, posn)
read!(iomap, io.map)
close(iomap)
io.currentvolume = vol
end
function create_map(io::JSeis)
iomap = open(joinpath(io.filename, "TraceMap"), "w")
write(iomap, zeros(Int32, prod(io.axis_lengths[3:end])))
close(iomap)
end
function fold_impl(io::JSeis, frm::Int64)
if io.mapped == false
return io.axis_lengths[2]
end
readmap(io, frm)
idx = mapposition(io, frm)
return Int(io.map[idx])
end
"""
fold(io, hdrs)
Compute the fold of a frame where io is JSeis corresponding to the dataset, and hdrs are the headers for the frame.
For example: `io=jsopen("file.js"); fold(io, readframehdrs(io,1))`
"""
function fold(io::JSeis, hdrs::Array{UInt8,2})
trctyp = prop(io, stockprop[:TRC_TYPE])
mapreduce(i->get(trctyp, hdrs, i) == tracetype[:live] ? 1 : 0, +, 1:size(hdrs,2))
end
"""
fold(io, idx...)
Compute the fold of a frame where idx is the frame/volume/hypercube indices. For example, `fold(jsopen("file.js"),1)`
for a 3D dataset, `fold(jsopen("file.js",1,2))` for a 4D dataset, and `fold(jsopen("file.js"),1,2,3)` for a 5D dataset.
"""
fold(io::JSeis, idx::Int64...) = fold_impl(io::JSeis, sub2ind(io, idx))
fold(io::JSeis, idx::CartesianIndex) = fold_impl(io::JSeis, sub2ind(io, idx))
function fold!(io::JSeis, frm::Int64, fld::Int)
if io.mapped == true
if volumeindex(io,frm) == io.currentvolume
io.map[mapposition(io, frm)] = unsafe_trunc(Int32,fld)
end
posn = (frm-1)*sizeof(Int32)
iomap = open(joinpath(io.filename, "TraceMap"), "a")
seek(iomap, posn)
write(iomap, Int32(fld))
close(iomap)
end
end
# extents
extentindex(extents, offset) = extents[div(offset, extents[1].size) + 1]
nextents_heuristic(dims::Array{Int64,1}, format::Type) = ceil(Int, clamp(10.0 + prod(dims)*sizeof(format)/(2.0*1024.0^3), 1, 256))