Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 479 lines (418 sloc) 15.562 kb
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
1 package JsonGenerator;
2
2770b45 make generate-names figure out its input files on its own
Mitch Skinner authored
3 use base 'Exporter';
4 our @EXPORT_OK = qw/ readJSON writeJSON modifyJSFile /;
5
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
6 use strict;
7 use warnings;
8
9 use NCList;
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
10 use LazyNCList;
2fd4ba1 require JSON version 2
Mitch Skinner authored
11 use JSON 2;
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
12 use IO::File;
13 use Fcntl ":flock";
7d261ab server-calculated histograms; evenly size lazy feature chunks
Mitch Skinner authored
14 use POSIX qw(ceil floor);
aa26dd6 choose a decent vertical scale for feature histograms
Mitch Skinner authored
15 use List::Util qw(min max sum reduce);
3fc56b1 add external sort mechanism, remove sort from NCList, add compression op...
Mitch Skinner authored
16 use PerlIO::gzip;
292aa08 actually set max_depth on all the writeJSON calls
Mitch Skinner authored
17 use constant MAX_JSON_DEPTH => 2048;
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
18
aa26dd6 choose a decent vertical scale for feature histograms
Mitch Skinner authored
19 #this series of numbers is used in JBrowse for zoom level relationships
3101e39 somewhat closer to working
Mitch Skinner authored
20 my @multiples = (1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000,
aa26dd6 choose a decent vertical scale for feature histograms
Mitch Skinner authored
21 10_000, 20_000, 50_000, 100_000, 200_000, 500_000, 1_000_000);
22
d7f84ac refactoring JSON generation, part 1: make NCList operate on flattened fe...
Mitch Skinner authored
23 my $startIndex = 0;
24 my $endIndex = 1;
0b46ab2 first cut of feature lazy-load (still have client-side work to do)
Mitch Skinner authored
25 #position of the lazy subfeature file name in the fake feature.
26 my $lazyIndex = 2;
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
27
b46e483 clean up JsonGenerator and speed up hist generation a bit
Mitch Skinner authored
28 my $histChunkSize = 10_000;
29
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
30 my %builtinDefaults =
31 (
c8c7b57 Revert "finish streaming pair matcher"
Mitch Skinner authored
32 "class" => "feature"
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
33 );
34
35 sub readJSON {
3fc56b1 add external sort mechanism, remove sort from NCList, add compression op...
Mitch Skinner authored
36 my ($file, $default, $skipAssign, $compress) = @_;
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
37 if (-s $file) {
38 my $OLDSEP = $/;
3e53f7b split refseq info from track info
Mitch Skinner authored
39 my $fh = new IO::File $file, O_RDONLY
40 or die "couldn't open $file: $!";
3fc56b1 add external sort mechanism, remove sort from NCList, add compression op...
Mitch Skinner authored
41 binmode($fh, ":gzip") if $compress;
3e53f7b split refseq info from track info
Mitch Skinner authored
42 flock $fh, LOCK_SH;
8207c40 beginning of subfeature json generation
Mitch Skinner authored
43 # optionally skip variable assignment line
44 $fh->getline() if $skipAssign;
45 undef $/;
3e53f7b split refseq info from track info
Mitch Skinner authored
46 $default = JSON::from_json(<$fh>);
47 $fh->close()
48 or die "couldn't close $file: $!";
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
49 $/ = $OLDSEP;
50 }
51 return $default;
52 }
53
54 sub writeJSON {
3fc56b1 add external sort mechanism, remove sort from NCList, add compression op...
Mitch Skinner authored
55 my ($file, $toWrite, $opts, $compress) = @_;
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
56
57 # create JSON object
58 my $json = new JSON;
59 # set opts
60 if (defined($opts) and ref($opts) eq 'HASH') {
61 for my $method (keys %$opts) {
62 $json->$method( $opts->{$method} );
63 }
64 }
65
66 # check depth
1d0c493 comment out findDepth stuff for performance
Mitch Skinner authored
67 #my $depth = findDepth($toWrite);
68 #my $maxDepth = $json->get_max_depth;
69 # if ($depth >= $maxDepth) {
70 # my @deepPath = @{deepestPath($toWrite)};
71 # my $warning = "WARNING: found deep path (depth = " . $depth . ", max depth allowed = " . $maxDepth . ")\n";
72 # warn $warning;
73 # for my $n (0..$#deepPath) {
74 # my $elem = $deepPath[$n];
75 # my $type = ref($elem);
76 # if ($type eq 'HASH') {
77 # warn $n, ": { ", join(", ", map("$_ => $$elem{$_}", keys %$elem)), " }\n";
78 # } elsif ($type eq 'ARRAY') {
79 # warn $n, ": [ ", join(", ", map(defined($_) ? $_ : "undef", @$elem)), " ]\n";
80 # } else {
81 # warn $n, ": ", $elem, "\n";
82 # }
83 # }
84 # warn $warning; # repeat the warning after printing the trace
85 # }
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
86
87 # write
3e53f7b split refseq info from track info
Mitch Skinner authored
88 my $fh = new IO::File $file, O_WRONLY | O_CREAT
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
89 or die "couldn't open $file: $!";
3e53f7b split refseq info from track info
Mitch Skinner authored
90 flock $fh, LOCK_EX;
91 $fh->seek(0, SEEK_SET);
92 $fh->truncate(0);
3fc56b1 add external sort mechanism, remove sort from NCList, add compression op...
Mitch Skinner authored
93 if ($compress) {
94 binmode($fh, ":gzip")
95 or die "couldn't set binmode: $!";
96 }
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
97 $fh->print($json->encode($toWrite));
3e53f7b split refseq info from track info
Mitch Skinner authored
98 $fh->close()
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
99 or die "couldn't close $file: $!";
100 }
101
102 sub modifyJSFile {
103 my ($file, $varName, $callback) = @_;
104 my ($data, $assign);
105 my $fh = new IO::File $file, O_RDWR | O_CREAT
106 or die "couldn't open $file: $!";
107 flock $fh, LOCK_EX;
108 # if the file is non-empty,
109 if (($fh->stat())[7] > 0) {
110 # get variable assignment line
111 $assign = $fh->getline();
112 # get data
113 my $jsonString = join("", $fh->getlines());
114 $data = JSON::from_json($jsonString) if (length($jsonString) > 0);
115 # prepare file for re-writing
116 $fh->seek(0, SEEK_SET);
117 $fh->truncate(0);
118 }
119 # add assignment line
120 $fh->print("$varName = \n");
121 # modify data, write back
3e53f7b split refseq info from track info
Mitch Skinner authored
122 $fh->print(JSON::to_json($callback->($data), {pretty => 1}));
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
123 $fh->close()
124 or die "couldn't close $file: $!";
125 }
126
c68640e first cut of json generator refactoring; supports new ucsc-to-json.pl an...
Mitch Skinner authored
127 sub writeTrackEntry {
128 my ($file, $entry) = @_;
129 modifyJSFile($file, "trackInfo",
130 sub {
131 my $origTrackList = shift;
132 my @trackList = grep { exists($_->{'label'}) } @$origTrackList;
133 my $i;
134 for ($i = 0; $i <= $#trackList; $i++) {
135 last if ($trackList[$i]->{'label'} eq $entry->{'label'});
136 }
137 $trackList[$i] = $entry;
138 return \@trackList;
139 });
140 }
141
7e5c01f feature click URL templates, feature extraData
Mitch Skinner authored
142 # turn perl subs from the config file into callable functions
143 sub evalSubStrings {
144 my $hashref = shift;
145 foreach my $key (keys %{$hashref}) {
146 next if ("CODE" eq (ref $hashref->{$key}));
147
148 if ("HASH" eq (ref $hashref->{$key})) {
149 evalSubStrings($hashref->{$key});
150 } else {
151 $hashref->{$key} = eval($hashref->{$key})
6648c91 Added rules to TWiki plugin makefile for (i) BED->GFF conversion, (ii) W...
Ian Holmes authored
152 if (defined($hashref->{$key}) && $hashref->{$key} =~ /^\s*sub\s*{.*}\s*$/);
7e5c01f feature click URL templates, feature extraData
Mitch Skinner authored
153 }
154 }
155 }
156
b488bf0 finish refactoring to support streaming flatfiles
Mitch Skinner authored
157 sub new {
0be2d1b get flatfile-to-json.pl working, hand names to JsonGenerator early, remo...
Mitch Skinner authored
158 my ($class, $outDir, $chunkBytes, $compress, $label, $segName,
160e14d separate names.json output from JsonGenerator, use feature counts to cal...
Mitch Skinner authored
159 $refStart, $refEnd, $setStyle, $headers, $subfeatHeaders,
160 $featureCount) = @_;
8207c40 beginning of subfeature json generation
Mitch Skinner authored
161
63bb802 JSON config, generalize subfeature styling
Mitch Skinner authored
162 my %style = ("key" => $label,
3e53f7b split refseq info from track info
Mitch Skinner authored
163 %builtinDefaults,
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
164 %$setStyle);
165
7e5c01f feature click URL templates, feature extraData
Mitch Skinner authored
166 evalSubStrings(\%style);
167
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
168 my $self = {
169 style => \%style,
170 label => $label,
171 outDir => $outDir,
172 chunkBytes => $chunkBytes,
173 compress => $compress,
174 sublistIndex => $#{$headers} + 1,
175 curMapHeaders => $headers,
176 subfeatHeaders => $subfeatHeaders,
177 ext => ($compress ? "jsonz" : "json"),
178 refStart => $refStart,
179 refEnd => $refEnd,
180 count => 0
181 };
182
160e14d separate names.json output from JsonGenerator, use feature counts to cal...
Mitch Skinner authored
183 # $featureCount is an optional parameter; if we don't know it,
184 # then arbitrarily estimate that there's 0.25 features per base
185 # (0.25 features/base is pretty dense, which gives us
186 # a relatively high-resolution histogram; we can always throw
187 # away the higher-resolution histogram data later, so a dense
188 # estimate is conservative. A dense estimate does cost more RAM, though)
189 $featureCount = $refEnd * 0.25 unless defined($featureCount);
190
191 # $histBinThresh is the approximate the number of bases per
192 # histogram bin at the zoom level where FeatureTrack.js switches
193 # to the histogram view by default
194 my $histBinThresh = ($refEnd * 2.5) / $featureCount;
195 $self->{histBinBases} = $multiples[0];
196 foreach my $multiple (@multiples) {
197 $self->{histBinBases} = $multiple;
198 last if $multiple > $histBinThresh;
199 }
3101e39 somewhat closer to working
Mitch Skinner authored
200
ae728a4 clean up commented code, make sure to unlink everything
Mitch Skinner authored
201 # initialize histogram arrays to all zeroes
160e14d separate names.json output from JsonGenerator, use feature counts to cal...
Mitch Skinner authored
202 $self->{hists} = [];
3101e39 somewhat closer to working
Mitch Skinner authored
203 for (my $i = 0; $i <= $#multiples; $i++) {
204 my $binBases = $self->{histBinBases} * $multiples[$i];
205 $self->{hists}->[$i] = [(0) x ceil($refEnd / $binBases)];
b46e483 clean up JsonGenerator and speed up hist generation a bit
Mitch Skinner authored
206 # somewhat arbitrarily cut off the histograms at 100 bins
207 last if $binBases * 100 > $refEnd;
3101e39 somewhat closer to working
Mitch Skinner authored
208 }
b488bf0 finish refactoring to support streaming flatfiles
Mitch Skinner authored
209
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
210 mkdir($outDir) unless (-d $outDir);
211 unlink (glob $outDir . "/hist*");
212 unlink (glob $outDir . "/lazyfeatures*");
ae728a4 clean up commented code, make sure to unlink everything
Mitch Skinner authored
213 unlink $outDir . "/trackData.json";
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
214
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
215 my $lazyPathTemplate = "$outDir/lazyfeatures-{chunk}." . $self->{ext};
216
a61c78b some more cleanup, make non-sorted error message more detailed, document...
Mitch Skinner authored
217 # $output writes out the feature JSON chunk file
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
218 my $output = sub {
219 my ($toWrite, $chunkId) = @_;
4e101ee make chunks not outgrow the feature immediately preceding them if they s...
Mitch Skinner authored
220 #print STDERR "writing chunk $chunkId\n";
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
221 (my $path = $lazyPathTemplate) =~ s/\{chunk\}/$chunkId/g;
222 writeJSON($path,
223 $toWrite,
224 {pretty => 0, max_depth => MAX_JSON_DEPTH},
225 $compress);
226 };
a61c78b some more cleanup, make non-sorted error message more detailed, document...
Mitch Skinner authored
227
228 # $measure measures the size of the feature in the final JSON
229 my $measure = sub {
230 # add 1 for the comma between features
231 # (ignoring, for now, the extra characters for sublist brackets)
232 return length(JSON::to_json($_[0])) + 1;
233 };
7e5c01f feature click URL templates, feature extraData
Mitch Skinner authored
234
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
235 $self->{sublistIndex} += 1 if ($self->{sublistIndex} == $lazyIndex);
236 $self->{features} = LazyNCList->new($startIndex, $endIndex,
237 $self->{sublistIndex},
238 $lazyIndex,
a61c78b some more cleanup, make non-sorted error message more detailed, document...
Mitch Skinner authored
239 $measure,
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
240 $output,
241 $chunkBytes);
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
242
b488bf0 finish refactoring to support streaming flatfiles
Mitch Skinner authored
243 bless $self, $class;
3fc56b1 add external sort mechanism, remove sort from NCList, add compression op...
Mitch Skinner authored
244 return $self;
b488bf0 finish refactoring to support streaming flatfiles
Mitch Skinner authored
245 }
246
247 sub addFeature {
248 my ($self, $feature) = @_;
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
249
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
250 $self->{features}->addSorted($feature);
251 $self->{count}++;
252
3101e39 somewhat closer to working
Mitch Skinner authored
253 my $histograms = $self->{hists};
2a9bf80 slightly optimize hist calculation and feature iteration
Mitch Skinner authored
254 my $curHist;
b46e483 clean up JsonGenerator and speed up hist generation a bit
Mitch Skinner authored
255 my $start = max(0, min($feature->[$startIndex], $self->{refEnd}));
256 my $end = min($feature->[$endIndex], $self->{refEnd});
257 return if ($end < 0);
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
258
3101e39 somewhat closer to working
Mitch Skinner authored
259 for (my $i = 0; $i <= $#multiples; $i++) {
260 my $binBases = $self->{histBinBases} * $multiples[$i];
2a9bf80 slightly optimize hist calculation and feature iteration
Mitch Skinner authored
261 $curHist = $histograms->[$i];
b46e483 clean up JsonGenerator and speed up hist generation a bit
Mitch Skinner authored
262 last unless defined($curHist);
263
264 my $firstBin = int($start / $binBases);
265 my $lastBin = int($end / $binBases);
3101e39 somewhat closer to working
Mitch Skinner authored
266 for (my $bin = $firstBin; $bin <= $lastBin; $bin++) {
2a9bf80 slightly optimize hist calculation and feature iteration
Mitch Skinner authored
267 $curHist->[$bin] += 1;
3101e39 somewhat closer to working
Mitch Skinner authored
268 }
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
269 }
c68640e first cut of json generator refactoring; supports new ucsc-to-json.pl an...
Mitch Skinner authored
270 }
271
5b5d681 add featureCount method, increase JSON depth limit
Mitch Skinner authored
272 sub featureCount {
273 my ($self) = @_;
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
274 return $self->{count};
5b5d681 add featureCount method, increase JSON depth limit
Mitch Skinner authored
275 }
276
7eeba41 bed support
Mitch Skinner authored
277 sub hasFeatures {
278 my ($self) = @_;
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
279 return $self->{count} >= 0;
7eeba41 bed support
Mitch Skinner authored
280 }
281
b488bf0 finish refactoring to support streaming flatfiles
Mitch Skinner authored
282 sub generateTrack {
0be2d1b get flatfile-to-json.pl working, hand names to JsonGenerator early, remo...
Mitch Skinner authored
283 my ($self) = @_;
3fc56b1 add external sort mechanism, remove sort from NCList, add compression op...
Mitch Skinner authored
284
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
285 my $ext = $self->{ext};
3fc56b1 add external sort mechanism, remove sort from NCList, add compression op...
Mitch Skinner authored
286 my $features = $self->{features};
b46e483 clean up JsonGenerator and speed up hist generation a bit
Mitch Skinner authored
287 $features->finish();
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
288
289 # approximate the number of bases per histogram bin at the zoom level where
290 # FeatureTrack.js switches to histogram view, by default
3101e39 somewhat closer to working
Mitch Skinner authored
291 my $histBinThresh = ($self->{refEnd} * 2.5) / $self->{count};
292
293 # find multiple of base hist bin size that's just over $histBinThresh
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
294 my $i;
3101e39 somewhat closer to working
Mitch Skinner authored
295 for ($i = 1; $i <= $#multiples; $i++) {
296 last if ($self->{histBinBases} * $multiples[$i]) > $histBinThresh;
74443df generate multi-level histograms
Mitch Skinner authored
297 }
298
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
299 my @histogramMeta;
74443df generate multi-level histograms
Mitch Skinner authored
300 # Generate more zoomed-out histograms so that the client doesn't
301 # have to load all of the histogram data when there's a lot of it.
b46e483 clean up JsonGenerator and speed up hist generation a bit
Mitch Skinner authored
302 for (my $j = $i - 1; $j <= $#multiples; $j += 1) {
3101e39 somewhat closer to working
Mitch Skinner authored
303 my $curHist = $self->{hists}->[$j];
b46e483 clean up JsonGenerator and speed up hist generation a bit
Mitch Skinner authored
304 last unless defined($curHist);
3101e39 somewhat closer to working
Mitch Skinner authored
305 my $histBases = $self->{histBinBases} * $multiples[$j];
306
74443df generate multi-level histograms
Mitch Skinner authored
307 my $chunks = chunkArray($curHist, $histChunkSize);
308 for (my $i = 0; $i <= $#{$chunks}; $i++) {
3101e39 somewhat closer to working
Mitch Skinner authored
309 writeJSON($self->{outDir} . "/hist-$histBases-$i.$ext",
74443df generate multi-level histograms
Mitch Skinner authored
310 $chunks->[$i],
3fc56b1 add external sort mechanism, remove sort from NCList, add compression op...
Mitch Skinner authored
311 {pretty => 0},
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
312 $self->{compress});
74443df generate multi-level histograms
Mitch Skinner authored
313 }
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
314 push @histogramMeta,
74443df generate multi-level histograms
Mitch Skinner authored
315 {
3101e39 somewhat closer to working
Mitch Skinner authored
316 basesPerBin => $histBases,
74443df generate multi-level histograms
Mitch Skinner authored
317 arrayParams => {
318 length => $#{$curHist} + 1,
a338a88 appears to work, still have to clean up, and figure out what to do with ...
Mitch Skinner authored
319 urlTemplate => "hist-$histBases-{chunk}.$ext",
74443df generate multi-level histograms
Mitch Skinner authored
320 chunkSize => $histChunkSize
321 }
322 };
ae728a4 clean up commented code, make sure to unlink everything
Mitch Skinner authored
323 }
74443df generate multi-level histograms
Mitch Skinner authored
324
aa26dd6 choose a decent vertical scale for feature histograms
Mitch Skinner authored
325 my @histStats;
3101e39 somewhat closer to working
Mitch Skinner authored
326 for (my $j = $i - 1; $j <= $#multiples; $j++) {
b46e483 clean up JsonGenerator and speed up hist generation a bit
Mitch Skinner authored
327 last unless defined($self->{hists}->[$j]);
3101e39 somewhat closer to working
Mitch Skinner authored
328 my $binBases = $self->{histBinBases} * $multiples[$j];
329 push @histStats, {'bases' => $binBases,
330 arrayStats($self->{hists}->[$j])};
aa26dd6 choose a decent vertical scale for feature histograms
Mitch Skinner authored
331 }
332
7e5c01f feature click URL templates, feature extraData
Mitch Skinner authored
333 my $trackData = {
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
334 'label' =>
335 $self->{label},
336 'key' =>
337 $self->{style}->{key},
338 'sublistIndex' =>
339 $self->{sublistIndex},
340 'lazyIndex' =>
341 $lazyIndex,
342 'headers' =>
343 $self->{curMapHeaders},
344 'featureCount' =>
345 $self->{count},
346 'type' =>
347 "FeatureTrack",
348 'className' =>
349 $self->{style}->{class},
350 'subfeatureClasses' =>
351 $self->{style}->{subfeature_classes},
352 'subfeatureHeaders' =>
ae23441 stop JsonGenerator from losing subfeatureHeaders
Mitch Skinner authored
353 $self->{subfeatHeaders},
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
354 'arrowheadClass' =>
355 $self->{style}->{arrowheadClass},
356 'clientConfig' =>
357 $self->{style}->{clientConfig},
358 'featureNCList' =>
a338a88 appears to work, still have to clean up, and figure out what to do with ...
Mitch Skinner authored
359 $self->{features}->topLevelList,
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
360 'lazyfeatureUrlTemplate' =>
a338a88 appears to work, still have to clean up, and figure out what to do with ...
Mitch Skinner authored
361 "lazyfeatures-{chunk}.$ext",
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
362 'histogramMeta' =>
363 \@histogramMeta,
364 'histStats' =>
365 \@histStats
7e5c01f feature click URL templates, feature extraData
Mitch Skinner authored
366 };
b488bf0 finish refactoring to support streaming flatfiles
Mitch Skinner authored
367 $trackData->{urlTemplate} = $self->{style}->{urlTemplate}
368 if defined($self->{style}->{urlTemplate});
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
369 writeJSON($self->{outDir} ."/trackData.$ext",
7e5c01f feature click URL templates, feature extraData
Mitch Skinner authored
370 $trackData,
3fc56b1 add external sort mechanism, remove sort from NCList, add compression op...
Mitch Skinner authored
371 {pretty => 0, max_depth => MAX_JSON_DEPTH},
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
372 $self->{compress});
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
373 }
374
aa26dd6 choose a decent vertical scale for feature histograms
Mitch Skinner authored
375 sub arrayStats {
376 my $arr = shift;
377 my $max = max(@$arr);
378 my $sum = sum(@$arr);
379 my $mean = $sum / ($#{$arr} + 1);
334463a fiddling with the statistics on the histogram arrays
Mitch Skinner authored
380 # my $var = sum(map {($_ - $mean) ** 2} @$arr) / ($#{$arr} + 1);
381 # return ('max' => $max, 'sum' => $sum,
382 # 'mean' => $mean, 'var' => $var,
383 # 'stddev' => sqrt($var));
74443df generate multi-level histograms
Mitch Skinner authored
384 return ('max' => $max, 'mean' => $mean);
aa26dd6 choose a decent vertical scale for feature histograms
Mitch Skinner authored
385 }
386
bf69d44 make histograms lazy loaded
Mitch Skinner authored
387 sub chunkArray {
388 my ($bigArray, $chunkSize) = @_;
389
390 my @result;
391 for (my $start = 0; $start <= $#{$bigArray}; $start += $chunkSize) {
392 my $lastIndex = $start + $chunkSize;
393 $lastIndex = $#{$bigArray} if $lastIndex > $#{$bigArray};
394
c5e2ebc WIP JSON generation refactoring
Mitch Skinner authored
395 push @result, [@{$bigArray}[$start..$lastIndex]];
bf69d44 make histograms lazy loaded
Mitch Skinner authored
396 }
397 return \@result;
398 }
399
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
400 # findDepth returns the depth of the deepest element(s) in the structure
38a4d65 commented recursive->iterative rewrite of deepest-path tests
Ian Holmes authored
401 # the code is the iterative form of findDepth($obj) = 1 + max(map(findDepth($_), childArray($obj)))
85b4f65 minor comment edit
Ian Holmes authored
402 # where childArray($obj) = values(%$obj) [for a hash], @$obj [for an array] or the empty list [for a scalar]
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
403 sub findDepth {
404 my ($obj) = @_;
ba205b2 rewrote deepest-path tests to be iterative instead of recursive
Ian Holmes authored
405 my ($depth, $childArray, $childIndex);
406 my @stack;
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
407
ba205b2 rewrote deepest-path tests to be iterative instead of recursive
Ian Holmes authored
408 FD_NEW_OBJ:
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
409 my $type = ref($obj);
ba205b2 rewrote deepest-path tests to be iterative instead of recursive
Ian Holmes authored
410 $childArray = $type eq 'HASH' ? [values %$obj] : ($type eq 'ARRAY' ? $obj : []);
411 $depth = 0;
412 $childIndex = 0;
413 FD_CHILD_LOOP:
414 if ($childIndex < @$childArray) {
415 push @stack, [$depth, $childArray, $childIndex];
416 $obj = $childArray->[$childIndex];
417 goto FD_NEW_OBJ;
418 } elsif (@stack) {
419 my $childDepth = $depth + 1;
420 my $vars = pop @stack;
421 ($depth, $childArray, $childIndex) = @$vars;
422 if ($childDepth > $depth) {
423 $depth = $childDepth;
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
424 }
ba205b2 rewrote deepest-path tests to be iterative instead of recursive
Ian Holmes authored
425 ++$childIndex;
426 goto FD_CHILD_LOOP;
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
427 }
428
429 return $depth + 1;
430 }
431
ba205b2 rewrote deepest-path tests to be iterative instead of recursive
Ian Holmes authored
432 # deepestPath returns the path to (the first of) the deepest element(s) in the structure
38a4d65 commented recursive->iterative rewrite of deepest-path tests
Ian Holmes authored
433 # the code is the iterative form of deepestPath($obj) = ($obj, longest(map(deepestPath($_), childArray($obj))))
85b4f65 minor comment edit
Ian Holmes authored
434 # where childArray($obj) = values(%$obj) [for a hash], @$obj [for an array] or the empty list [for a scalar]
38a4d65 commented recursive->iterative rewrite of deepest-path tests
Ian Holmes authored
435 # and longest(@x1, @x2, ... @xn) returns the longest of the given arrays (or the first such, in the event of a tie)
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
436 sub deepestPath {
437 my ($obj) = @_;
ba205b2 rewrote deepest-path tests to be iterative instead of recursive
Ian Holmes authored
438 my ($trace, $childArray, $childIndex);
439 my @stack;
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
440
ba205b2 rewrote deepest-path tests to be iterative instead of recursive
Ian Holmes authored
441 DP_NEW_OBJ:
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
442 my $type = ref($obj);
ba205b2 rewrote deepest-path tests to be iterative instead of recursive
Ian Holmes authored
443 $childArray = $type eq 'HASH' ? [values %$obj] : ($type eq 'ARRAY' ? $obj : []);
444 $trace = [];
445 $childIndex = 0;
446 DP_CHILD_LOOP:
447 if ($childIndex < @$childArray) {
448 push @stack, [$obj, $trace, $childArray, $childIndex];
449 $obj = $childArray->[$childIndex];
450 goto DP_NEW_OBJ;
451 } elsif (@stack) {
452 my $childTrace = [$obj, @$trace];
453 my $vars = pop @stack;
454 ($obj, $trace, $childArray, $childIndex) = @$vars;
455 if (@$childTrace > @$trace) {
456 $trace = $childTrace;
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
457 }
ba205b2 rewrote deepest-path tests to be iterative instead of recursive
Ian Holmes authored
458 ++$childIndex;
459 goto DP_CHILD_LOOP;
7cb2824 Added debugging trace of deeply nested JSON structures, in order to trac...
Ian Holmes authored
460 }
461
462 return [$obj, @$trace];
463 }
464
e9e6c74 first cut of gff-to-json.pl, factor JsonGenerator out of generate-json.p...
Mitch Skinner authored
465 1;
f49f4c9 add copyright notices
Mitch Skinner authored
466
467 =head1 AUTHOR
468
469 Mitchell Skinner E<lt>mitch_skinner@berkeley.eduE<gt>
470
471 Copyright (c) 2007-2009 The Evolutionary Software Foundation
472
473 This package and its accompanying libraries are free software; you can
474 redistribute it and/or modify it under the terms of the LGPL (either
475 version 2.1, or at your option, any later version) or the Artistic
476 License 2.0. Refer to LICENSE for the full license text.
477
478 =cut
Something went wrong with that request. Please try again.