tweak HashStore sorting batch size and sort mem

GMOD · Jan 31, 2013 · 9ae93d1 · 9ae93d1
1 parent 806d5fe
commit 9ae93d1
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/src/perl5/Bio/JBrowse/ExternalSorter.pm b/src/perl5/Bio/JBrowse/ExternalSorter.pm
@@ -148,7 +148,7 @@ sub flush {
         or croak "couldn't flush segment file: $!\n";
     $fh->close()
         or croak "couldn't close $fn: $!\n";
-    push @{$self->{segments}}, $fn;
+    push @{$self->{segments}}, "$fn";
     $self->{curList} = [];
     $self->{curSize} = 0;
 }

diff --git a/src/perl5/Bio/JBrowse/HashStore.pm b/src/perl5/Bio/JBrowse/HashStore.pm
@@ -39,7 +39,7 @@ use Bio::JBrowse::ExternalSorter;
 my $bucket_class = 'Bio::JBrowse::HashStore::Bucket';
 
 
-=head2 open( dir => "/path/to/dir", hash_bits => 16, sort_mem => 1_000_000_000 )
+=head2 open( dir => "/path/to/dir", hash_bits => 16, sort_mem => 256 * 2**20 )
 
 =cut
 
@@ -148,8 +148,8 @@ sub sort_stream {
 
     $self->{sort_state} = 0;
 
-    # sort up to 10 million records at a time
-    my $batch_size = 10_000_000;
+    # sort up to 40 million records at a time
+    my $batch_size = 40_000_000;
 
     my $sorted_stream = $self->_sort_batch( $in_stream, $batch_size );
 
@@ -174,7 +174,7 @@ sub _sort_batch {
     my $sorter = Bio::JBrowse::ExternalSorter->new(
         sub ($$) {
             $_[0]->[0] cmp $_[1]->[0]
-        }, $self->{sort_mem} || 32_000_000 );
+        }, $self->{sort_mem} || 256*1024*1024 );
 
     local $SIG{INT} = sub {
         $sorter->cleanup;