Re-instate block merging, but with a size limit

Increases performance a lot for queries that involve many blocks.

Reverts "No merge adjacent blocks and fix unzip bug related to this",
commit 584adc4.

src/csi.js

@@ -5,7 +5,7 @@ const { unzip } = require('./unzip')
 const VirtualOffset = require('./virtualOffset')
 const Chunk = require('./chunk')
 
-const { longToNumber, checkAbortSignal } = require('./util')
+const { longToNumber, checkAbortSignal, canMergeBlocks } = require('./util')
 
 const CSI1_MAGIC = 21582659 // CSI\1
 const CSI2_MAGIC = 38359875 // CSI\2
@@ -295,6 +295,17 @@ class CSI {
     for (let i = 1; i < numOffsets; i += 1)
       if (off[i - 1].maxv.compareTo(off[i].minv) >= 0)
         off[i - 1].maxv = off[i].minv
+    // merge adjacent blocks
+    l = 0
+    for (let i = 1; i < numOffsets; i += 1) {
+      if (canMergeBlocks(off[l], off[i])) off[l].maxv = off[i].maxv
+      else {
+        l += 1
+        off[l].minv = off[i].minv
+        off[l].maxv = off[i].maxv
+      }
+    }
+    numOffsets = l + 1
 
     return off.slice(0, numOffsets)
   }

src/tbi.js

@@ -8,7 +8,7 @@ const { unzip } = require('./unzip')
 const TBI_MAGIC = 21578324 // TBI\1
 const TAD_LIDX_SHIFT = 14
 
-const { longToNumber, checkAbortSignal } = require('./util')
+const { longToNumber, checkAbortSignal, canMergeBlocks } = require('./util')
 
 /**
  * calculate the list of bins that may overlap with region [beg,end) (zero-based half-open)
@@ -274,6 +274,17 @@ class TabixIndex {
     for (let i = 1; i < numOffsets; i += 1)
       if (off[i - 1].maxv.compareTo(off[i].minv) >= 0)
         off[i - 1].maxv = off[i].minv
+    // merge adjacent blocks
+    l = 0
+    for (let i = 1; i < numOffsets; i += 1) {
+      if (canMergeBlocks(off[l], off[i])) off[l].maxv = off[i].maxv
+      else {
+        l += 1
+        off[l].minv = off[i].minv
+        off[l].maxv = off[i].maxv
+      }
+    }
+    numOffsets = l + 1
 
     return off.slice(0, numOffsets)
   }

src/unzip.js

@@ -49,9 +49,6 @@ function unzipChunk(inputData, chunk) {
     }
     if (fileStartingOffset + pos >= chunk.maxv.blockPosition) {
       // this is the last chunk, trim it and stop decompressing
-      // note if it is the same block is minv it subtracts that already
-      // trimmed part of the slice length
-
       decompressedBlocks[decompressedBlocks.length - 1] = decompressedBlocks[
         decompressedBlocks.length - 1
       ].slice(

src/util.js

@@ -1,3 +1,5 @@
+const MAX_MERGED_BLOCK_SIZE = 30 * 2 ** 20 // 30 MB
+
 module.exports = {
   longToNumber(long) {
     if (
@@ -35,4 +37,12 @@ module.exports = {
       }
     }
   },
+
+  canMergeBlocks(block1, block2) {
+    return (
+      block1.maxv.blockPosition === block2.minv.blockPosition &&
+      block2.maxv.blockPosition - block1.minv.blockPosition <=
+        MAX_MERGED_BLOCK_SIZE
+    )
+  },
 }

test/csi.test.js

@@ -47,7 +47,7 @@ describe('csi index', () => {
     let blocks = await ti.blocksForRange('1', 1, 4000)
     expect(blocks.length).toEqual(0)
     blocks = await ti.blocksForRange('1', 0, 2000046092)
-    expect(blocks.length).toEqual(4)
+    expect(blocks.length).toEqual(1)
     expect(blocks[0].minv.blockPosition).toEqual(0)
     expect(blocks[0].minv.dataPosition).toEqual(2560)
     // console.log( blocks );