Skip to content

Commit

Permalink
cache cram objects so that we don't have to re-download the index whe…
Browse files Browse the repository at this point in the history 
…n switching chromosomes
  • Loading branch information
@rbuels
rbuels committed Jul 13, 2018
1 parent 568ce07 commit 79f83ca
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 18 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Expand Up @@ -68,6 +68,7 @@
"imports-loader": "^0.8.0",
"jasmine-node": "^1.14.5",
"jszlib": "git+https://github.com/cmdcolin/jszlib.git#c966ceac3b283d2248764c09e07a4b268edd5032",
"lru-cache": "^4.1.3",
"node-sass": "^4.7.2",
"phantomjs-prebuilt": "^2.1.16",
"regexp-replace-loader": "^1.0.1",
Expand Down
45 changes: 30 additions & 15 deletions src/JBrowse/Store/SeqFeature/CRAM.js
View file
@@ -1,8 +1,11 @@
const LRU = cjsRequire('lru-cache')
const { IndexedCramFile, CraiIndex } = cjsRequire('@gmod/cram/src')
const { CramSizeLimitError } = cjsRequire('@gmod/cram/src/errors')

const { Buffer } = cjsRequire('buffer')

const cramIndexedFilesCache = LRU(5)

define( [
'dojo/_base/declare',
'JBrowse/Errors',
Expand Down Expand Up @@ -61,6 +64,11 @@ class BlobWrapper {
this.blob.stat(resolve, reject)
})
}

toString() {
return ( this.blob.url ? this.blob.url :
this.blob.blob ? this.blob.blob.name : undefined ) || undefined;
}
}


Expand All @@ -77,32 +85,39 @@ return declare( [ SeqFeatureStore, DeferredStatsMixin, DeferredFeaturesMixin, Gl
* @constructs
*/
constructor: function( args ) {
const cramArgs = {}

let dataBlob
if (args.cram)
dataBlob = args.cram
dataBlob = new BlobWrapper(args.cram)
else if (args.urlTemplate)
dataBlob = new XHRBlob(this.resolveUrl(args.urlTemplate || 'data.cram'))
dataBlob = new BlobWrapper(new XHRBlob(this.resolveUrl(args.urlTemplate || 'data.cram')))
else throw new Error('must provide either `cram` or `urlTemplate`')
cramArgs.cramFilehandle = new BlobWrapper(dataBlob)

let indexBlob
if (args.crai)
cramArgs.index = new CraiIndex({ filehandle: new BlobWrapper(args.crai)})
indexBlob = new BlobWrapper(args.crai)
else if (args.craiUrlTemplate)
cramArgs.index = new CraiIndex({filehandle: new BlobWrapper(new XHRBlob(this.resolveUrl(args.craiUrlTemplate)))})
indexBlob = new BlobWrapper(new XHRBlob(this.resolveUrl(args.craiUrlTemplate)))
else if (args.urlTemplate)
cramArgs.index = new CraiIndex({filehandle: new BlobWrapper(new XHRBlob(this.resolveUrl(args.urlTemplate+'.crai')))})
indexBlob = new BlobWrapper(new XHRBlob(this.resolveUrl(args.urlTemplate+'.crai')))
else throw new Error('no index provided, must provide a CRAM index')
// TODO: need to add .csi index support

cramArgs.seqFetch = this._seqFetch.bind(this)
cramArgs.checkSequenceMD5 = false

this.cram = new IndexedCramFile(cramArgs)
this.source = dataBlob.toString().match( /\/([^/\#\?]+)($|[\#\?])/ )[1]

// LRU-cache the CRAM object so we don't have to re-download the
// index when we switch chromosomes
const cacheKey = `data: ${dataBlob}, index: ${indexBlob}`
this.cram = cramIndexedFilesCache.get(cacheKey)
if (!this.cram) {
this.cram = new IndexedCramFile({
cramFilehandle: dataBlob,
index: new CraiIndex({filehandle: indexBlob}),
seqFetch: this._seqFetch.bind(this),
checkSequenceMD5: false,
})

this.source = ( dataBlob.url ? dataBlob.url.match( /\/([^/\#\?]+)($|[\#\?])/ )[1] :
dataBlob.blob ? dataBlob.blob.name : undefined ) || undefined;
cramIndexedFilesCache.set(cacheKey, this.cram)
}

// pre-download the index before running the statistics estimation so that the stats
// estimation doesn't time out
Expand Down Expand Up @@ -256,7 +271,7 @@ return declare( [ SeqFeatureStore, DeferredStatsMixin, DeferredFeaturesMixin, Gl
.catch(err => {
// map the CramSizeLimitError to JBrowse Errors.DataOverflow
if (err instanceof CramSizeLimitError) {
err = new Errors.DataOverflow(err.msg)
err = new Errors.DataOverflow(err)
}

errorCallback(err)
Expand Down
69 changes: 66 additions & 3 deletions yarn.lock
View file
Expand Up @@ -120,6 +120,17 @@
dependencies:
arrify "^1.0.1"

"@gmod/cram@../cram-js/":
version "0.0.1"
dependencies:
babel-runtime "^6.26.0"
buffer-crc32 "^0.2.13"
cross-fetch "^2.1.0"
es6-promisify "^6.0.0"
long "^4.0.0"
lru-cache "^4.1.3"
md5 "^2.2.1"

"@gmod/gff@^1.1.1":
version "1.1.1"
resolved "https://registry.yarnpkg.com/@gmod/gff/-/gff-1.1.1.tgz#ea7076bb81e8e99c24e85d0248081dcf41a8307f"
Expand Down Expand Up @@ -1255,7 +1266,7 @@ buffer-alloc@^1.1.0:
buffer-alloc-unsafe "^0.1.0"
buffer-fill "^0.1.0"

buffer-crc32@^0.2.1:
buffer-crc32@^0.2.1, buffer-crc32@^0.2.13:
version "0.2.13"
resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"

Expand Down Expand Up @@ -1449,6 +1460,10 @@ chardet@^0.4.0:
version "0.4.2"
resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.4.2.tgz#b5473b33dc97c424e5d98dc87d55d4d8a29c8bf2"

charenc@~0.0.1:
version "0.0.2"
resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667"

chokidar@^1.4.2:
version "1.7.0"
resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-1.7.0.tgz#798e689778151c8076b4b360e5edd28cda2bb468"
Expand Down Expand Up @@ -1878,6 +1893,13 @@ create-hmac@^1.1.0, create-hmac@^1.1.2, create-hmac@^1.1.4:
safe-buffer "^5.0.1"
sha.js "^2.4.8"

cross-fetch@^2.1.0:
version "2.2.2"
resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-2.2.2.tgz#a47ff4f7fc712daba8f6a695a11c948440d45723"
dependencies:
node-fetch "2.1.2"
whatwg-fetch "2.0.4"

cross-spawn@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-3.0.1.tgz#1256037ecb9f0c5f79e3d6ef135e30770184b982"
Expand All @@ -1893,6 +1915,10 @@ cross-spawn@^5.0.1, cross-spawn@^5.1.0:
shebang-command "^1.2.0"
which "^1.2.9"

crypt@~0.0.1:
version "0.0.2"
resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b"

cryptiles@2.x.x:
version "2.0.5"
resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8"
Expand Down Expand Up @@ -2679,6 +2705,12 @@ evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3:
md5.js "^1.3.4"
safe-buffer "^5.1.1"

exec-sh@^0.2.0:
version "0.2.2"
resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.2.2.tgz#2a5e7ffcbd7d0ba2755bdecb16e5a427dfbdec36"
dependencies:
merge "^1.2.0"

execa@^0.7.0:
version "0.7.0"
resolved "https://registry.yarnpkg.com/execa/-/execa-0.7.0.tgz#944becd34cc41ee32a63a9faf27ad5a65fc59777"
Expand Down Expand Up @@ -3710,7 +3742,7 @@ is-binary-path@^1.0.0:
dependencies:
binary-extensions "^1.0.0"

is-buffer@^1.1.5:
is-buffer@^1.1.5, is-buffer@~1.1.1:
version "1.1.6"
resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"

Expand Down Expand Up @@ -4310,6 +4342,10 @@ lodash@^4.0.0, lodash@^4.14.0, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.3.0, lod
version "4.17.10"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7"

long@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"

longest@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097"
Expand Down Expand Up @@ -4339,7 +4375,7 @@ lru-cache@2:
version "2.7.3"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.7.3.tgz#6d4524e8b955f95d4f5b58851ce21dd72fb4e952"

lru-cache@^4.0.1, lru-cache@^4.1.1:
lru-cache@^4.0.1, lru-cache@^4.1.1, lru-cache@^4.1.3:
version "4.1.3"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.3.tgz#a1175cf3496dfc8436c156c334b4955992bce69c"
dependencies:
Expand Down Expand Up @@ -4407,6 +4443,14 @@ md5.js@^1.3.4:
hash-base "^3.0.0"
inherits "^2.0.1"

md5@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/md5/-/md5-2.2.1.tgz#53ab38d5fe3c8891ba465329ea23fac0540126f9"
dependencies:
charenc "~0.0.1"
crypt "~0.0.1"
is-buffer "~1.1.1"

media-typer@0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748"
Expand Down Expand Up @@ -4443,6 +4487,10 @@ merge-descriptors@1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61"

merge@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/merge/-/merge-1.2.0.tgz#7531e39d4949c281a66b8c5a6e0265e8b05894da"

methods@~1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
Expand Down Expand Up @@ -4718,6 +4766,10 @@ node-dir@^0.1.10:
dependencies:
minimatch "^3.0.2"

node-fetch@2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.1.2.tgz#ab884e8e7e57e38a944753cec706f788d1768bb5"

node-getopt@^0.2.3:
version "0.2.4"
resolved "https://registry.yarnpkg.com/node-getopt/-/node-getopt-0.2.4.tgz#3afb554717e5479f4bb10eb7327504650811c7a2"
Expand Down Expand Up @@ -7205,6 +7257,13 @@ vm-browserify@0.0.4:
version "0.0.12"
resolved "https://registry.yarnpkg.com/walkdir/-/walkdir-0.0.12.tgz#2f24f1ade64aab1e458591d4442c8868356e9281"

watch@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/watch/-/watch-1.0.2.tgz#340a717bde765726fa0aa07d721e0147a551df0c"
dependencies:
exec-sh "^0.2.0"
minimist "^1.2.0"

watchpack@^1.4.0:
version "1.6.0"
resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-1.6.0.tgz#4bc12c2ebe8aa277a71f1d3f14d685c7b446cd00"
Expand Down Expand Up @@ -7285,6 +7344,10 @@ wgxpath@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/wgxpath/-/wgxpath-1.0.0.tgz#eef8a4b9d558cc495ad3a9a2b751597ecd9af690"

whatwg-fetch@2.0.4:
version "2.0.4"
resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-2.0.4.tgz#dde6a5df315f9d39991aa17621853d720b85566f"

whet.extend@~0.9.9:
version "0.9.9"
resolved "https://registry.yarnpkg.com/whet.extend/-/whet.extend-0.9.9.tgz#f877d5bf648c97e5aa542fadc16d6a259b9c11a1"
Expand Down

0 comments on commit 79f83ca

Please sign in to comment.