Merge pull request #420 from piotr-gawron/string-virtual-file

Dynamically generated vcf content

src/main/AbstractFile.js

@@ -0,0 +1,43 @@
+/**
+ * AbstractFile is an abstract representation of a file. There are two implementation:
+ * 1. RemoteFile  - representation of a file on a remote server which can be
+ * fetched in chunks, e.g. using a Range request.
+ * 2. LocalStringFile is a representation of a file that was created from input string. 
+ * Used for testing and small input files.
+ * @flow
+ */
+'use strict';
+
+//import Q from 'q';
+
+class AbstractFile {
+  constructor() {
+    //how to prevent instantation of this class???
+    //this code doesn't pass npm run flow
+//    if (new.target === AbstractFile) { 
+//      throw new TypeError("Cannot construct AbstractFile instances directly");
+//    }
+  }
+
+  getBytes(start: number, length: number):Object {//: Q.Promise<ArrayBuffer> {
+    throw new TypeError("Method getBytes is not implemented");
+  }
+
+  // Read the entire file -- not recommended for large files!
+  getAll():Object {//: Q.Promise<ArrayBuffer> {
+    throw new TypeError("Method getAll is not implemented");
+  }
+
+  // Reads the entire file as a string (not an ArrayBuffer).
+  // This does not use the cache.
+  getAllString():Object {//: Q.Promise<string> {
+    throw new TypeError("Method getAllString is not implemented");
+  }
+
+  // Returns a promise for the number of bytes in the remote file.
+  getSize():Object {//: Q.Promise<number> {
+    throw new TypeError("Method getSize is not implemented");
+  }
+}
+
+module.exports = AbstractFile;

src/main/LocalStringFile.js

@@ -0,0 +1,66 @@
+/**
+ * LocalStringFile is a representation of a file that was created from input string. Used for testing and small input files.
+ * @flow
+ */
+'use strict';
+
+import Q from 'q';
+import AbstractFile from './AbstractFile';
+
+class LocalStringFile extends AbstractFile {
+  fileLength: number;
+  content: string;  //content of this "File"
+  buffer: ArrayBuffer;
+
+  constructor(content: string) {
+    super();
+    this.fileLength = content.length;  
+    this.buffer = new ArrayBuffer(content.length); // 2 bytes for each char
+    this.content = content;
+
+    var bufView = new Uint8Array(this.buffer);
+    for (var i=0; i < this.fileLength; i++) {
+      bufView[i] = content.charCodeAt(i);
+    }
+  }
+
+  getBytes(start: number, length: number): Q.Promise<ArrayBuffer> {
+    if (length < 0) {
+      return Q.reject(`Requested <0 bytes (${length})`);
+    }
+
+    // If the remote file length is known, clamp the request to fit within it.
+    var stop = start + length - 1;
+    if (this.fileLength != -1) {
+      stop = Math.min(this.fileLength - 1, stop);
+    }
+
+    // First check the cache.
+    var buf = this.getFromCache(start, stop);
+    return Q.when(buf);
+  }
+
+  // Read the entire file -- not recommended for large files!
+  getAll(): Q.Promise<ArrayBuffer> {
+   var buf = this.getFromCache(0, this.fileLength - 1);
+   return Q.when(buf);
+  }
+
+  // Reads the entire file as a string (not an ArrayBuffer).
+  // This does not use the cache.
+  getAllString(): Q.Promise<string> {
+    return Q.when(this.content);
+  }
+
+  // Returns a promise for the number of bytes in the remote file.
+  getSize(): Q.Promise<number> {
+    return Q.when(this.fileLength);
+  }
+
+  getFromCache(start: number, stop: number): ?ArrayBuffer {
+    return this.buffer.slice(start, stop + 1);
+  }
+
+}
+
+module.exports = LocalStringFile;

src/main/RemoteFile.js

@@ -6,6 +6,7 @@
 'use strict';
 
 import Q from 'q';
+import AbstractFile from './AbstractFile';
 
 type Chunk = {
   start: number;
@@ -15,13 +16,14 @@ type Chunk = {
 }
 
 
-class RemoteFile {
+class RemoteFile extends AbstractFile{
   url: string;
   fileLength: number;
   chunks: Array<Chunk>;  // regions of file that have already been loaded.
   numNetworkRequests: number;  // track this for debugging/testing
 
   constructor(url: string) {
+    super();
     this.url = url;
     this.fileLength = -1;  // unknown
     this.chunks = [];

src/main/data/vcf.js

@@ -8,7 +8,7 @@
 'use strict';
 
 import type ContigInterval from '../ContigInterval';
-import type RemoteFile from '../RemoteFile';
+import type AbstractFile from '../AbstractFile';
 import type Q from 'q';
 
 export type Variant = {
@@ -146,10 +146,10 @@ class ImmediateVcfFile {
 
 
 class VcfFile {
-  remoteFile: RemoteFile;
+  remoteFile: AbstractFile;
   immediate: Q.Promise<ImmediateVcfFile>;
 
-  constructor(remoteFile: RemoteFile) {
+  constructor(remoteFile: AbstractFile) {
     this.remoteFile = remoteFile;
 
     this.immediate = this.remoteFile.getAllString().then(txt => {

src/main/sources/VcfDataSource.js

@@ -13,6 +13,7 @@ import Q from 'q';
 
 import ContigInterval from '../ContigInterval';
 import RemoteFile from '../RemoteFile';
+import LocalStringFile from '../LocalStringFile';
 import VcfFile from '../data/vcf';
 
 export type VcfDataSource = {
@@ -91,13 +92,15 @@ function createFromVcfFile(remoteSource: VcfFile): VcfDataSource {
   return o;
 }
 
-function create(data: {url:string}): VcfDataSource {
-  var url = data.url;
-  if (!url) {
-    throw new Error(`Missing URL from track: ${JSON.stringify(data)}`);
+function create(data: {url?: string, content?: string}): VcfDataSource {
+  var {url, content} = data;
+  if (url) {
+    return createFromVcfFile(new VcfFile(new RemoteFile(url)));
+  } else if (content) {
+    return createFromVcfFile(new VcfFile(new LocalStringFile(content)));
   }
-
-  return createFromVcfFile(new VcfFile(new RemoteFile(url)));
+  // If no URL or content is passed, fail
+  throw new Error(`Missing URL or content from track: ${JSON.stringify(data)}`);
 }
 
 module.exports = {

src/test/LocalStringFile-test.js

@@ -0,0 +1,113 @@
+/* @flow */
+'use strict';
+
+import {expect} from 'chai';
+
+import LocalStringFile from '../main/LocalStringFile';
+import jBinary from 'jbinary';
+
+describe('LocalStringFile', () => {
+  function bufferToText(buf) {
+    return new jBinary(buf).read('string');
+  }
+
+  it('should fetch a subset of a file', function() {
+    var f = new LocalStringFile('0123456789\n');
+    var promisedData = f.getBytes(4, 5);
+
+    return promisedData.then(buf => {
+      expect(buf.byteLength).to.equal(5);
+      expect(bufferToText(buf)).to.equal('45678');
+    });
+  });
+
+  it('should fetch subsets from cache', function() {
+    var f = new LocalStringFile('0123456789\n');
+    return f.getBytes(0, 10).then(buf => {
+      expect(buf.byteLength).to.equal(10);
+      expect(bufferToText(buf)).to.equal('0123456789');
+      return f.getBytes(4, 5).then(buf => {
+        expect(buf.byteLength).to.equal(5);
+        expect(bufferToText(buf)).to.equal('45678');
+      });
+    });
+  });
+
+  it('should fetch entire files', function() {
+    var f = new LocalStringFile('0123456789\n');
+    return f.getAll().then(buf => {
+      expect(buf.byteLength).to.equal(11);
+      expect(bufferToText(buf)).to.equal('0123456789\n');
+    });
+  });
+
+  it('should determine file lengths', function() {
+    var f = new LocalStringFile('0123456789\n');
+    return f.getSize().then(size => {
+      expect(size).to.equal(11);
+    });
+  });
+
+  it('should get file lengths from full requests', function() {
+    var f = new LocalStringFile('0123456789\n');
+    return f.getAll().then(buf => {
+      return f.getSize().then(size => {
+        expect(size).to.equal(11);
+      });
+    });
+  });
+
+  it('should get file lengths from range requests', function() {
+    var f = new LocalStringFile('0123456789\n');
+    return f.getBytes(4, 5).then(buf => {
+      return f.getSize().then(size => {
+        expect(size).to.equal(11);
+      });
+    });
+  });
+
+  it('should cache requests for full files', function() {
+    var f = new LocalStringFile('0123456789\n');
+    f.getAll().then(buf => {
+      expect(buf.byteLength).to.equal(11);
+      expect(bufferToText(buf)).to.equal('0123456789\n');
+      return f.getAll().then(buf => {
+        expect(buf.byteLength).to.equal(11);
+        expect(bufferToText(buf)).to.equal('0123456789\n');
+      });
+    });
+  });
+
+  it('should serve range requests from cache after getAll', function() {
+    var f = new LocalStringFile('0123456789\n');
+    return f.getAll().then(buf => {
+      expect(buf.byteLength).to.equal(11);
+      expect(bufferToText(buf)).to.equal('0123456789\n');
+      return f.getBytes(4, 5).then(buf => {
+        expect(buf.byteLength).to.equal(5);
+        expect(bufferToText(buf)).to.equal('45678');
+      });
+    });
+  });
+
+  it('should truncate requests past EOF', function() {
+    var f = new LocalStringFile('0123456789\n');
+    var promisedData = f.getBytes(4, 100);
+
+    return promisedData.then(buf => {
+      expect(buf.byteLength).to.equal(7);
+      expect(bufferToText(buf)).to.equal('456789\n');
+      return f.getBytes(6, 90).then(buf => {
+        expect(buf.byteLength).to.equal(5);
+        expect(bufferToText(buf)).to.equal('6789\n');
+      });
+    });
+  });
+
+  it('should fetch entire files as strings', function() {
+    var f = new LocalStringFile('0123456789\n');
+    return f.getAllString().then(txt => {
+      expect(txt).to.equal('0123456789\n');
+    });
+  });
+});

src/test/data/vcf-test.js

@@ -6,26 +6,43 @@ import {expect} from 'chai';
 import VcfFile from '../../main/data/vcf';
 import ContigInterval from '../../main/ContigInterval';
 import RemoteFile from '../../main/RemoteFile';
+import LocalStringFile from '../../main/LocalStringFile';
 
 describe('VCF', function() {
-  it('should respond to queries', function() {
-    var vcf = new VcfFile(new RemoteFile('/test-data/snv.vcf'));
-    var range = new ContigInterval('20', 63799, 69094);
-    return vcf.getFeaturesInRange(range).then(features => {
-      expect(features).to.have.length(6);
+  describe('should respond to queries', function() {
+    var testQueries = (vcf) => {
+      var range = new ContigInterval('20', 63799, 69094);
+      return vcf.getFeaturesInRange(range).then(features => {
+        expect(features).to.have.length(6);
+
+        var v0 = features[0],
+            v5 = features[5];
 
-      var v0 = features[0],
-          v5 = features[5];
+        expect(v0.contig).to.equal('20');
+        expect(v0.position).to.equal(63799);
+        expect(v0.ref).to.equal('C');
+        expect(v0.alt).to.equal('T');
 
-      expect(v0.contig).to.equal('20');
-      expect(v0.position).to.equal(63799);
-      expect(v0.ref).to.equal('C');
-      expect(v0.alt).to.equal('T');
+        expect(v5.contig).to.equal('20');
+        expect(v5.position).to.equal(69094);
+        expect(v5.ref).to.equal('G');
+        expect(v5.alt).to.equal('A');
+      });
+    };
+
+    var remoteFile = new RemoteFile('/test-data/snv.vcf');
+
+    it('remote file', function() {
+      var vcf = new VcfFile(remoteFile);
+      testQueries(vcf);
+    });
 
-      expect(v5.contig).to.equal('20');
-      expect(v5.position).to.equal(69094);
-      expect(v5.ref).to.equal('G');
-      expect(v5.alt).to.equal('A');
+    it('local file from string', function() {
+      return remoteFile.getAllString().then(content => {
+        var localFile = new LocalStringFile(content);
+        var vcf = new VcfFile(localFile);
+        testQueries(vcf);
+      });
     });
   });