Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

[lib] add first iteration of streaming unpacking of pack .idx files

  • Loading branch information...
commit 1004414c1e2767268c81aee4d5bd2170224f9f44 1 parent bd32b63
@tmpvar tmpvar authored
View
124 lib/pack/index.js
@@ -1,2 +1,126 @@
+var Stream = require('stream').Stream,
+ fsm = require('../fsm');
+
module.exports.unpack = require('./unpack');
module.exports.pack = require('./pack');
+
+module.exports.createParserStream = function() {
+
+ var where = 0;
+ var fwd = function(count) { where += count; };
+ var s = new Stream();
+ s.writable = true;
+ s.paused = false;
+
+ var obj = {
+ fanout : [],
+ entries : []
+ };
+
+ var parser = fsm({
+ init : function() {
+
+ this.where = 0;
+
+ this.fwd = function(amount) {
+ this.where += amount;
+ };
+
+ this.change('ident');
+
+ },
+ ident : function(data) {
+ this.want(4, data, function(buffer) {
+ obj.ident = buffer;
+ this.change('version')
+ });
+ },
+ version : function(data) {
+ this.want(4, data, function(buffer) {
+ obj.version = buffer.readUInt32BE(0);
+ if (obj.version === 2) {
+ this.change('first level fanout');
+ } else {
+ throw new Error('fanout version 1.0 not implemented');
+ }
+
+ });
+ },
+
+ 'first level fanout' : function(data) {
+ this.want(4, data, function(buffer) {
+ obj.fanout.push(buffer.readUInt32BE(0));
+ if (obj.fanout.length === 256) {
+ obj.total = obj.fanout[obj.fanout.length-1];
+ this.change('entries')
+ }
+ });
+ },
+ entries : function(data) {
+ this.want(20, data, function(buffer) {
+ obj.entries.push({
+ sha : buffer.toString('hex')
+ });
+
+ if (obj.entries.length >= obj.total) {
+ this.change('crc32')
+ }
+ });
+ },
+ crc32 : function(data) {
+ this.want(4, data, function(buffer, i) {
+ obj.entries[i].crc32 = buffer.toString('hex');
+ if (i >= obj.entries.length - 1) {
+ this.change('small offset');
+ }
+ });
+ },
+
+ 'small offset' : function(data) {
+ this.want(4, data, function(buffer, i) {
+ obj.entries[i].offset = buffer.readUInt32BE(0);
+ if (i >= obj.entries.length - 1) {
+ // TODO: >4gb packfiles
+ // basically this means the offsets that we collected
+ // are indexes into another larger table
+ //this.change('pack offset');
+ this.change('packfile checksum');
+ }
+ });
+ },
+
+ 'pack offset' : function(data) {
+ this.want(8, data, function(buffer, i) {
+ obj.entries[i].largeOffset = buffer.readDoubleBE(0);
+ if (i >= obj.entries.length - 1) {
+ this.change('packfile checksum');
+ }
+ });
+ },
+
+ 'packfile checksum' : function(data) {
+ this.want(20, data, function(buffer) {
+ obj.packfileChecksum = buffer.toString('hex');
+ this.change('indexfile checksum');
+ })
+ },
+
+ 'indexfile checksum' : function(data) {
+ this.want(20, data, function(buffer) {
+ obj.indexfileChecksum = buffer.toString('hex');
+ this.done(obj);
+ })
+ }
+ }, function(obj) {
+ s.end(obj);
+ });
+
+
+ s.write = parser;
+
+ s.end = function(data) {
+ this.emit('end', data);
+ };
+
+ return s;
+};
View
41 test/fixtures/repo.js
@@ -29,17 +29,22 @@ module.exports.getPackFile = function(fn) {
var verifyParts = verifyString.split('\n'), verifyObjs = [];
verifyParts.pop();
verifyParts.pop();
+ verifyParts.pop();
+ var verifyHash = {};
verifyParts.forEach(function(line) {
- var parts = line.split(' ');
+ var parts = line.replace(/[ ]+/g, ' ').split(' ');
+
+ var sha = parts.shift();
var obj = {
- sha : parts.shift(),
+ sha : sha,
type : parts.shift(),
- uncompressedSize : parts.shift(),
- compressedSize : parts.shift(),
- offset : parts.shift(),
+ uncompressedSize : parseInt(parts.shift(), 10),
+ compressedSize : parseInt(parts.shift(), 10),
+ offset : parseInt(parts.shift(), 10)
};
verifyObjs.push(obj);
+ verifyHash[sha] = obj;
});
@@ -54,19 +59,23 @@ module.exports.getPackFile = function(fn) {
var packFile = files[0];
- fs.readFile(packFile, function(err, buffer) {
+ fs.readFile(packFile, function(err, packFileBuffer) {
if (err) throw err;
-
- // cleanup
- rimraf(packRepoPath, function() {
- fn(null, {
- totalObjects: count,
- buffer: buffer,
- verifyString: verifyString,
- verifyObjs: verifyObjs
- });
- });
+ fs.readFile(packFile.replace('.pack', '.idx'), function(err, indexBuffer) {
+
+ // cleanup
+ //rimraf(packRepoPath, function() {
+ fn(null, {
+ totalObjects: count,
+ buffer: packFileBuffer,
+ indexBuffer: indexBuffer,
+ verifyString: verifyString,
+ verifyObjs: verifyObjs,
+ verifyHash: verifyHash
+ });
+ //});
+ })
});
});
});
View
67 test/tests/pack-index.js
@@ -0,0 +1,67 @@
+
+var fs = require('fs');
+var test = require('tap').test;
+var repo = require('../fixtures/repo');
+var Buffer = require('buffer').Buffer;
+var objects = require('../../lib/objects')
+
+var pack = require('../../lib/pack');
+var noop = function() { return null };
+
+module.exports = {
+ "packfile index: create parser stream (sanity)" : function(t) {
+ var s = pack.createParserStream();
+ t.ok(s);
+ t.ok(s.pipe);
+ t.end();
+ },
+
+ "packfile index: parse stream (one byte at a time)" : function(t) {
+ var ps = pack.createParserStream();
+ repo.getPackFile(function(err, packFile) {
+ t.plan(2 + packFile.verifyObjs.length * 2)
+ // delay for binds, and flow of code
+ process.nextTick(function() {
+ var buffer = packFile.indexBuffer;
+ for (var i=0; i<buffer.length;) {
+ ps.write(buffer.slice(i, ++i));
+ }
+ });
+
+ ps.on('end', function(result) {
+ t.ok(result.entries);
+
+ t.equal(packFile.verifyObjs.length, result.entries.length);
+
+ result.entries.forEach(function(entry) {
+ t.equal(entry.sha, packFile.verifyHash[entry.sha].sha);
+ t.equal(entry.offset, packFile.verifyHash[entry.sha].offset);
+ });
+
+ t.end();
+ });
+ });
+ },
+
+ // "packfile index: parse stream (one large chunk)" : function(t) {
+
+ // t.end();
+ // }
+
+
+ // "unpack: valid object count (real file)" : function(t) {
+ // t.plan(2);
+ // repo.getPackFile(function(err, packFile) {
+ // pack.unpack(packFile.buffer, function(err, obj) {
+
+ // // from previous observation, the packfile had 3 objects in it
+ // // ensure sanity before we assume that we actually parsed the packfile
+ // // correctly.
+ // t.ok(packFile.totalObjects > 0);
+
+ // t.equal(obj.count, packFile.totalObjects);
+ // t.end();
+ // }, noop);
+ // });
+ // },
+}
View
1  test/tests/unpack.js
@@ -166,7 +166,6 @@ module.exports = {
packFile.buffer,
function(err, obj) {},
function make(type, data) {
- // TODO: sha
t.ok(type);
t.ok(data);
t.ok(Buffer.isBuffer(data));
Please sign in to comment.
Something went wrong with that request. Please try again.