git-dump

#!/usr/bin/env node
/**
 * Dump the contents of a remote git repository without directory listing
 * enabled
 *
 * Author: Dave Eddy <dave@daveeddy.com>
 * Date: October 20, 2016
 * License: MIT
 */

var execFile = require('child_process').execFile;
var fs = require('fs');
var path = require('path');
var util = require('util');

var mkdirp = require('mkdirp');
var request = require('request');
var vasync = require('vasync');

var baseurl = process.argv[2];
if (!baseurl) {
  console.error('Usage: git-dump <url> [dir=.] [workers=3]');
  process.exit(1);
}
baseurl = baseurl.replace(/\/+$/, '');

var dir = process.argv[3] || process.cwd();
try {
  mkdirp.sync(dir);
} catch (e) {}
process.chdir(dir);

var workers = +process.argv[4] || 3;

var STARTING_FILES = [
  'HEAD',
  'objects/info/packs',
  'description',
  'config',
  'COMMIT_EDITMSG',
  'index',
  'packed-refs',
  'refs/heads/master',
  'refs/remotes/origin/HEAD',
  'refs/stash',
  'logs/HEAD',
  'logs/refs/heads/master',
  'logs/refs/remotes/origin/HEAD',
  'info/refs',
  'info/exclude'
];

function extracthashes(s) {
  return s.match(/[0-9a-fA-F]{40}/g) || [];
}
function extractpacks(s) {
  return s.match(/pack-([0-9a-fA-F]{40})/g) || [];
}

function hashtofile(s) {
  return util.format('objects/%s/%s',
      s.substr(0, 2),
      s.substr(2));
}
function filetohash(f) {
  return f.replace('objects/', '').replace(/\//g, '');
}

var downloaded = {};
var queue = vasync.queue(function (f, cb) {
  f = f.replace(/^\/*/, '');

  // we've already processed this file
  if (downloaded.hasOwnProperty(f)) {
    console.log('%s already downloaded', f);
    cb();
    return;
  }

  // mkdir all parent direcotries of the file
  var dirname = path.dirname(f);
  mkdirp(dirname, function (err) {
    if (err) {
      cb(err);
      return;
    }

    // GET the url constructed
    var url = util.format('%s/%s', baseurl, f);
    request({url: url, encoding: null}, function (err, res, body) {
      downloaded[f] = true;

      if (err) {
        cb(err);
        return;
      }

      if (res.statusCode < 200 || res.statusCode >= 300) {
        cb(new Error('bad statusCode: ' + res.statusCode));
        return;
      }

      // write the file out
      fs.writeFile(f, body, function (err) {
        if (err) {
          cb(err);
          return;
        }

        console.log('> GET %s --> %s', url, f);
        var hashes = extracthashes(body.toString('utf8'));
        if (hashes.length > 0) {
          console.log('found %d hashes in %s', hashes.length, f);
          queue.push(hashes.map(function (hash) {
            return hashtofile(hash);
          }));
        }

        // were done if the file can not be converted in a hash
        if (!f.match(/\/[a-f0-9]{2}\/[a-f0-9]{38}/)) {
          cb();
          return;
        }

        // print the file based on hash and find more hashes
        var hash = filetohash(f);
        execFile('git', ['cat-file', '-p', hash],
          function (err, stdout, stderr) {
          if (err) {
            // this is ok to ignore
            cb();
            return;
          }

          hashes = extracthashes(stdout);
          if (hashes.length > 0) {
            console.log('found %d hashes in `git cat-file -p %s`',
              hashes.length, filetohash(f));
            hashes = hashes.map(function (hash) {
              return hashtofile(hash);
            });
            queue.push(hashes);
          }

          var packs = extractpacks(stdout);
          if (packs.length > 0) {
            console.log('found %d packs in `git cat-file -p %s`',
              packs.length, filetohash(f));
            var newpacks = [];
            packs.forEach(function (pack) {
              newpacks.push(util.format('objects/pack/%s.pack', pack));
              newpacks.push(util.format('objects/pack/%s.idx', pack));
            });
            queue.push(newpacks);
          }

          cb();
        });
      });
    });
  });
}, workers);

queue.push(STARTING_FILES);