Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
base fork: toolness/thimble-gallery-lol
base: 74515594cd
...
head fork: toolness/thimble-gallery-lol
compare: 3b390901e5
Checking mergeability… Don't worry, you can still create the pull request.
  • 2 commits
  • 5 files changed
  • 0 commit comments
  • 1 contributor
View
4 package.json
@@ -3,9 +3,11 @@
, "version": "0.1.0"
, "private": true
, "scripts" :
- { "test" : "node test.js" }
+ { "test" : "node node_modules/mocha/bin/mocha -R spec" }
, "dependencies":
{ "express": "2.5.x"
+, "mocha": "1.3.x"
+, "expect.js": "0.1.x"
, "redis": "0.7.x" }
, "engines":
{ "node": "*" }
View
157 published-page-tracker.js
@@ -2,81 +2,105 @@ var rebase = require('./rebase'),
https = require('https'),
redis = require('redis'),
url = require('url'),
- config = require('./config'),
- client = redis.createClient(config.redis.port, config.redis.host);
+ globalConfig = require('./config');
-var HASH_DIR = config.hashDir + "/",
- THIMBLE_URL = url.parse(config.baseThimbleURL),
- REDIS_PREFIX = THIMBLE_URL.hostname + ':';
-
-client.on('error', function(err) {
- console.log("REDIS ERROR", err);
-});
-
-function hashExists(key, cb) {
- client.exists(REDIS_PREFIX + key + ".hash", function(err, exists) {
- if (err) {
- console.log("existence check for key", key, "failed with", err);
- console.log("retrying in 1s");
- setTimeout(function() { hashExists(key, cb); }, 1000);
- } else
- cb(!!exists);
+function makeRedisClient(config) {
+ var client = redis.createClient(config.port, config.host);
+ client.on('error', function(err) {
+ console.log("REDIS ERROR", err);
});
+ return client;
}
-function readHash(key, cb) {
- client.get(REDIS_PREFIX + key + ".hash", cb);
-}
+function HashFinder(options) {
+ options = options || {};
-function writeHash(key, hash, cb) {
- client.set(REDIS_PREFIX + key + ".hash", hash, cb);
-}
+ var idToKey = options.idToKey || rebase,
+ baseThimbleURL = options.baseThimbleURL || globalConfig.baseThimbleURL,
+ client = options.client || makeRedisClient(globalConfig.redis);
-function findHash(id, cb) {
- var key = rebase(id);
- hashExists(key, function(exists) {
- if (exists)
- return readHash(key, function(err, hash) {
- cb(err, id, key, hash);
- });
- var req = https.request({
- host: THIMBLE_URL.hostname,
- port: 443,
- path: THIMBLE_URL.pathname + key,
- method: 'HEAD'
- }, function(res) {
- if (res.statusCode == 200) {
- writeHash(key, res.headers.etag, function(err) {
- cb(err, id, key, res.headers.etag);
- });
+ var THIMBLE_URL = url.parse(baseThimbleURL),
+ REDIS_PREFIX = THIMBLE_URL.hostname + ':';
+
+ function hashExists(key, cb) {
+ client.exists(REDIS_PREFIX + key + ".hash", function(err, exists) {
+ if (err) {
+ console.log("existence check for key", key, "failed with", err);
+ console.log("retrying in 1s");
+ setTimeout(function() { hashExists(key, cb); }, 1000);
} else
- cb("http error " + res.statusCode, id, key, null);
+ cb(!!exists);
});
- req.end();
- });
-};
+ }
-function findManyHashes(start, count, cb) {
- var numLeft = count;
- var errors = [];
- var hashes = {};
+ function readHash(key, cb) {
+ client.get(REDIS_PREFIX + key + ".hash", cb);
+ }
- function done(err, id, key, hash) {
- numLeft--;
- if (err)
- errors.push({
- id: id,
- err: err
+ function writeHash(key, hash, cb) {
+ client.set(REDIS_PREFIX + key + ".hash", hash, cb);
+ }
+
+ function findHash(id, cb) {
+ var key = idToKey(id);
+ hashExists(key, function(exists) {
+ if (exists)
+ return readHash(key, function(err, hash) {
+ cb(err, id, key, hash);
+ });
+ var req = https.request({
+ host: THIMBLE_URL.hostname,
+ port: 443,
+ path: THIMBLE_URL.pathname + key,
+ method: 'HEAD'
+ }, function(res) {
+ if (res.statusCode == 200) {
+ writeHash(key, res.headers.etag, function(err) {
+ cb(err, id, key, res.headers.etag);
+ });
+ } else
+ cb("http error " + res.statusCode, id, key, null);
});
- else
- hashes[key] = hash;
+ req.end();
+ });
+ };
+
+ function findManyHashes(start, count, cb) {
+ var numLeft = count;
+ var errors = [];
+ var hashes = {};
- if (numLeft == 0)
- cb(errors, hashes);
+ function done(err, id, key, hash) {
+ numLeft--;
+ if (err)
+ errors.push({
+ id: id,
+ err: err
+ });
+ else
+ hashes[key] = hash;
+
+ if (numLeft == 0)
+ cb(errors, hashes);
+ }
+
+ for (var i = start; i < start+count; i++)
+ findHash(i, done);
}
- for (var i = start; i < start+count; i++)
- findHash(i, done);
+ return {
+ findManyHashes: findManyHashes,
+ hashExists: hashExists,
+ readHash: readHash,
+ writeHash: writeHash,
+ flushAllHashes: function(cb) {
+ client.keys(REDIS_PREFIX + "*", function(err, keys) {
+ if (err)
+ return cb(err);
+ client.del(keys, function(err) { cb(err); });
+ });
+ }
+ };
}
function PublishedPageTracker(options) {
@@ -86,10 +110,11 @@ function PublishedPageTracker(options) {
batchSize = options.batchSize || 10,
retryDelay = options.retryDelay || 30000,
allHashes = {},
- uniqueHashes = 0;
+ uniqueHashes = 0,
+ hashFinder = options.hashFinder || HashFinder();
function getNextBatch(i) {
- findManyHashes(i, batchSize, function(errors, hashes) {
+ hashFinder.findManyHashes(i, batchSize, function(errors, hashes) {
Object.keys(hashes).forEach(function(key) {
var hash = hashes[key];
if (!(hash in allHashes)) {
@@ -115,14 +140,14 @@ function PublishedPageTracker(options) {
getNextBatch(start);
return {
- pageExists: hashExists,
+ pageExists: hashFinder.hashExists,
allHashes: allHashes
};
}
module.exports = PublishedPageTracker;
-PublishedPageTracker.writeHash = writeHash;
-PublishedPageTracker.client = client;
+PublishedPageTracker.makeRedisClient = makeRedisClient;
+PublishedPageTracker.HashFinder = HashFinder;
if (!module.parent)
PublishedPageTracker({verbose: true});
View
18 test.js
@@ -1,18 +0,0 @@
-var assert = require('assert'),
- fs = require('fs'),
- PublishedPageTracker = require('./published-page-tracker'),
- config = require('./config');
-
-console.log("verifying existence of", config.imageDir);
-assert(fs.existsSync(config.imageDir));
-
-console.log("verifying redis server configuration");
-
-var client = PublishedPageTracker.client;
-
-client.ping(function(err, response) {
- assert(!err);
- assert.equal(response, "PONG");
- client.quit();
- console.log('all tests pass!');
-});
View
22 test/test-config.js
@@ -0,0 +1,22 @@
+var expect = require('expect.js'),
+ fs = require('fs'),
+ PublishedPageTracker = require('../published-page-tracker'),
+ config = require('../config');
+
+describe('config.imageDir', function() {
+ it('should exist on filesystem', function() {
+ expect(fs.existsSync(config.imageDir)).to.be.ok();
+ });
+});
+
+describe('redis', function() {
+ it('should respond to PING', function(done) {
+ var client = PublishedPageTracker.makeRedisClient(config.redis);
+
+ client.ping(function(err, response) {
+ expect(err).to.be(null);
+ expect(response).to.be("PONG");
+ done();
+ });
+ });
+});
View
24 test/test-published-page-tracker.js
@@ -0,0 +1,24 @@
+var expect = require('expect.js'),
+ PublishedPageTracker = require('../published-page-tracker');
+
+describe('HashFinder', function() {
+ it('should work', function(done) {
+ var hashFinder = PublishedPageTracker.HashFinder({
+ baseThimbleURL: "http://test-thimble.testing.test/"
+ });
+ hashFinder.flushAllHashes(function(err) {
+ expect(err).to.be(null);
+ hashFinder.hashExists("BOOP", function(exists) {
+ expect(exists).to.be(false);
+ hashFinder.writeHash("BOOP", "u", function(err) {
+ expect(err).to.be(null);
+ hashFinder.readHash("BOOP", function(err, hash) {
+ expect(err).to.be(null);
+ expect(hash).to.be("u");
+ done();
+ });
+ });
+ });
+ });
+ });
+});

No commit comments for this range

Something went wrong with that request. Please try again.