Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Added early support for caching files in a local folder, etag support…

… only
  • Loading branch information...
commit cdab4238002dbd931152af33492500df2c9f45f6 1 parent 57de793
Damon Oehlman authored March 12, 2012
3  .gitignore
... ...
@@ -1,2 +1,3 @@
1 1
 .DS_Store
2  
-node_modules
  2
+node_modules
  3
+test/_cache/
143  lib/getit.js
@@ -2,13 +2,81 @@ var debug = require('debug')('getit'),
2 2
     fs = require('fs'),
3 3
     path = require('path'),
4 4
     request = require('request'),
  5
+    mkdirp = require('mkdirp'),
5 6
     url = require('url'),
6 7
     reRemote = /^\w[\w\.\+\-]+\:\/\//,
  8
+    reStatusCached = /^304$/,
7 9
     reStatusOK = /^(2|3)\d+/,
8 10
     reAlias = /^([\w\-]+)\!(.*)$/,
  11
+    reInvalidCacheChars = /(\:\/+|\/+|\.(?!\w+$))/g,
  12
+    reCharset = /^.*charset\=(.*)$/,
9 13
     reTrailingSlash = /\/$/,
10 14
     reLeadingSlash = /^\//;
11 15
     
  16
+// cache helpers
  17
+
  18
+function _getCacheData(target, opts, callback) {
  19
+    var cacheData = {};
  20
+    
  21
+    // if we have no cache folder, then trigger the callback with no data
  22
+    if (! opts.cachePath) {
  23
+        callback(cacheData);
  24
+    }
  25
+    // otherwise, look for an etag file
  26
+    else {
  27
+        var cacheFile = path.resolve(opts.cachePath, getCacheTarget(target)),
  28
+            metaFile = cacheFile + '.meta';
  29
+            
  30
+        // read the etag file
  31
+        fs.readFile(metaFile, 'utf8', function(err, data) {
  32
+            var match, encoding;
  33
+            
  34
+            if (! err) {
  35
+                cacheData = JSON.parse(data);
  36
+                
  37
+                // look for an encoding specification in the metadata
  38
+                match = reCharset.exec(cacheData['content-type']);
  39
+                encoding = match ? match[1] : 'utf8';
  40
+                
  41
+                fs.readFile(cacheFile, encoding, function(err, data) {
  42
+                    if (! err) {
  43
+                        cacheData.data = data;
  44
+                    }
  45
+                    
  46
+                    callback(cacheData);
  47
+                });
  48
+            }
  49
+            else {
  50
+                callback(cacheData);
  51
+            }
  52
+        });
  53
+    }
  54
+}
  55
+
  56
+function _updateCache(target, opts, resErr, res, body, callback) {
  57
+    if (opts.cachePath && (! resErr) && res.headers && res.headers.etag) {
  58
+        var cacheFile = path.resolve(opts.cachePath, getCacheTarget(target)),
  59
+            metaFile = cacheFile + '.meta';
  60
+        
  61
+        mkdirp(opts.cachePath, function(err) {
  62
+            if (! err) {
  63
+                // create the metadata file
  64
+                fs.writeFile(metaFile, JSON.stringify(res.headers), 'utf8', function(err) {
  65
+                    var match = reCharset.exec(res.headers['content-type']),
  66
+                        encoding = match ? match[1] : 'utf8';
  67
+                    
  68
+                    fs.writeFile(cacheFile, body, encoding, function(err) {
  69
+                        callback();
  70
+                    });
  71
+                });
  72
+            }
  73
+        });
  74
+    }
  75
+    else {
  76
+        callback();
  77
+    }
  78
+}
  79
+    
12 80
 function getit(target, opts, callback) {
13 81
     // check for options being omitted
14 82
     if (typeof opts == 'function') {
@@ -27,29 +95,57 @@ function getit(target, opts, callback) {
27 95
     // check if the target looks like a remote target
28 96
     if (isRemote(target)) {
29 97
         // get the target url
30  
-        var targetUrl = getUrl(target);
31  
-        
32  
-        // make the request
33  
-        if (callback) {
34  
-            debug('requesting remote resource (' + targetUrl + '), for target: ' + target);
35  
-            request(targetUrl, function(err, res, body) {
36  
-                debug('received response for target: ' + target);
  98
+        var targetUrl = getUrl(target),
  99
+            requestOpts = {
  100
+                method: 'GET',
  101
+                uri: targetUrl
  102
+            };
  103
+            
  104
+        _getCacheData(target, opts, function(cacheData) {
  105
+            // make the request
  106
+            if (callback) {
  107
+                // if we have cache data then add the if-none-match header
  108
+                if (cacheData.etag) {
  109
+                    requestOpts.headers = {
  110
+                        'If-None-Match': cacheData.etag
  111
+                    };
  112
+                }
  113
+                
  114
+                debug('requesting remote resource (' + targetUrl + '), for target: ' + target);
  115
+                request(requestOpts, function(err, res, body) {
  116
+                    debug('received response for target: ' + target);
37 117
 
38  
-                // ensure we have a response to work with
39  
-                res = res || {};
  118
+                    // ensure we have a response to work with
  119
+                    res = res || {};
  120
+                    
  121
+                    if (reStatusCached.test(res.statusCode)) {
  122
+                        callback(err, cacheData.data);
  123
+                    }
  124
+                    else if (! reStatusOK.test(res.statusCode)) {
  125
+                        err = new Error((res.headers || {}).status || 'Not found');
  126
+                    }
  127
+                    else {
  128
+                        _updateCache(target, opts, err, res, body, function() {
  129
+                            callback(err, err ? null : body);
  130
+                        });
  131
+                    }
  132
+                });
40 133
 
41  
-                if (! reStatusOK.test(res.statusCode)) {
42  
-                    err = new Error((res.headers || {}).status || 'Not found');
  134
+                return null;
  135
+            }
  136
+            else {
  137
+                if (cacheData) {
  138
+                    return _checkCache(targetUrl, cacheData);
43 139
                 }
44  
-
45  
-                callback(err, err ? null : body);
46  
-            });
47  
-            
48  
-            return null;
49  
-        }
50  
-        else {
51  
-            return request(targetUrl);
52  
-        }
  140
+                else {
  141
+                    var res = request(requestOpts);
  142
+                    
  143
+                    // TODO: handle caching
  144
+                    
  145
+                    return res;
  146
+                }
  147
+            }
  148
+        });
53 149
     }
54 150
     else {
55 151
         var targetFile = path.resolve(opts.cwd, target);
@@ -72,6 +168,10 @@ function getit(target, opts, callback) {
72 168
     }
73 169
 }
74 170
 
  171
+function getCacheTarget(target) {
  172
+    return target.replace(reInvalidCacheChars, '-');
  173
+}
  174
+
75 175
 function getUrl(target) {
76 176
     var parts = url.parse(target),
77 177
         scheme = parts.protocol.slice(0, parts.protocol.length - 1),
@@ -115,4 +215,5 @@ function isRemote(target, opts) {
115 215
 exports = module.exports = getit;
116 216
 exports.getUrl = getUrl;
117 217
 exports.expandAliases = expandAliases;
118  
-exports.isRemote = isRemote;
  218
+exports.isRemote = isRemote;
  219
+exports.getCacheTarget = getCacheTarget;
5  package.json
@@ -10,17 +10,20 @@
10 10
   },
11 11
   "dependencies": {
12 12
     "debug": "*",
  13
+    "mkdirp": "*",
13 14
     "request": "2.x.x"
14 15
   },
15 16
 
16 17
   "devDependencies": {
17  
-    "chai": "0.3.x"
  18
+    "chai": "0.3.x",
  19
+    "rimraf": "2.x.x"
18 20
   },
19 21
 
20 22
   "repository": {
21 23
     "type": "git",
22 24
     "url": "git://github.com/DamonOehlman/getit.git"
23 25
   },
  26
+  
24 27
   "bugs": {
25 28
     "url": "http://github.com/DamonOehlman/getit/issues"
26 29
   },
74  test/caching.js
... ...
@@ -0,0 +1,74 @@
  1
+var fs = require('fs'),
  2
+    path = require('path'),
  3
+    expect = require('chai').expect,
  4
+    getit = require('../'),
  5
+    rimraf = require('rimraf'),
  6
+    testContent,
  7
+    cacheFolder = path.resolve(__dirname, '_cache'),
  8
+    target = 'github://DamonOehlman/getit/test/files/test.txt',
  9
+    cacheFile = path.join(cacheFolder, getit.getCacheTarget(target)),
  10
+    metaFile = cacheFile + '.meta',
  11
+    cacheTextOverride = 'cached_text';
  12
+    
  13
+describe('local loading test', function() {
  14
+    before(function(done) {
  15
+        fs.readFile(path.resolve(__dirname, 'files/test.txt'), 'utf8', function(err, data) {
  16
+            if (! err) {
  17
+                testContent = data;
  18
+            }
  19
+            
  20
+            done(err);
  21
+        });
  22
+    });
  23
+    
  24
+    before(function(done) {
  25
+        rimraf(cacheFolder, done);
  26
+    });
  27
+    
  28
+    it('should be able to create a filename suitable for caching', function() {
  29
+        expect(getit.getCacheTarget(target)).to.equal('github-DamonOehlman-getit-test-files-test.txt');
  30
+    });
  31
+    
  32
+    it('should be able to get the non-cached version of the file', function(done) {
  33
+        getit(target, { cachePath: cacheFolder }, function(err, data) {
  34
+            expect(err).to.not.exist;
  35
+            expect(data).to.equal(testContent);
  36
+            done(err);
  37
+        });
  38
+    });
  39
+    
  40
+    it('should have created a cache file', function(done) {
  41
+        fs.readFile(cacheFile, 'utf8', function(err, data) {
  42
+            expect(err).to.not.exist;
  43
+            expect(data).to.equal(testContent);
  44
+            done(err);
  45
+        });
  46
+    });
  47
+    
  48
+    it('should have created an metadata file', function(done) {
  49
+        fs.readFile(metaFile, 'utf8', function(err, data) {
  50
+            expect(err).to.not.exist;
  51
+            done(err);
  52
+        });
  53
+    });
  54
+    
  55
+    it('should used the cached file if we have an etag match', function(done) {
  56
+        // update the cache file with content that we can test
  57
+        fs.writeFile(cacheFile, cacheTextOverride, 'utf8', function(err) {
  58
+            if (err) {
  59
+                done(err);
  60
+            }
  61
+            else {
  62
+                getit(target, { cachePath: cacheFolder }, function(err, data) {
  63
+                    expect(err).to.not.exist;
  64
+                    expect(data).to.equal(cacheTextOverride);
  65
+                    done(err);
  66
+                });
  67
+            }
  68
+        });
  69
+    });
  70
+    
  71
+    it('should be able to cache a file locally', function(done) {
  72
+        done();
  73
+    });
  74
+});

0 notes on commit cdab423

Please sign in to comment.
Something went wrong with that request. Please try again.