Permalink
Browse files

adding simple caching, new dependency on hashlib

  • Loading branch information...
pagameba committed Nov 25, 2010
1 parent fc14ba4 commit c69852da1d0916f17c1967214a84d5a7ecc5f5b5
Showing with 265 additions and 56 deletions.
  1. +3 −0 .gitmodules
  2. +1 −0 deps/hashlib
  3. +184 −0 examples/cache.js
  4. +77 −56 examples/mapserv.js
View
@@ -0,0 +1,3 @@
+[submodule "deps/hashlib"]
+ path = deps/hashlib
+ url = https://github.com/brainfucker/hashlib.git
Submodule hashlib added at 9f1d3f
View
@@ -0,0 +1,184 @@
+/**
+ * Class: Cache
+ * A simple, hopefully efficient, in-memory caching system. The basic concept
+ * of the cache is to use a modified Least Recently Used (LRU) displacement
+ * algorithm that also tracks the number of cache hits for each cached object.
+ * The cache maintains several LRU queues. An object that gets multiple cache
+ * hits gets promoted periodically to the next higher queue. As objects get
+ * stale in a higher queue and are displaced by new items, they get demoted to
+ * lower queues. Queues are allocated on integral values of ln(<hit count>)
+ * which means that there is a queue for 1 hit, a queue for 2-10, a queue for
+ * 11-100 and so on.
+ */
+
+var Cache = function(max) {
+ // hash for all objects so we can find them relatively quickly
+ var cache = {};
+
+ /**
+ * Class: Queue
+ * manage a linked list of cache objects to a maximum size
+ */
+ var Queue = function(max) {
+ var n = 0,
+ that = this,
+ last = null;
+ that.next = null;
+ /**
+ * Method: push
+ * push an object at the head of the queue
+ *
+ * Parameters:
+ * o - the object ot push
+ *
+ * Returns: an object displaced from the bottom of the queue if the queue
+ * is full or null.
+ */
+ this.push = function(o) {
+ if (!o) return;
+ if (that.next) {
+ o.next = that.next;
+ that.next = o;
+ } else {
+ o.next = null;
+ last = o;
+ }
+ o.prev = that;
+ o.q = this;
+ n++;
+ if (n>max) {
+ return this.pop();
+ }
+ };
+ /**
+ * Method: pop
+ * pull an object off the bottom of the queue and return it
+ */
+ this.pop = function() {
+ if (last) {
+ n--;
+ var o = last;
+ last.prev.next = null;
+ last = last.prev;
+ return o;
+ }
+ };
+ /**
+ * Method: remove
+ * remove an object from this queue if it is owned by this queue and
+ * return it, or return null if the object is not owned by this queue.
+ *
+ * Parameters:
+ * o - the object to remove
+ */
+ this.remove = function(o) {
+ if (!o || !o.queue == this) return null;
+ o.prev.next = o.next;
+ if (o.next) {
+ o.next.prev = o.prev;
+ }
+ o.queue = null;
+ return o;
+ };
+ /**
+ * Method: promote
+ * promote an object to the top of the queue
+ *
+ * Parameters:
+ * o - the object to promote
+ */
+ this.promote = function(o) {
+ this.push(this.remove(o));
+ };
+ };
+
+ /**
+ * Method: get
+ * return an object from the cache and refresh it, or null if not found
+ *
+ * Parameters:
+ * key - the key by which the data is identified
+ */
+ this.get = function(key) {
+ var o = cache[key],
+ q,
+ qn,
+ popped;
+ if (o) {
+ // track the cache hit
+ o['time'] = Date();
+ o.count += 1;
+ // check the queue based on power of 10 on hit count
+ qn = Math.ceil(Math.log(o.count)/Math.LN10);
+ q = queues[qn];
+ if (!q) {
+ var ql = Math.floor(max / Math.pow(2,qn+1));
+ if (ql) {
+ queues[qn] = q = new Queue(ql);
+ } else {
+ qn --;
+ q = queues[qn];
+ }
+ }
+ if (o.queue != q) {
+ if (o.queue) {
+ o.queue.remove(o);
+ }
+ q.push(o);
+ } else {
+ // promote to top of current queue
+ q.promote(o);
+ }
+ }
+ return o;
+ };
+
+ /**
+ * Method: set
+ * put some data in the Cache. If the data is already in the cache,
+ * update it.
+ *
+ * Parameters:
+ * key - the key by which the data is identified
+ * data - arbitrary data to store in the cache.
+ */
+ this.set = function(key, data) {
+ var o = this.get(key),
+ popped;
+ if (!o) {
+ o = {
+ 'time': Date(),
+ data: data,
+ count: 1
+ };
+ cache[key] = o;
+ // discard things popped from the 0th queue
+ queues[0].push(o);
+ } else {
+ o.data = data;
+ }
+ return o;
+ };
+
+ /**
+ * Method: cascade
+ * cascade an object being displaced from a queue onto another queue,
+ * recursing to handle an object being displaced as a result of this.
+ *
+ * Parameters:
+ * o - the object being cascaded
+ * qn - the queue number that the object is being cascaded onto
+ */
+ this.cascade = function(o, qn) {
+ var q = queues[qn];
+ if (o && q) {
+ o.count = Math.pow(10,qn) - 1;
+ this.cascade(q.push(o, qn-1));
+ }
+ };
+
+ var queues = [new Queue(Math.floor(max / 2))];
+
+};
+
+exports.Cache = Cache;
View
@@ -14,24 +14,32 @@ var util = require('util'),
url = require("url"),
mapserver = require("../mapserver"),
path = require("path"),
- fs = require('fs');
+ fs = require('fs'),
+ Cache = require('./cache').Cache,
+ md5 = require('../deps/hashlib/build/default/hashlib').md5;
var port = 8080;
var maps = {};
var map_config = {
- gmap: '/ms4w/apps/gmap/htdocs/gmap75.map'
+ gmap: '/ms4w/apps/gmap/htdocs/gmap75.map',
+ premium: '/ms4w/data/maps4ms_premium_2009.2_mercator/map_print/maps4ms_premium_2009.2_mercator_canada.map'
};
for (var key in map_config) {
var mappath = map_config[key];
- maps[key] = mapserver.loadMap(mappath, path.dirname(mappath));
+ try {
+ maps[key] = mapserver.loadMap(mappath, path.dirname(mappath));
+ } catch (e) {
+ console.log(mapserver.getError());
+ }
}
+var cache = new Cache(100000);
+
http.createServer(function(request, response) {
var parsed = url.parse(request.url, true);
- console.log(parsed.pathname);
var components = parsed.pathname.split('/').splice(1);
var page = components[0];
if (page == '') {
@@ -68,66 +76,79 @@ http.createServer(function(request, response) {
});
} else if (maps[page] != undefined) {
- console.log(maps[page].imagetype);
- var map = maps[page].copy();
- if (parsed.query.map_size) {
- var mapsize = parsed.query.map_size.split(' ');
- map.width = mapsize[0];
- map.height = mapsize[1];
- }
- if (parsed.query.mapext) {
- var extent = parsed.query.mapext.split(' ');
- if (extent.length == 4) {
- map.setExtent(parseInt(extent[0]),
- parseInt(extent[1]),
- parseInt(extent[2]),
- parseInt(extent[3])
- );
+ var key = md5(parsed.search);
+ console.log(parsed.search+' : '+key);
+ var o = cache.get(key);
+ if (o) {
+ console.log('cache-hit ' + key);
+ response.writeHead(200, {
+ 'Content-Type': o.data.mimetype
+ });
+ response.end(o.data.buffer);
+
+ } else {
+ console.log('cache-miss ' + key);
+ var map = maps[page].copy();
+ if (parsed.query.map_size) {
+ var mapsize = parsed.query.map_size.split(' ');
+ map.width = mapsize[0];
+ map.height = mapsize[1];
}
- }
- if (parsed.query.layers) {
- var layers = parsed.query.layers.split(' ');
- if (layers.length == 1 && layers[0] == 'all') {
- for (var i=0; i<map.layers.length; i++) {
- if (map.layers[i].status != mapserver.MS_DELETE) {
- map.layers[i].status = mapserver.MS_ON;
- }
+ if (parsed.query.mapext) {
+ var extent = parsed.query.mapext.split(' ');
+ if (extent.length == 4) {
+ map.setExtent(parseInt(extent[0]),
+ parseInt(extent[1]),
+ parseInt(extent[2]),
+ parseInt(extent[3])
+ );
}
- } else {
- for (var i=0; i<map.layers.length; i++) {
- var layer = map.layers[i];
- if (layers.indexOf(layer.name) != -1 && layer.status != mapserver.MS_DELETE) {
- layer.status = mapserver.MS_ON;
- } else if (layer.status != mapserver.MS_DELETE && layer.status != mapserver.MS_DEFAULT) {
- layer.status = mapserver.MS_OFF;
+ }
+ if (parsed.query.layers) {
+ var layers = parsed.query.layers.split(' ');
+ if (layers.length == 1 && layers[0] == 'all') {
+ for (var i=0; i<map.layers.length; i++) {
+ if (map.layers[i].status != mapserver.MS_DELETE) {
+ map.layers[i].status = mapserver.MS_ON;
+ }
+ }
+ } else {
+ for (var i=0; i<map.layers.length; i++) {
+ var layer = map.layers[i];
+ if (layers.indexOf(layer.name) != -1 && layer.status != mapserver.MS_DELETE) {
+ layer.status = mapserver.MS_ON;
+ } else if (layer.status != mapserver.MS_DELETE && layer.status != mapserver.MS_DEFAULT) {
+ layer.status = mapserver.MS_OFF;
+ }
}
}
}
- }
- if (parsed.query.map_imagetype) {
- try {
- map.selectOutputFormat(parsed.query.map_imagetype);
- map.imagetype = parsed.query.map_imagetype;
- } catch(e) {
- console.log('Error selecting output format ' + parsed.query.map_imagetype + ' ' + e);
+ if (parsed.query.map_imagetype) {
+ try {
+ map.selectOutputFormat(parsed.query.map_imagetype);
+ map.imagetype = parsed.query.map_imagetype;
+ } catch(e) {
+ console.log('Error selecting output format ' + parsed.query.map_imagetype + ' ' + e);
+ }
}
- }
- map.drawMap(function(err, buffer) {
- if (err) {
- console.log(err);
- response.writeHead(200, {
- 'Content-Type':'text/plain'
- });
- response.end("MapServer Error: " + err.code + " ("+err.codeStr+"): " + err.message + ' in ' + err.routine);
- } else {
- response.writeHead(200, {
- 'Content-Type': map.mimetype
- });
- response.end(buffer);
- }
- });
+ map.drawMap(function(err, buffer) {
+ if (err) {
+ console.log(err);
+ response.writeHead(200, {
+ 'Content-Type':'text/plain'
+ });
+ response.end("MapServer Error: " + err.code + " ("+err.codeStr+"): " + err.message + ' in ' + err.routine);
+ } else {
+ cache.set(key, {mimetype: map.mimetype, buffer: buffer});
+ response.writeHead(200, {
+ 'Content-Type': map.mimetype
+ });
+ response.end(buffer);
+ }
+ });
+ }
} else {
response.writeHead(404, {});
response.end('File not found.');

0 comments on commit c69852d

Please sign in to comment.