fix bug 747166, fix bug 757151: Improved caching

Now using memcache for response and template loader caching.

* Moved page inclusion helper from dekiwiki API stub into kuma API.

* Reload (Cache-Control: max-age=0) will cause just the current page to
  be re-evaluated

* Shift-reload (Cache-Control: no-cache) will cause the current page and
  all cached includes to be revalidated - eg. auto-required template
  libs, content included from other pages, etc.

* Use memcache for response caching and template loaders, instead of
  process-local memory. This will allow both multiple node processes
  and the Django app to share common cache data and invalidation cues.

* Honor Cache-Control: no-cache down at the loader cache level, so that
  auto-required libs and content included from other pages gets
  revalidated when a user shift-reloads

* Separate template source loading from compilation, to make source
  caching easier.

* Move FakeMemcached to utils.js, since it's used by more than api.js
  now. Also, md5 function added.

* Fixes to template loading in test-macros.js broken test.

* Remove local memory caching test in test-loaders.js, since that's no
  longer applicable.

TODO.md

@@ -3,31 +3,17 @@ TODO
 
 ## v1.0
 
+* Cache invalidation in KS
+    * In Django, set kuma:modified:{full path hash} = modified whenever a Document is saved
+    * In KS, store cached-at times for everything cached
+    * Consider KS cached data invalid if the cached-at time is older than the
+      modified-at time of the page on which the cached data is based.
+
 * Problem with kuma page slugs containing spaces
 
 * Problem with HTML encoding, can't use an URL with ampersands as the parameter
   to a macro.
 
-* bug 730707: Complete the MindTouch-compat API
-    * Continue burning through most-used and longest-source MDN templates
-
-* General kumascript env and metadata vars in headers with JSON-encoded values?
-    * For use by the API methods.
-    * Examples:
-        * x-kumascript-var-locale: "en-US"
-        * x-kumascript-var-slug: "DOM/Storage"
-        * x-kumascript-var-title: "DOM Storage"
-        * x-kumascript-var-username: "lmorchard"
-        * x-kumascript-var-userlocale: "en-US"
-        * x-kumascript-var-tags: [ "JavaScript", "HTML5", "CSS" ]
-
-* bug 731655: Handle language alternates in markup?
-    * eg. span lang="en-US"; lang="zh-CN"; lang="*"
-    * See also Template:JSInherits
-
-* More backends for response caching
-    * memcache backend, local memory with LRU backend
-
 ## Future
 
 * Revisit Sandbox, or some kind of process separation for executing templates

lib/kumascript/api.js

@@ -109,51 +109,45 @@ var KumaAPI = ks_utils.Class(BaseAPI, {
             var doc_url = $this.baseUrl() + path + '?raw'; 
             var opts = { method: 'HEAD', url: doc_url };
             request(opts, function (err, resp, body) {
+                var result = '';
                 if (resp && 200 == resp.statusCode) {
                     result = true;
                 }
                 next(result);
             });            
-        })
-    }
-
-});
+        });
+    },
 
-// ### PageAPI
-// <http://developer.mindtouch.com/en/docs/DekiScript/Reference/Wiki_Functions_and_Variables/Page>
-var PageAPI = ks_utils.Class(BaseAPI, {
-
-    initialize: function (options) {
-        BaseAPI.prototype.initialize.call(this, options);
+    // #### include
+    // Include content rendered from another page
+    // TODO: Support more DekiScript params like:
+    //  * section - Section name
+    //  * revision - revision #
+    //  * heading - (1-5) set the current heading level, adjust headings in included content
+    // [See also](http://developer.mindtouch.com/en/docs/DekiScript/Reference/Wiki_Functions_and_Variables/Wiki.Page)
+    include: function (path) {
         var $this = this;
-
-        // TODO: Need to thread through page details from Server to APIContext to here.
-        this.setVars({
-            uri: 'http://example.com/en/HTML/FakePage',
-            language: 'en'
+        var key = 'kumascript:include:' + $this.parent.md5(path.toLowerCase());
+        return this.parent.cacheFn(key, 3600, function (next) {
+            var doc_url = $this.baseUrl() + path + '?raw=1&macros=1&include=1';
+            var opts = {
+                method: 'GET',
+                headers: { 'Cache-Control': $this.parent.env.cache_control },
+                url: doc_url
+            };
+            request(opts, function (err, resp, body) {
+                var result = '';
+                if (resp && 200 == resp.statusCode) {
+                    result = body;
+                }
+                next(result);
+            });
         });
+
     }
 
 });
 
-// ### FakeMemcached
-//
-// A minimal stub replacement for Memcached, in case it's missing from the
-// config. That way, kumascript can be used without memcache, even if that's
-// not recommended.
-var FakeMemcached = ks_utils.Class({
-    initialize: function (options) {
-        this._cache = {};
-    },
-    set: function (key, value, tm_out, next) {
-        this._cache[key] = value;
-        next(null, true);
-    },
-    get: function (key, next) {
-        next(null, this._cache[key]);
-    }
-});
-
 // ### APIContext
 //
 // Instances of this class manage instances of sub-APIs, supplying them with
@@ -187,7 +181,7 @@ var APIContext = ks_utils.Class({
             this.memcached = new Memcached(mo.server, mo.options || {});
         } else {
             // If the configuration is missing, use the fake stub cache
-            this.memcached = new FakeMemcached();
+            this.memcached = new ks_utils.FakeMemcached();
         }
 
         // Create a new cache for required templates.
@@ -209,7 +203,11 @@ var APIContext = ks_utils.Class({
     //
     // TODO: Very permissive. Should there be more restrictions on net access?
     request: request,
-
+
+    // #### md5
+    // Return the MD5 hex digest of the given utf8 string
+    md5: ks_utils.md5,
+
     // Install a new instance of the given API class, with the given name.
     installAPI: function (cls, name) {
         setCaseVariantAliases(this, name, new cls({parent: this}));
@@ -286,22 +284,14 @@ var APIContext = ks_utils.Class({
                     mc.set(key, val, tm_out, function (err, c_result) {
                         result = val;
                         f['return']();
-                    })
-                })
+                    });
+                });
             }
         });
         f.wait();
         return result;
     },
 
-    // #### md5
-    // Return the MD5 hex digest of the given utf8 string
-    md5: function (str) {
-        var md5 = crypto.createHash('md5')
-        md5.update(str, 'utf8');
-        return md5.digest('hex')
-    },
-
     // #### template(name, arguments)
     //
     // Attempt to load and execute a template with the given name and