Merge pull request #550 from d00rman/mathoid/texvcinfo

Mathoid: store texvcinfo check results and reuse them

package.json

@@ -1,6 +1,6 @@
 {
   "name": "restbase",
-  "version": "0.11.3",
+  "version": "0.11.4",
   "description": "REST storage and service dispatcher",
   "main": "lib/server.js",
   "scripts": {

projects/wikimedia.org.yaml

@@ -55,6 +55,10 @@ paths:
     x-modules:
       - spec:
           paths:
+            /mathoid:
+              x-modules:
+                - path: sys/mathoid.js
+                  options: '{{options.mathoid}}'
             /table:
               x-modules:
                 - type: npm
@@ -74,4 +78,4 @@ paths:
               x-modules:
                 - path: sys/events.js
                   options: '{{options.events}}'
-        options: '{{options}}'
+        options: '{{options}}'

projects/wikimedia.org_sqlite.yaml

@@ -55,6 +55,10 @@ paths:
     x-modules:
       - spec:
           paths:
+            /mathoid:
+              x-modules:
+                - path: sys/mathoid.js
+                  options: '{{options.mathoid}}'
             /table:
               x-modules:
                 - type: npm
@@ -74,4 +78,4 @@ paths:
               x-modules:
                 - path: sys/events.js
                   options: '{{options.events}}'
-        options: '{{options}}'
+        options: '{{options}}'

sys/mathoid.js

@@ -0,0 +1,137 @@
+'use strict';
+
+
+var P = require('bluebird');
+var HyperSwitch = require('hyperswitch');
+var URI = HyperSwitch.URI;
+var HTTPError = HyperSwitch.HTTPError;
+
+
+function MathoidService(options) {
+
+    this.options = options;
+
+}
+
+
+MathoidService.prototype.checkInput = function(hyper, req) {
+
+    var self = this;
+    var rp = req.params;
+    var hash;
+    var origHash;
+    var checkRes;
+
+    // start by calculating the hash
+    return hyper.post({
+        uri: new URI([rp.domain, 'sys', 'post_data', 'mathoid.input', 'hash']),
+        body: { q: req.body.q, type: rp.type }
+    }).then(function(res) {
+        hash = origHash = res.body;
+        // short-circuit if it's a no-cache request
+        if (req.headers && /no-cache/.test(req.headers['cache-control'])) {
+            return P.reject(new HTTPError({ status: 404 }));
+        }
+        // check the post storage
+        return hyper.get({
+            uri: new URI([rp.domain, 'sys', 'key_value', 'mathoid.check', hash])
+        }).catch({ status: 404 }, function(err) {
+            // let's try to find an indirection
+            return hyper.get({
+                uri: new URI([rp.domain, 'sys', 'key_value', 'mathoid.hash_table', hash])
+            }).then(function(hashRes) {
+                // we have a normalised version of the formula
+                hash = hashRes.body;
+                // grab that version from storage
+                return hyper.get({
+                    uri: new URI([rp.domain, 'sys', 'key_value', 'mathoid.check', hash])
+                });
+            });
+        });
+    }).catch({ status: 404 }, function() {
+        // if we are here, it means this is a new input formula
+        // so call mathoid
+        return hyper.post({
+            uri: self.options.host + '/texvcinfo',
+            headers: { 'content-type': 'application/json' },
+            body: {
+                q: req.body.q,
+                type: rp.type
+            }
+        }).then(function(res) {
+            checkRes = res;
+            // store the normalised version
+            return hyper.put({
+                uri: new URI([rp.domain, 'sys', 'post_data', 'mathoid.input', '']),
+                headers: { 'content-type': 'application/json' },
+                body: {
+                    q: res.body.checked,
+                    type: rp.type
+                }
+            });
+        }).then(function(res) {
+            var indirectionP = P.resolve();
+            hash = res.body;
+            // add the indirection to the hash table if the hashes don't match
+            if (hash !== origHash) {
+                indirectionP = hyper.put({
+                    uri: new URI([rp.domain, 'sys', 'key_value', 'mathoid.hash_table', origHash]),
+                    headers: { 'content-type': 'text/plain' },
+                    body: hash
+                });
+            }
+            // store the result
+            checkRes.headers = {
+                'content-type': 'application/json',
+                'cache-control': 'no-cache',
+                'x-resource-location': hash
+            };
+            return P.join(
+                hyper.put({
+                    uri: new URI([rp.domain, 'sys', 'key_value', 'mathoid.check', hash]),
+                    headers: checkRes.headers,
+                    body: checkRes.body
+                }),
+                indirectionP,
+                function() {
+                    return checkRes;
+                }
+            );
+        });
+    });
+
+};
+
+
+module.exports = function(options) {
+
+    var mathoidSrv = new MathoidService(options);
+
+    return {
+        spec: {
+            paths: {
+                '/check/{type}': {
+                    post: {
+                        operationId: 'checkInput'
+                    }
+                }
+            }
+        },
+        operations: {
+            checkInput: mathoidSrv.checkInput.bind(mathoidSrv)
+        },
+        resources: [
+            {
+                uri: '/{domain}/sys/post_data/mathoid.input'
+            }, {
+                uri: '/{domain}/sys/key_value/mathoid.hash_table',
+                body: { valueType: 'string' }
+            }, {
+                uri: '/{domain}/sys/key_value/mathoid.check',
+                body: { valueType: 'json' }
+            }
+        ]
+    };
+
+};
+

sys/post_data.js

@@ -45,6 +45,16 @@ function calculateHash(storedData) {
                  .digest('hex');
 }
 
+PostDataBucket.prototype.calculateHash = function(hyper, req) {
+    return {
+        status: 200,
+        headers: {
+            'content-type': 'text/plain'
+        },
+        body: calculateHash(req.body || {})
+    };
+};
+
 PostDataBucket.prototype.putRevision = function(hyper, req) {
     var rp = req.params;
     var storedData = req.body || {};
@@ -89,7 +99,8 @@ module.exports = function(options) {
         operations: {
             createBucket: postDataBucket.createBucket.bind(postDataBucket),
             getRevision: postDataBucket.getRevision.bind(postDataBucket),
-            putRevision: postDataBucket.putRevision.bind(postDataBucket)
+            putRevision: postDataBucket.putRevision.bind(postDataBucket),
+            calculateHash: postDataBucket.calculateHash.bind(postDataBucket)
         }
     };
-};
+};

sys/post_data.yaml

@@ -14,4 +14,8 @@ paths:
 
   /{bucket}/{key}{/tid}:
     get:
-      operationId: getRevision
+      operationId: getRevision
+
+  /{bucket}/hash:
+    post:
+      operationId: calculateHash

test/features/mathoid.js

@@ -0,0 +1,79 @@
+'use strict';
+
+
+var assert = require('../utils/assert.js');
+var server = require('../utils/server.js');
+var preq   = require('preq');
+var P = require('bluebird');
+
+
+describe('Mathoid', function() {
+
+    var f = 'c^2 = a^2 + b^2';
+    var nf = 'c^{2}=a^{2}+b^{2}';
+    var uri = server.config.hostPort + '/wikimedia.org/v1/media/math';
+
+    this.timeout(20000);
+
+    before(function () { return server.start(); });
+
+    it('checks the formula with Mathoid', function() {
+        var slice = server.config.logStream.slice();
+        return preq.post({
+            uri: uri + '/check/tex',
+            headers: { 'content-type': 'application/json' },
+            body: { q: f }
+        }).then(function(res) {
+            slice.halt();
+            assert.localRequests(slice, false);
+            assert.remoteRequests(slice, true);
+            assert.checkString(res.headers['cache-control'], 'no-cache');
+        });
+    });
+
+    it('retrieves the check output from storage', function() {
+        var slice = server.config.logStream.slice();
+        return preq.post({
+            uri: uri + '/check/tex',
+            headers: { 'content-type': 'application/json' },
+            body: { q: f }
+        }).then(function(res) {
+            slice.halt();
+            assert.localRequests(slice, true);
+            assert.remoteRequests(slice, false);
+            assert.checkString(res.headers['cache-control'], 'no-cache');
+        });
+    });
+
+    it('retrieves the check output of the normalised version', function() {
+        var slice = server.config.logStream.slice();
+        return preq.post({
+            uri: uri + '/check/tex',
+            headers: { 'content-type': 'application/json' },
+            body: { q: nf }
+        }).then(function(res) {
+            slice.halt();
+            assert.localRequests(slice, true);
+            assert.remoteRequests(slice, false);
+            assert.checkString(res.headers['cache-control'], 'no-cache');
+        });
+    });
+
+    it('ignores stored version for no-cache', function() {
+        var slice = server.config.logStream.slice();
+        return preq.post({
+            uri: uri + '/check/tex',
+            headers: {
+                'content-type': 'application/json',
+                'cache-control': 'no-cache'
+            },
+            body: { q: f }
+        }).then(function(res) {
+            slice.halt();
+            assert.localRequests(slice, false);
+            assert.remoteRequests(slice, true);
+            assert.checkString(res.headers['cache-control'], 'no-cache');
+        });
+    });
+
+});

test/utils/assert.js

@@ -17,13 +17,24 @@ function contentType(res, expected) {
  * slice were routed to local recipients
  */
 function localRequests(slice, expected) {
+    var hasRec = false;
+    var localReqs = !slice.get().some(function(line) {
+        var entry = JSON.parse(line);
+        if (!entry.req) {
+            return false;
+        }
+        hasRec = true;
+        // if the URI starts with a slash,
+        // it's a local request
+        return !/^\//.test(entry.req.uri);
+    });
+    if (!hasRec) {
+        // there were no records in the slice, so
+        // we cannot really decide what that means
+        return;
+    }
     deepEqual(
-        !slice.get().some(function(line) {
-            var entry = JSON.parse(line);
-            // if the URI starts with a slash,
-            // it's a local request
-            return entry.req && !/^\//.test(entry.req.uri);
-        }),
+        localReqs,
         expected,
         expected ?
           'Should not have made local request' :
@@ -36,11 +47,22 @@ function localRequests(slice, expected) {
  * slice were made to remote entities
  */
 function remoteRequests(slice, expected) {
+    var hasRec = false;
+    var remoteReqs = slice.get().some(function(line) {
+        var entry = JSON.parse(line);
+        if (!entry.req) {
+            return false;
+        }
+        hasRec = true;
+        return entry.req && /^http/.test(entry.req.uri);
+    });
+    if (!hasRec) {
+        // there were no records in the slice, so
+        // we cannot really decide what that means
+        return;
+    }
     deepEqual(
-        slice.get().some(function(line) {
-            var entry = JSON.parse(line);
-            return entry.req && /^http/.test(entry.req.uri);
-        }),
+        remoteReqs,
         expected,
         expected ?
           'Should have made a remote request' :
@@ -125,6 +147,15 @@ function fails(promise, onRejected) {
     return promise.catch(trackFailure).then(check);
 }
 
+function checkString(result, expected, message) {
+    if (expected.constructor === RegExp) {
+        assert.ok(expected.test(result), '' + expected + '.test(' + result + ') fails');
+    } else {
+        var de = assert.deepStrictEqual || assert.deepEqual;
+        de(result, expected, expected + ' !== ' + result);
+    }
+}
+
 module.exports.ok             = assert.ok;
 module.exports.fails          = fails;
 module.exports.deepEqual      = deepEqual;
@@ -135,4 +166,4 @@ module.exports.contentType    = contentType;
 module.exports.localRequests  = localRequests;
 module.exports.remoteRequests = remoteRequests;
 module.exports.findParsoidRequest = findParsoidRequest;
-
+module.exports.checkString    = checkString;