From fdac57c7244717e6f1520939a4eccc40dde4a885 Mon Sep 17 00:00:00 2001 From: harthur Date: Sun, 19 Sep 2010 00:48:12 -0700 Subject: [PATCH] add browser sanity test, move test types into subdirs --- README.md | 6 +- test/browser/brain-0.3.js | 644 ++++++++++++++++++ test/browser/browser.html | 13 + test/browser/browser.js | 84 +++ test/browser/underscore-min.js | 18 + test/{ => cvalidate}/cvtests.json | 0 test/{cvtests.js => cvalidate/runcv.js} | 0 test/{ => sanity}/bayesian/basictext.js | 0 test/{ => sanity}/bayesian/redis.js | 0 test/{ => sanity}/bayesian/thresholds.js | 0 test/{ => sanity}/neuralnetwork/bitwise.js | 0 .../{ => sanity}/neuralnetwork/errorthresh.js | 0 test/{ => sanity}/neuralnetwork/grow.js | 0 test/{ => sanity}/neuralnetwork/hash.js | 0 test/{ => sanity}/neuralnetwork/json.js | 0 test/{ => sanity}/neuralnetwork/layers.js | 0 test/{ => sanity}/neuralnetwork/tofunction.js | 0 test/{ => sanity}/runtests.js | 0 18 files changed, 762 insertions(+), 3 deletions(-) create mode 100644 test/browser/brain-0.3.js create mode 100644 test/browser/browser.html create mode 100644 test/browser/browser.js create mode 100644 test/browser/underscore-min.js rename test/{ => cvalidate}/cvtests.json (100%) rename test/{cvtests.js => cvalidate/runcv.js} (100%) rename test/{ => sanity}/bayesian/basictext.js (100%) rename test/{ => sanity}/bayesian/redis.js (100%) rename test/{ => sanity}/bayesian/thresholds.js (100%) rename test/{ => sanity}/neuralnetwork/bitwise.js (100%) rename test/{ => sanity}/neuralnetwork/errorthresh.js (100%) rename test/{ => sanity}/neuralnetwork/grow.js (100%) rename test/{ => sanity}/neuralnetwork/hash.js (100%) rename test/{ => sanity}/neuralnetwork/json.js (100%) rename test/{ => sanity}/neuralnetwork/layers.js (100%) rename test/{ => sanity}/neuralnetwork/tofunction.js (100%) rename test/{ => sanity}/runtests.js (100%) diff --git a/README.md b/README.md index a37d3dd..a7e56be 100644 --- a/README.md +++ b/README.md @@ -40,16 +40,16 @@ The `NeuralNetwork` works in the browser. Download the latest [brain.js](http:// # tests Running the tests requires [node.js](http://nodejs.org/). To run the suite of API tests: - node test/runtests.js + node test/sanity/runtests.js ### cross-validation tests The in-repo tests are just sanity/API checks, to really test out the library, run the cross-validation tests. These test the classifiers on large sets of real training data and give an error value (between 0 and 1) that indicates how good the classifier is at training. You can run the default cross-validation tests with: - node test/runcv.js + node test/cvalidate/runcv.js (requires network access to the dbs of training data). Specify your own db and options to pass in: - node test/runcv.js --type=neuralnetwork --db=http://localhost:5984/nndata --options='{learningRate:0.6}' + node test/cvalidate/runcv.js --type=neuralnetwork --db=http://localhost:5984/nndata --options='{learningRate:0.6}' The db must be a [CouchDB](http://couchdb.com) database of JSON objects with 'input' and 'output' fields. diff --git a/test/browser/brain-0.3.js b/test/browser/brain-0.3.js new file mode 100644 index 0000000..455386e --- /dev/null +++ b/test/browser/brain-0.3.js @@ -0,0 +1,644 @@ +/* +Copyright (c) 2010 Heather Arthur + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +var brain = (function(){ + var exports = {}; + + /* Neural Network */ + NeuralNetwork = function(options) { + this.learningRate = 0.5; + this.growthRate = 0.5; + if(options) + this.setOptions(options); + + this.createLayers(this.hidden); + } + + NeuralNetwork.prototype = { + setOptions : function(options) { + for(option in options) + this[option] = options[option]; + }, + + createLayers : function(hidden, json) { + var nlayers = 3; // one hidden layer is default + if(hidden) + nlayers = hidden.length + 2; + else if(json) + nlayers = json.layers.length; + + this.layers = []; + for(var i = 0; i < nlayers; i++) { + var nnodes = hidden ? hidden[i - 1] : 0; + var layerJSON = json ? json.layers[i] : null; + var layer = new Layer(this, layer, nnodes, layerJSON); + this.layers.push(layer); + } + + this.inputLayer = this.layers[0]; + this.outputLayer = this.layers[nlayers - 1]; + if(!hidden && !json) + this.hiddenLayer = this.layers[1]; // hold onto for growing + else + this.hiddenLayer = null; + }, + + run : function(inputs) { + this.inputLayer.createNodes(inputs); + if(this.hiddenLayer) + this.hiddenLayer.growLayer(this.inputLayer.getSize()); + + this.inputLayer.setOutputs(inputs); + for(var i = 1; i < this.layers.length; i++) + this.layers[i].calcOutputs(); + + var outputs = this.outputLayer.getOutputs(); + return this.formatOutput(outputs); + }, + + trainItem : function(inputs, targets) { + this.outputLayer.createNodes(targets); + + this.run(inputs); + + this.outputLayer.calcErrors(targets); + for(var i = this.layers.length - 2; i >= 0; i--) + this.layers[i].calcErrors(); + + for(var i = 1; i < this.layers.length; i++) + this.layers[i].adjustWeights(); + + return this.outputLayer.getError(); + }, + + train : function(data, iterations, errorThresh, callback, resolution) { + if(!iterations) + iterations = 20000; + if(!errorThresh) + errorThresh = 0.005; + + var error = 1; + for(var i = 0; i < iterations && error > errorThresh; i++) { + var sum = 0; + for(var j = 0; j < data.length; j++) { + var err = this.trainItem(data[j].input, data[j].output); + sum += Math.pow(err, 2); + } + error = Math.sqrt(sum) / data.length; // mean squared error + + if(callback && (i % resolution == 0)) + callback({error: error, iterations: i}); + } + return {error: error, iterations: i}; + }, + + trainAll : function(data) { // called by brain.crossValidate() + this.train(data); + }, + + getError : function(output, target) { + var error = 0, count = 0; + for(var id in output) { + error += Math.pow(output[id] - target[id], 2); + count++; + } + return error / count; // average mse + }, + + test : function(data) { + var error = 0; + for(var i = 0; i < data.length; i++) { + var output = this.run(data[i].input); + error += this.getError(output, data[i].output); + } + return error / data.length; // average error + }, + + formatOutput : function(outputs) { + /* we use hashes internally, turn back into array if needed */ + var values = []; + for(var id in outputs) { + if(parseInt(id) != id) // not an array index + return outputs; + values.push(outputs[id]); + } + return values; + }, + + toJSON : function() { + var json = {layers: []}; + for(var i = 0; i < this.layers.length; i++) + json.layers.push(this.layers[i].toJSON()); + return json; + }, + + fromJSON : function(json) { + this.createLayers(null, json); + return this; + }, + + toFunction: function() { + var json = this.toJSON(); + // currying w/ closures won't do, this needs to be standalone + return new Function("inputs", + ' var net = ' + JSON.stringify(json) + ';\n\n\ + for(var i = 1; i < net.layers.length; i++) {\n\ + var nodes = net.layers[i].nodes;\n\ + var outputs = {};\n\ + for(var id in nodes) {\n\ + var node = nodes[id];\n\ + var sum = node.bias;\n\ + for(var iid in node.weights)\n\ + sum += node.weights[iid] * inputs[iid];\n\ + outputs[id] = (1/(1 + Math.exp(-sum)));\n\ + }\n\ + inputs = outputs;\n\ + }\n\ + return outputs;'); + // note: this doesn't handle never-been-seen before inputs + }, + + toString : function() { + return JSON.stringify(this.toJSON()); + } + } + + function Layer(network, prevLayer, numNodes, json) { + this.network = network; + this.prevLayer = prevLayer; + if(this.prevLayer) + this.prevLayer.nextLayer = this; + + this.nodes = {}; + if(json) { + this.fromJSON(json); + } + else if(numNodes) { + for(var i = 0; i < numNodes; i++) + this.createNode(i); + } + } + + Layer.prototype = { + getOutputs : function() { // output is kept as state for backpropagation + return this.map(function(node) { return node.output; }); + }, + + setOutputs : function(outputs) { + this.map(function(node, id) { node.output = outputs[id] || 0; }); + }, + + getError : function() { + var sum = this.reduce(function(sum, node) { return sum + Math.pow(node.error, 2);}, 0); + return Math.sqrt(sum) / this.getSize(); // mean squared error + }, + + getSize : function() { + return this.reduce(function(count) { return ++count;}, 0); + }, + + map : function(callback) { + var values = {}; + for(var id in this.nodes) + values[id] = callback(this.nodes[id], id); + return values; + }, + + reduce : function(callback, value) { + for(var id in this.nodes) + value = callback(value, this.nodes[id]); + return value; + }, + + growLayer : function(inputSize) { + var targetSize = inputSize; + if(inputSize > 5) + targetSize *= this.network.growthRate; + for(var i = this.getSize(); i < targetSize; i++) + this.createNode(i); + }, + + createNodes : function(ids) { + for(var id in ids) { + if(!this.nodes[id]) + this.createNode(id); + } + }, + + createNode : function(id) { + var node = new Node(this, id); + this.nodes[id] = node; + + if(this.nextLayer) { + var outgoing = this.nextLayer.nodes; + for(var outid in outgoing) + outgoing[outid].addIncoming(id); + } + }, + + calcOutputs : function() { + this.map(function(node) { node.calcOutput(); }); + }, + + calcErrors : function(targets) { + this.map(function(node) { node.calcError(targets); }); + }, + + adjustWeights : function() { + this.map(function(node) { node.adjustWeights(); }); + }, + + toJSON : function() { + var json = { nodes: {}}; + for(var id in this.nodes) + json.nodes[id] = this.nodes[id].toJSON(); + return json; + }, + + fromJSON : function(json) { + this.nodes = {}; + for(var id in json.nodes) + this.nodes[id] = new Node(this, id, json.nodes[id]); + }, + } + + function Node(layer, id, json) { + this.layer = layer; + this.id = id; + this.output = 0; + + if(json) { + this.fromJSON(json); + } + else if(this.layer.prevLayer) { + this.weights = {}; + for(var id in this.getIncoming()) + this.addIncoming(id); + this.bias = this.randomWeight(); // instead of having a seperate bias node + } + } + + Node.prototype = { + getInputs : function() { return this.layer.prevLayer.getOutputs(); }, + + getIncoming : function() { return this.layer.prevLayer.nodes; }, + + getOutgoing : function() { return this.layer.nextLayer.nodes; }, + + randomWeight : function() { + return Math.random() * 0.4 - 0.2; + }, + + sigmoid : function(num) { + return 1/(1 + Math.exp(-num)); + }, + + dsigmoid : function(num) { + return num * (1 - num); + }, + + addIncoming : function(id) { + this.weights[id] = this.randomWeight(); + }, + + calcOutput : function() { + var sum = this.bias; + for(var id in this.weights) + sum += this.weights[id] * this.getInputs()[id]; + this.output = this.sigmoid(sum); + }, + + calcError : function(targets) { + if(targets) { + var expected = targets[this.id] || 0; + this.error = (expected - this.output); + } + else { + this.error = 0; + var outgoing = this.getOutgoing(); + for(var id in outgoing) + this.error += outgoing[id].delta * outgoing[id].weights[this.id]; + } + this.delta = this.error * this.dsigmoid(this.output); + }, + + adjustWeights : function() { + var rate = this.layer.network.learningRate; + for(var id in this.getInputs()) + this.weights[id] += rate * this.delta * this.getInputs()[id]; + this.bias += rate * this.delta; + }, + + toJSON : function() { + return { weights: this.weights, bias: this.bias }; + }, + + fromJSON : function(json) { + this.weights = json.weights; + this.bias = json.bias; + }, + } + + exports.NeuralNetwork = NeuralNetwork; + + /* Bayesian */ + var LocalStorageBackend = function(options) { + var options = options || {}; + var name = options.name || Math.floor(Math.random() * 100000); + + this.prefix = 'brain.bayesian.' + name; + + if(options.testing) + localStorage = {}; + } + + LocalStorageBackend.prototype = { + async : false, + + getCats : function() { + return JSON.parse(localStorage[this.prefix + '.cats'] || '{}'); + }, + + setCats : function(cats) { + localStorage[this.prefix + '.cats'] = JSON.stringify(cats); + }, + + getWordCount : function(word) { + return JSON.parse(localStorage[this.prefix + '.words.' + word] || '{}'); + }, + + setWordCount : function(word, counts) { + localStorage[this.prefix + '.words.' + word] = JSON.stringify(counts); + }, + + getWordCounts : function(words) { + var counts = {}; + words.forEach(function(word) { + counts[word] = this.getWordCount(word); + }, this); + return counts; + }, + + incCounts : function(catIncs, wordIncs) { + var cats = this.getCats(); + _(catIncs).each(function(inc, cat) { + cats[cat] = cats[cat] + inc || inc; + }, this); + this.setCats(cats); + + _(wordIncs).each(function(incs, word) { + var wordCounts = this.getWordCount(word); + _(incs).each(function(inc, cat) { + wordCounts[cat] = wordCounts[cat] + inc || inc; + }, this); + this.setWordCount(word, wordCounts); + }, this); + } + } + + var MemoryBackend = function() { + this.catCounts = {}; + this.wordCounts = {}; + } + + MemoryBackend.prototype = { + async : false, + + incCounts : function(catIncs, wordIncs) { + _(catIncs).each(function(inc, cat) { + this.catCounts[cat] = this.catCounts[cat] + inc || inc; + }, this); + + _(wordIncs).each(function(incs, word) { + this.wordCounts[word] = this.wordCounts[word] || {}; + _(incs).each(function(inc, cat) { + this.wordCounts[word][cat] = this.wordCounts[word][cat] + inc || inc; + }, this); + }, this); + }, + + getCats : function() { + return this.catCounts; + }, + + getWordCounts : function(words, cats) { + return this.wordCounts; + } + } + + BayesianClassifier = function(options) { + options = options || {} + this.thresholds = options.thresholds || {}; + this.def = options.def || 'unclassified'; + this.weight = options.weight || 1; + this.assumed = options.assumed || 0.5; + + var backend = options.backend || {type: 'memory'}; + switch(backend.type.toLowerCase()) { + case 'localstorage': + this.backend = new LocalStorageBackend(backend.options); + break; + default: + this.backend = new MemoryBackend(); + } + } + + BayesianClassifier.prototype = { + getCats : function(callback) { + return this.backend.getCats(callback); + }, + + getWordCounts : function(words, cats, callback) { + return this.backend.getWordCounts(words, cats, callback); + }, + + incDocCounts : function(docs, callback) { + // accumulate all the pending increments + var wordIncs = {}; + var catIncs = {}; + docs.forEach(function(doc) { + var cat = doc.cat; + catIncs[cat] = catIncs[cat] ? catIncs[cat] + 1 : 1; + + var words = this.getWords(doc.doc); + words.forEach(function(word) { + wordIncs[word] = wordIncs[word] || {}; + wordIncs[word][cat] = wordIncs[word][cat] ? wordIncs[word][cat] + 1 : 1; + }, this); + }, this); + + return this.backend.incCounts(catIncs, wordIncs, callback); + }, + + setThresholds : function(thresholds) { + this.thresholds = thresholds; + }, + + getWords : function(doc) { + if(_(doc).isArray()) + return doc; + var words = doc.split(/\W+/); + return _(words).uniq(); + }, + + train : function(doc, cat, callback) { + this.incDocCounts([{doc: doc, cat: cat}], function(err, ret) { + callback(ret); + }); + }, + + trainAll : function(data, callback) { + docs = data.map(function(item) { + return {doc: item.input, cat: item.output}; + }); + this.incDocCounts(docs, function(err, ret) { + callback(ret); + }); + }, + + wordProb : function(word, cat, cats, counts) { + // times word appears in a doc in this cat / docs in this cat + var prob = (counts[cat] || 0) / cats[cat]; + + // get weighted average with assumed so prob won't be extreme on rare words + var total = _(cats).reduce(function(sum, p, cat) { + return sum + (counts[cat] || 0); + }, 0, this); + return (this.weight * this.assumed + total * prob) / (this.weight + total); + }, + + getCatProbs : function(cats, words, counts) { + var numDocs = _(cats).reduce(function(sum, count) { + return sum + count; + }, 0); + + var probs = {}; + _(cats).each(function(catCount, cat) { + var catProb = (catCount || 0) / numDocs; + + var docProb = _(words).reduce(function(prob, word) { + var wordCounts = counts[word] || {}; + return prob * this.wordProb(word, cat, cats, wordCounts); + }, 1, this); + + // the probability this doc is in this category + probs[cat] = catProb * docProb; + }, this); + return probs; + }, + + getProbs : function(doc, callback) { + var that = this; + this.getCats(function(cats) { + var words = that.getWords(doc); + that.getWordCounts(words, cats, function(counts) { + var probs = that.getCatProbs(cats, words, counts); + callback(probs); + }); + }); + }, + + getProbsSync : function(doc, callback) { + var words = this.getWords(doc); + var cats = this.getCats(); + var counts = this.getWordCounts(words, cats); + return this.getCatProbs(cats, words, counts); + }, + + bestMatch : function(probs) { + var max = _(probs).reduce(function(max, prob, cat) { + return max.prob > prob ? max : {cat: cat, prob: prob}; + }, {prob: 0}); + + var category = max.cat; + var threshold = this.thresholds[max.cat] || 1; + _(probs).map(function(prob, cat) { + if(!(cat == max.cat) && prob * threshold > max.prob) + category = this.def; // not greater than other category by enough + }, this); + + return category; + }, + + classify : function(doc, callback) { + if(!this.backend.async) + return this.classifySync(doc); + + var that = this; + this.getProbs(doc, function(probs) { + callback(that.bestMatch(probs)); + }); + }, + + classifySync : function(doc) { + var probs = this.getProbsSync(doc); + return this.bestMatch(probs); + }, + + test : function(data) { // only for sync + var error = 0; + data.forEach(function(datum) { + var output = this.classify(datum.input); + error += output == datum.output ? 0 : 1; + }, this); + return error / data.length; + } + } + + exports.BayesianClassifier = BayesianClassifier; + + /* crossValidate */ + function testSet(classifierFunc, options, trainingSet, testingSet) { + var classifier = new classifierFunc(options); + var t1 = Date.now(); + classifier.trainAll(trainingSet); + var t2 = Date.now(); + var error = classifier.test(testingSet); + var t3 = Date.now(); + + return { + error : error, + trainTime : t2 - t1, + testTime : t3 - t2, + trainSize: trainingSet.length, + testSize: testingSet.length + }; + } + + var crossValidate = function(classifierFunc, options, data, slices) { + var sliceSize = data.length / slices; + var partitions = _.range(slices).map(function(i) { + var dclone = _(data).clone(); + return [dclone.splice(i * sliceSize, sliceSize), dclone]; + }); + + var results = _(partitions).map(function(partition, i) { + return testSet(classifierFunc, options, partition[1], partition[0]); + }); + return results; + } + + exports.crossValidate = crossValidate; + + return exports; +})(); \ No newline at end of file diff --git a/test/browser/browser.html b/test/browser/browser.html new file mode 100644 index 0000000..3612902 --- /dev/null +++ b/test/browser/browser.html @@ -0,0 +1,13 @@ + + +brain.js browser tests + + + + + +
+ Check the console for errors (: +
+ + diff --git a/test/browser/browser.js b/test/browser/browser.js new file mode 100644 index 0000000..5b79676 --- /dev/null +++ b/test/browser/browser.js @@ -0,0 +1,84 @@ +/* sanity tests for the browser */ + +var assert = { + ok : function(truth, message) { + if(!truth) + console.error("FAIL" + message); + }, + equal : function(a, b, message) { + if(a != b) + console.error("FAIL" + message); + } +}; + +console.log("brain.js tests started"); + +(function testNeural() { + var wiggle = 0.1; + + function testBitwise(data, op) { + var net = new brain.NeuralNetwork(); + net.train(data); + + for(var i in data) { + var output = net.run(data[i].input); + var target = data[i].output; + assert.ok(output < (target + wiggle) && output > (target - wiggle), + "failed to train " + op + " - output: " + output + " target: " + target); + } + } + + var and = [{input: [0, 0], output: [0]}, + {input: [0, 1], output: [0]}, + {input: [1, 0], output: [0]}, + {input: [1, 1], output: [1]}]; + testBitwise(and, "and"); + + var or = [{input: [0, 0], output: [0]}, + {input: [0, 1], output: [1]}, + {input: [1, 0], output: [1]}, + {input: [1, 1], output: [1]}]; + testBitwise(or, "or"); +})(); + +(function testBayesian() { + function testBasic(bayes) { + var spam = ["vicodin pharmacy", + "all quality replica watches marked down", + "cheap replica watches", + "receive more traffic by gaining a higher ranking in search engines", + "viagra pills", + "watches chanel tag heuer", + "watches at low prices"]; + + var not = ["unknown command line parameters", + "I don't know if this works on Windows", + "recently made changed to terms of service agreement", + "does anyone know about this", + "this is a bit out of date", + "the startup options need linking"] + + spam.forEach(function(text) { bayes.train(text, 'spam'); }); + not.forEach(function(text) { bayes.train(text, 'notspam'); }); + + assert.equal(bayes.classify("replica watches"),"spam"); + assert.equal(bayes.classify("check out the docs"), "notspam"); + assert.equal(bayes.classify("recently, I've been thinking that I should"), "notspam"); + assert.equal(bayes.classify("come buy these cheap pills"), "spam"); + } + + // test the synchronous backends + testBasic(new brain.BayesianClassifier()); + + testBasic(new brain.BayesianClassifier({ + backend : { + type: 'localStorage', + options: { + name: 'testnamespace', + testing: true + } + } + })); +})(); + +console.log("brain.js tests finished"); \ No newline at end of file diff --git a/test/browser/underscore-min.js b/test/browser/underscore-min.js new file mode 100644 index 0000000..18fed07 --- /dev/null +++ b/test/browser/underscore-min.js @@ -0,0 +1,18 @@ +(function(){var n=this,A=n._,r=typeof StopIteration!=="undefined"?StopIteration:"__break__",j=Array.prototype,l=Object.prototype,o=j.slice,B=j.unshift,C=l.toString,p=l.hasOwnProperty,s=j.forEach,t=j.map,u=j.reduce,v=j.reduceRight,w=j.filter,x=j.every,y=j.some,m=j.indexOf,z=j.lastIndexOf;l=Array.isArray;var D=Object.keys,b=function(a){return new k(a)};if(typeof exports!=="undefined")exports._=b;n._=b;b.VERSION="1.1.0";var i=b.forEach=function(a,c,d){try{if(s&&a.forEach===s)a.forEach(c,d);else if(b.isNumber(a.length))for(var e= +0,f=a.length;e=e.computed&&(e={value:f,computed:g})});return e.value};b.min=function(a,c,d){if(!c&&b.isArray(a))return Math.min.apply(Math,a);var e={computed:Infinity};i(a,function(f,g,h){g=c?c.call(d,f,g,h):f;gh? +1:0}),"value")};b.sortedIndex=function(a,c,d){d=d||b.identity;for(var e=0,f=a.length;e>1;d(a[g])=0})})};b.zip=function(){for(var a=b.toArray(arguments),c=b.max(b.pluck(a,"length")),d=new Array(c),e=0;e0?f-c:c-f)>=0)return e;e[g++]=f}};b.bind=function(a,c){var d=b.rest(arguments,2);return function(){return a.apply(c||{},d.concat(b.toArray(arguments)))}};b.bindAll=function(a){var c=b.rest(arguments);if(c.length==0)c=b.functions(a);i(c,function(d){a[d]=b.bind(a[d],a)});return a};b.memoize=function(a,c){var d={};c=c||b.identity;return function(){var e=c.apply(this,arguments);return e in +d?d[e]:(d[e]=a.apply(this,arguments))}};b.delay=function(a,c){var d=b.rest(arguments,2);return setTimeout(function(){return a.apply(a,d)},c)};b.defer=function(a){return b.delay.apply(b,[a,1].concat(b.rest(arguments)))};b.wrap=function(a,c){return function(){var d=[a].concat(b.toArray(arguments));return c.apply(c,d)}};b.compose=function(){var a=b.toArray(arguments);return function(){for(var c=b.toArray(arguments),d=a.length-1;d>=0;d--)c=[a[d].apply(this,c)];return c[0]}};b.keys=D||function(a){if(b.isArray(a))return b.range(0, +a.length);var c=[];for(var d in a)p.call(a,d)&&c.push(d);return c};b.values=function(a){return b.map(a,b.identity)};b.functions=function(a){return b.filter(b.keys(a),function(c){return b.isFunction(a[c])}).sort()};b.extend=function(a){i(b.rest(arguments),function(c){for(var d in c)a[d]=c[d]});return a};b.clone=function(a){if(b.isArray(a))return a.slice(0);return b.extend({},a)};b.tap=function(a,c){c(a);return a};b.isEqual=function(a,c){if(a===c)return true;var d=typeof a;if(d!=typeof c)return false; +if(a==c)return true;if(!a&&c||a&&!c)return false;if(a.isEqual)return a.isEqual(c);if(b.isDate(a)&&b.isDate(c))return a.getTime()===c.getTime();if(b.isNaN(a)&&b.isNaN(c))return false;if(b.isRegExp(a)&&b.isRegExp(c))return a.source===c.source&&a.global===c.global&&a.ignoreCase===c.ignoreCase&&a.multiline===c.multiline;if(d!=="object")return false;if(a.length&&a.length!==c.length)return false;d=b.keys(a);var e=b.keys(c);if(d.length!=e.length)return false;for(var f in a)if(!(f in c)||!b.isEqual(a[f], +c[f]))return false;return true};b.isEmpty=function(a){if(b.isArray(a)||b.isString(a))return a.length===0;for(var c in a)if(p.call(a,c))return false;return true};b.isElement=function(a){return!!(a&&a.nodeType==1)};b.isArray=l||function(a){return!!(a&&a.concat&&a.unshift&&!a.callee)};b.isArguments=function(a){return a&&a.callee};b.isFunction=function(a){return!!(a&&a.constructor&&a.call&&a.apply)};b.isString=function(a){return!!(a===""||a&&a.charCodeAt&&a.substr)};b.isNumber=function(a){return a=== ++a||C.call(a)==="[object Number]"};b.isBoolean=function(a){return a===true||a===false};b.isDate=function(a){return!!(a&&a.getTimezoneOffset&&a.setUTCFullYear)};b.isRegExp=function(a){return!!(a&&a.test&&a.exec&&(a.ignoreCase||a.ignoreCase===false))};b.isNaN=function(a){return b.isNumber(a)&&isNaN(a)};b.isNull=function(a){return a===null};b.isUndefined=function(a){return typeof a=="undefined"};b.noConflict=function(){n._=A;return this};b.identity=function(a){return a};b.times=function(a,c,d){for(var e= +0;e",interpolate:/<%=(.+?)%>/g};b.template=function(a,c){var d=b.templateSettings,e=new RegExp("'(?=[^"+d.end.substr(0,1)+"]*"+d.end.replace(/([.*+?^${}()|[\]\/\\])/g,"\\$1")+")","g");d=new Function("obj","var p=[],print=function(){p.push.apply(p,arguments);};with(obj||{}){p.push('"+a.replace(/\r/g, +"\\r").replace(/\n/g,"\\n").replace(/\t/g,"\\t").replace(e,"\u2704").split("'").join("\\'").split("\u2704").join("'").replace(d.interpolate,"',$1,'").split(d.start).join("');").split(d.end).join("p.push('")+"');}return p.join('');");return c?d(c):d};b.each=b.forEach;b.foldl=b.inject=b.reduce;b.foldr=b.reduceRight;b.select=b.filter;b.all=b.every;b.any=b.some;b.contains=b.include;b.head=b.first;b.tail=b.rest;b.methods=b.functions;var k=function(a){this._wrapped=a},q=function(a,c){return c?b(a).chain(): +a},E=function(a,c){k.prototype[a]=function(){var d=b.toArray(arguments);B.call(d,this._wrapped);return q(c.apply(b,d),this._chain)}};b.mixin(b);i(["pop","push","reverse","shift","sort","splice","unshift"],function(a){var c=j[a];k.prototype[a]=function(){c.apply(this._wrapped,arguments);return q(this._wrapped,this._chain)}});i(["concat","join","slice"],function(a){var c=j[a];k.prototype[a]=function(){return q(c.apply(this._wrapped,arguments),this._chain)}});k.prototype.chain=function(){this._chain= +true;return this};k.prototype.value=function(){return this._wrapped}})(); diff --git a/test/cvtests.json b/test/cvalidate/cvtests.json similarity index 100% rename from test/cvtests.json rename to test/cvalidate/cvtests.json diff --git a/test/cvtests.js b/test/cvalidate/runcv.js similarity index 100% rename from test/cvtests.js rename to test/cvalidate/runcv.js diff --git a/test/bayesian/basictext.js b/test/sanity/bayesian/basictext.js similarity index 100% rename from test/bayesian/basictext.js rename to test/sanity/bayesian/basictext.js diff --git a/test/bayesian/redis.js b/test/sanity/bayesian/redis.js similarity index 100% rename from test/bayesian/redis.js rename to test/sanity/bayesian/redis.js diff --git a/test/bayesian/thresholds.js b/test/sanity/bayesian/thresholds.js similarity index 100% rename from test/bayesian/thresholds.js rename to test/sanity/bayesian/thresholds.js diff --git a/test/neuralnetwork/bitwise.js b/test/sanity/neuralnetwork/bitwise.js similarity index 100% rename from test/neuralnetwork/bitwise.js rename to test/sanity/neuralnetwork/bitwise.js diff --git a/test/neuralnetwork/errorthresh.js b/test/sanity/neuralnetwork/errorthresh.js similarity index 100% rename from test/neuralnetwork/errorthresh.js rename to test/sanity/neuralnetwork/errorthresh.js diff --git a/test/neuralnetwork/grow.js b/test/sanity/neuralnetwork/grow.js similarity index 100% rename from test/neuralnetwork/grow.js rename to test/sanity/neuralnetwork/grow.js diff --git a/test/neuralnetwork/hash.js b/test/sanity/neuralnetwork/hash.js similarity index 100% rename from test/neuralnetwork/hash.js rename to test/sanity/neuralnetwork/hash.js diff --git a/test/neuralnetwork/json.js b/test/sanity/neuralnetwork/json.js similarity index 100% rename from test/neuralnetwork/json.js rename to test/sanity/neuralnetwork/json.js diff --git a/test/neuralnetwork/layers.js b/test/sanity/neuralnetwork/layers.js similarity index 100% rename from test/neuralnetwork/layers.js rename to test/sanity/neuralnetwork/layers.js diff --git a/test/neuralnetwork/tofunction.js b/test/sanity/neuralnetwork/tofunction.js similarity index 100% rename from test/neuralnetwork/tofunction.js rename to test/sanity/neuralnetwork/tofunction.js diff --git a/test/runtests.js b/test/sanity/runtests.js similarity index 100% rename from test/runtests.js rename to test/sanity/runtests.js