feat: Add the logistic regression

Author: jajoe

__tests__/test.js

@@ -0,0 +1,35 @@
+const {Matrix} = require('ml-matrix');
+const LogisticRegressionTwoClasses = require('../src/logreg_2classes');
+const LogisticRegression = require('../src/logreg');
+
+// test for 2 classes
+
+var X = new Matrix([[0,-1], [1,0], [1,1], [1,-1], [2,0], [2,1], [2,-1], [3,2], [0,4], [1,3], [1,4], [1,5], [2,3], [2,4], [2,5], [3,4]]);
+var Y = Matrix.columnVector([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]);
+
+var Xtest = new Matrix([[0, -2], [1, 0.5], [1.5, -1], [1, 4.5], [2, 3.5], [1.5, 5]])
+var Ytest = Matrix.columnVector([0, 0, 0, 1, 1, 1]);
+
+var logreg = new LogisticRegressionTwoClasses(num_steps = 500000, learning_rate = 5e-4);
+logreg.train(X, Y);
+// console.log(logreg.weights); // results should be [-12.07083366   9.94427349]
+var results = logreg.test(Xtest); // compute results of the training set
+console.log(results);
+
+
+
+
+
+
+// test for 3 classes
+
+var X = new Matrix([[0,-1], [1,0], [1,1], [1,-1], [2,0], [2,1], [2,-1], [3,2], [0,4], [1,3], [1,4], [1,5], [2,3], [2,4], [2,5], [3,4], [1, 10], [1, 12], [2, 10], [2,11], [2, 14], [3, 11]]);
+var Y = Matrix.columnVector([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2]);
+
+var Xtest = new Matrix([[0, -2], [1, 0.5], [1.5, -1], [1, 2.5], [2, 3.5], [1.5, 4], [1, 10.5], [2.5, 10.5], [2, 11.5]])
+var Ytest = Matrix.columnVector([0, 0, 0, 1, 1, 1, 2, 2, 2]);
+
+var logreg = new LogisticRegression(num_steps = 10000000, learning_rate = 5e-5);
+logreg.train(X,Y);
+var finalResults = logreg.test(Xtest);
+console.log(finalResults);

package.json

@@ -0,0 +1,14 @@
+{
+  "name": "logreg",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "jajoe",
+  "license": "MIT",
+  "dependencies": {
+    "ml-matrix": "^3.0.0"
+  }
+}

src/logreg.js

@@ -0,0 +1,58 @@
+const {Matrix} = require('ml-matrix');
+const LogisticRegressionTwoClasses = require('./logreg_2classes');
+
+function transform_classes_for_one_vs_all(Y, oneClass){
+    y = Y.to1DArray();
+    for (var i = 0; i < y.length; i++){
+        if (y[i] == oneClass) {
+            y[i] = 0;
+        } else {
+            y[i] = 1;
+        }
+    }
+    return Matrix.columnVector(y);
+}
+
+class LogisticRegression {
+
+    constructor(options) {
+        options = options || {};
+        this.numSteps = options. numSteps || 500000;
+        this.learningRate = options.learningRate || 5e-4;
+        this.classifiers = [];
+        this.numberClasses = 0;
+    }
+
+    train(X, Y) {
+        this.numberClasses = new Set(Y).size;
+        this.classifiers = new Array(this.numberClasses);
+
+        // train the classifiers
+        for (var i = 0; i < this.numberClasses; i++){
+            this.classifiers[i] = new LogisticRegressionTwoClasses({numSteps: this.numSteps, learningRate: this.learningRate});
+            var y = Y.clone();
+            y = transform_classes_for_one_vs_all(y, i);
+            this.classifiers[i].train(X,y);
+        }
+    }
+
+    test(Xtest) {
+        var resultsOneClass = new Array(this.numberClasses).fill(0);
+        for (var i = 0; i < this.numberClasses; i++){
+            resultsOneClass[i]= this.classifiers[i].testScores(Xtest);
+        }
+        var finalResults = new Array(Xtest.rows).fill(0);
+        for (var i = 0; i < Xtest.rows; i++){
+            var minimum = 100000;
+            for (var j = 0; j < this.numberClasses; j++){
+                if (resultsOneClass[j][i] < minimum) {
+                    minimum = resultsOneClass[j][i];
+                    finalResults[i] = j;
+                }
+            }
+        }
+        return finalResults;
+    }
+}
+
+module.exports = LogisticRegression;

src/logreg_2classes.js

@@ -0,0 +1,52 @@
+const {Matrix} = require('ml-matrix');
+
+class LogisticRegressionTwoClasses {
+
+    constructor(options) {
+        options = options || {};
+        this.numSteps = options.numSteps || 500000;
+        this.learningRate = options.learningRate || 5e-4;
+        this.weights = [];
+    }
+
+    sigmoid(scores) {
+        scores = scores.to1DArray();
+        var result = [];
+        for (var i = 0; i < scores.length; i++) {
+            result.push(1 / (1 + Math.exp(-scores[i])));
+        }
+        return result;
+    }
+
+    train(features, target) {
+        var weights = Matrix.zeros(1, features.columns);
+    
+        for (var step = 0; step < this.numSteps; step++) {
+            var scores = features.mmul(weights.transpose());
+            var predictions = this.sigmoid(scores);
+
+            // Update weights with gradient
+            var output_error_signal = Matrix.columnVector(predictions).neg().add(target);
+            var gradient = features.transpose().mmul(output_error_signal);
+            weights = weights.add(gradient.mul(this.learningRate).transpose());
+        }
+
+        this.weights = weights
+    }
+
+    testScores(features) {
+        var final_data = features.mmul(this.weights.transpose());
+        var predictions = this.sigmoid(final_data);
+        predictions = Matrix.columnVector(predictions);
+        return predictions.to1DArray();
+    }
+
+    test(features) {
+        var final_data = features.mmul(this.weights.transpose());
+        var predictions = this.sigmoid(final_data);
+        predictions = Matrix.columnVector(predictions).round();
+        return predictions.to1DArray();
+    }
+}
+
+module.exports = LogisticRegressionTwoClasses;