Merge c079cf6 into 48b03dc

51030b0f-947e-4e65-9322-1197651bc555.json

@@ -0,0 +1 @@
+{"_events":{},"_eventsCount":0,"docs":[],"features":{},"stemmer":{},"lastAdded":0}

fc938015-69f7-43df-a926-b5dc5363a770.json

@@ -0,0 +1 @@
+{"_events":{},"_eventsCount":0,"docs":[],"features":{},"stemmer":{},"lastAdded":0}

io_spec/bayes_classifier_spec.js

@@ -82,7 +82,7 @@ describe('Bayes classifier file I/O', function () {
       sandbox.restore()
     })
     it('should pass an error to the callback function', function () {
-      sandbox.stub(baseClassifier, 'load', function (filename, cb) {
+      sandbox.stub(baseClassifier, 'load', function (filename, stemmer, cb) {
         cb(new Error('An error occurred'))
       })
       natural.BayesClassifier.load('/spec/test_data/tfidf_document1.txt', null, function (err, newClassifier) {

lib/natural/classifiers/bayes_classifier.js

@@ -44,7 +44,7 @@ class BayesClassifier extends Classifier {
   }
 
   static load (filename, stemmer, callback) {
-    Classifier.load(filename, function (err, classifier) {
+    Classifier.load(filename, stemmer, function (err, classifier) {
       if (err) {
         return callback(err)
       } else {

lib/natural/classifiers/classifier.js

@@ -21,6 +21,7 @@ THE SOFTWARE.
 */
 
 'use strict'
+const fs = require('fs')
 
 const EventEmitter = require('events')
 const PorterStemmer = require('../stemmers/porter_stemmer')
@@ -34,6 +35,11 @@ class Classifier extends EventEmitter {
     this.features = {}
     this.stemmer = stemmer || PorterStemmer
     this.lastAdded = 0
+
+    // Add methods for parallel training
+    this.trainParallel = parallelTrainer.trainParallel
+    this.retrainParallel = parallelTrainer.retrainParallel
+    this.trainParallelBatches = parallelTrainer.trainParallelBatches
   }
 
   addDocument (text, classification) {
@@ -114,8 +120,8 @@ class Classifier extends EventEmitter {
       this.emit('trainedWithDocument', { index: i, total: totalDocs, doc: this.docs[i] })
       this.lastAdded++
     }
-    this.emit('doneTraining', true)
     this.classifier.train()
+    this.emit('doneTraining', true)
   }
 
   retrain () {
@@ -139,7 +145,6 @@ class Classifier extends EventEmitter {
 
   save (filename, callback) {
     const data = JSON.stringify(this)
-    const fs = require('fs')
     const classifier = this
     fs.writeFile(filename, data, 'utf8', function (err) {
       if (callback) {
@@ -148,9 +153,7 @@ class Classifier extends EventEmitter {
     })
   }
 
-  static load (filename, callback) {
-    const fs = require('fs')
-
+  static load (filename, stemmer, callback) {
     if (!callback) {
       return
     }
@@ -159,7 +162,7 @@ class Classifier extends EventEmitter {
         callback(err, null)
       } else {
         const classifier = JSON.parse(data)
-        callback(err, classifier)
+        callback(err, Classifier.restore(classifier, stemmer))
       }
     })
   }
@@ -177,33 +180,6 @@ class Classifier extends EventEmitter {
   setOptions (options) {
     this.keepStops = !!(options.keepStops)
   }
-
-  ClassifiertrainParallel (numThreads, callback) {
-    if (parallelTrainer.Threads) {
-      return parallelTrainer.trainParallel(numThreads, callback)
-    } else {
-      this.emit('No threads available')
-      return this.train()
-    }
-  }
-
-  trainParallelBatches (options) {
-    if (parallelTrainer.Threads) {
-      return parallelTrainer.trainParallelBatches(options)
-    } else {
-      this.emit('No threads available')
-      return this.train()
-    }
-  }
-
-  retrainParallel (numThreads, callback) {
-    if (parallelTrainer.Threads) {
-      return parallelTrainer.trainParallel(numThreads, callback)
-    } else {
-      this.emit('No threads available')
-      return this.retrain()
-    }
-  }
 }
 
 module.exports = Classifier

lib/natural/classifiers/classifier_train_parallel.js

@@ -5,8 +5,8 @@ let Threads = null
 try {
   Threads = require('webworker-threads')
 } catch (e) {
-  // Since webworker-threads are optional, only thow if the module is found
-  if (e.code !== 'MODULE_NOT_FOUND') throw e
+  // Silently set Threads to null
+  Threads = null
 }
 
 function checkThreadSupport () {
@@ -266,6 +266,8 @@ function trainParallelBatches (options) {
 }
 
 function retrainParallel (numThreads, callback) {
+  checkThreadSupport()
+
   this.classifier = new (this.classifier.constructor)()
   this.lastAdded = 0
   this.trainParallel(numThreads, callback)

lib/natural/classifiers/index.d.ts

@@ -1,6 +1,7 @@
 import events from 'events'
 import { Corpus, Sentence } from '../brill_pos_tagger'
 import { Stemmer } from '../stemmers'
+import { StorageBackend } from '../util'
 
 // Start apparatus declarations
 
@@ -71,6 +72,7 @@ declare interface ClassifierOptions {
 }
 
 declare type ClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: ClassifierBase) => void
+declare type ParallelTrainerCallback = (err: NodeJS.ErrnoException | null) => void
 
 declare class ClassifierBase extends events.EventEmitter {
   classifier: ApparatusClassifier
@@ -89,14 +91,25 @@ declare class ClassifierBase extends events.EventEmitter {
   classify (observation: string | string[]): string
   setOptions (options: ClassifierOptions): void
   save (filename: string, callback?: ClassifierCallback): void
+  static load (filename: string, stemmer: Stemmer | null | undefined, callback: ClassifierCallback): void
+  saveTo (storage: StorageBackend): String
+  static loadFrom (storage: StorageBackend): ClassifierBase
+
+  trainParallel (numThreads: number, callback: ParallelTrainerCallback): void
+  trainParallelBatches (options: { numThreads: number, batchSize: number }): void
+  retrainParallel (numThreads: number, callback: ParallelTrainerCallback): void
 }
 
 declare type BayesClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: BayesClassifier) => void
 
 export class BayesClassifier extends ClassifierBase {
+  classifier: ApparatusBayesClassifier
+
   constructor (stemmer?: Stemmer, smoothing?: number)
   static load (filename: string, stemmer: Stemmer | null | undefined, callback: BayesClassifierCallback): void
   static restore (classifier: BayesClassifier, stemmer?: Stemmer): BayesClassifier
+  saveTo (storage: StorageBackend): String
+  static loadFrom (storage: StorageBackend): ClassifierBase
 }
 
 declare type LogisticRegressionClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: LogisticRegressionClassifier) => void
@@ -105,6 +118,8 @@ export class LogisticRegressionClassifier extends ClassifierBase {
   constructor (stemmer?: Stemmer)
   static load (filename: string, stemmer: Stemmer | null | undefined, callback: LogisticRegressionClassifierCallback): void
   static restore (classifier: LogisticRegressionClassifier, stemmer?: Stemmer): LogisticRegressionClassifier
+  saveTo (storage: StorageBackend): String
+  static loadFrom (storage: StorageBackend): ClassifierBase
 }
 
 declare type MaxEntClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: MaxEntClassifier | null) => void

lib/natural/util/index.d.ts

@@ -70,3 +70,5 @@ export class LongestPathTree {
   hasPathTo (vertex: number): boolean
   pathTo (vertex: number): number[]
 }
+
+export * from './storage'

lib/natural/util/storage/index.d.ts

@@ -0,0 +1,39 @@
+/*
+Copyright (c) 2024, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+export declare enum STORAGE_TYPES {
+  POSTGRES = 'POSTGRES',
+  REDIS = 'REDIS',
+  MONGODB = 'MONGODB',
+  MEMCACHED = 'MEMCACHED',
+  FILE = 'FILE'
+}
+
+export class StorageBackend {
+  private readonly storageType: STORAGE_TYPES
+  private readonly client: any
+
+  constructor (type: STORAGE_TYPES)
+  setStorageType (storageType: STORAGE_TYPES): void
+  store (key: string, value: any): Promise<String>
+  retrieve (key: string): Promise<any>
+}

package.json

@@ -11,6 +11,7 @@
     "node": ">=0.4.10"
   },
   "dependencies": {
+    "@types/jasmine": "^5.1.4",
     "afinn-165": "^1.0.2",
     "afinn-165-financialmarketnews": "^3.0.0",
     "apparatus": "^0.0.10",
@@ -63,6 +64,7 @@
   },
   "scripts": {
     "benchmark": "node benchmarks",
+    "build": "tsc",
     "clean": "rimraf *~ #* *classifier.json",
     "test": "NODE_PATH=. node ./node_modules/jasmine/bin/jasmine.js --random=false spec/*_spec.js",
     "coverage": "nyc --reporter=lcov npm run test && nyc npm run test_io",

spec/bayes_classifier_spec.js

@@ -47,7 +47,7 @@ describe('bayes classifier', function () {
     it('should classify with parallel training', function () {
       const classifier = setupClassifier()
       // Check for parallel method
-      if (classifier.trainParallel) {
+      if (classifier.Threads) {
         classifier.trainParallel(2, function (err) {
           if (err) {
             console.log(err)
@@ -62,7 +62,7 @@ describe('bayes classifier', function () {
 
     it('should classify with parallel batched training', function () {
       const classifier = setupClassifier()
-      if (classifier.trainParallelBatches) {
+      if (classifier.Threads) {
         // Check for parallel method
         classifier.on('doneTraining', function () {
           expect(classifier.classify(['bug', 'code'])).toBe('computing')