chore/benchmark

.github/workflows/nodejs.yml

@@ -48,6 +48,45 @@ jobs:
         path: dist
     - run: npm test
 
+  benchmark:
+    needs: build
+    strategy:
+      matrix:
+        node-version: [ 20.x ]
+        os: [ ubuntu-latest ]
+
+    runs-on: ${{ matrix.os }}
+
+    timeout-minutes: 30
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Benchmarking Node.js ${{ matrix.node-version }} on ${{ matrix.os }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+    - run: npm ci
+    - uses: actions/download-artifact@v3
+      with:
+        name: dist
+        path: dist
+    - name: Run benchmark
+      run: npm run perf
+    - name: Store benchmark result
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        name: EYE JS Benchmark
+        tool: 'benchmarkjs'
+        output-file-path: perf/output.txt
+        github-token: ${{ secrets.GH_TOKEN }}
+        comment-on-alert: true
+        summary-always: true
+        alert-comment-cc-users: '@jeswr'
+        alert-threshold: "110%"
+        fail-threshold: "125%"
+        gh-pages-branch: bench-pages
+        auto-push: true
+
   eslint:
     runs-on: ubuntu-latest
     steps:
@@ -64,7 +103,7 @@ jobs:
 
   release:
     if: ${{ github.ref == 'refs/heads/main' }}
-    needs: [ eslint, test ]
+    needs: [ eslint, test, benchmark ]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.gitignore

@@ -3,3 +3,4 @@ coverage
 dist
 bundle
 lib/eye.ts
+perf/output.txt

package.json

@@ -29,7 +29,7 @@
     "eye:pvm:test": "ts-node scripts/run-pvm",
     "eye:prepare": "npm run eye:pvm",
     "eye:update": "ts-node scripts/update",
-    "perf": "ts-node perf/socrates"
+    "perf": "ts-node perf/bench 2>&1 | tee perf/output.txt"
   },
   "repository": {
     "type": "git",
@@ -60,12 +60,14 @@
     "@memlab/api": "^1.0.20",
     "@qiwi/semantic-release-gh-pages-plugin": "^5.2.4",
     "@rollup/plugin-commonjs": "^24.0.1",
+    "@types/benchmark": "^2.1.2",
     "@types/express": "^4.17.17",
     "@types/fs-extra": "^11.0.1",
     "@types/jest": "^29.4.0",
     "@types/n3": "^1.10.4",
     "@typescript-eslint/eslint-plugin": "^5.51.0",
     "@typescript-eslint/parser": "^5.51.0",
+    "benchmark": "^2.1.4",
     "cross-fetch": "^3.1.5",
     "deep-taxonomy-benchmark": "^1.1.1",
     "eslint": "^8.34.0",

perf/bench.ts

@@ -0,0 +1,71 @@
+import Benchmark, { type Event } from 'benchmark';
+import { data, query } from '../data/socrates';
+import { queryOnce, SwiplEye, n3reasoner } from '../dist';
+import { generateDeepTaxonomy } from 'deep-taxonomy-benchmark/dist';
+import { Parser } from 'n3';
+
+const suite = new Benchmark.Suite;
+
+const deepTaxonomyBenchmark10 = [
+  ...generateDeepTaxonomy(10, true),
+  ...(new Parser({ format: 'n3' })).parse('{ ?s a ?o . ?o <http://www.w3.org/2000/01/rdf-schema#subClassOf> ?o2 . } => { ?s a ?o2 . } .'),
+]
+
+const deepTaxonomyBenchmark50 = [
+  ...generateDeepTaxonomy(50, true),
+  ...(new Parser({ format: 'n3' })).parse('{ ?s a ?o . ?o <http://www.w3.org/2000/01/rdf-schema#subClassOf> ?o2 . } => { ?s a ?o2 . } .'),
+]
+
+const deepTaxonomyBenchmark100 = [
+  ...generateDeepTaxonomy(100, true),
+  ...(new Parser({ format: 'n3' })).parse('{ ?s a ?o . ?o <http://www.w3.org/2000/01/rdf-schema#subClassOf> ?o2 . } => { ?s a ?o2 . } .'),
+]
+
+function deferred(fn: () => Promise<any>): Benchmark.Options {
+  return {
+    defer: true,
+    fn: (deferred: { resolve: () => void }) => fn().then(() => deferred.resolve())
+  }
+}
+
+// For performance testing prior to migrating to benchmark.js see
+// https://github.com/eyereasoner/eye-js/blob/87147784aa1c91f1d42092d703ff554c1a7c5a34/perf/socrates.ts#L6-L12
+async function main() {
+  const Module = await SwiplEye({ print: () => { } });
+
+  const LoadedModule = await SwiplEye({ print: () => { } });
+  LoadedModule.FS.writeFile('data.n3', data);
+  LoadedModule.FS.writeFile('query.n3', query);
+
+  // add tests
+  suite
+    .add(
+      'Initialise SWIPL with EYE image',
+      deferred(() => SwiplEye({ print: () => { } })),
+    ).add(
+      'Run socrates query',
+      deferred(() => n3reasoner(data, query)),
+    ).add(
+      'Load data into a module',
+      () => Module.FS.writeFile('data.n3', data),
+    ).add(
+      'Load query into a module',
+      () => Module.FS.writeFile('query.n3', query),
+    ).add(
+      'Executing the socrates query',
+      () => queryOnce(LoadedModule, 'main', ['--nope', '--quiet', './data.n3', '--query', './query.n3']),
+    ).add(
+      'Run deep taxonomy benchmark [10]',
+      deferred(() => n3reasoner(deepTaxonomyBenchmark10)),
+    ).add(
+      'Run deep taxonomy benchmark [50]',
+      deferred(() => n3reasoner(deepTaxonomyBenchmark50)),
+    ).add(
+      'Run deep taxonomy benchmark [100]',
+      deferred(() => n3reasoner(deepTaxonomyBenchmark100)),
+    ).on('cycle', (event: Event) => {
+      console.log(event.target.toString());
+    }).run();
+}
+
+main();