feat: use transformer.js for embeddings (blib-la#74)

## Motivation

- added
[xenova/transformers.js](https://github.com/xenova/transformers.js)
- added CustomHuggingFaceTransformersEmbedding to truncate the input
text to maxTokens
- updated the vector-store test to use the new embedding
- added a test for the new embedding
- use the new embedding when creating the vector store

.github/workflows/test-e2e.yml

@@ -2,14 +2,14 @@ name: e2e Tests
 
 on:
   push:
-    branches: [ alpha, beta, rc, main ]
+    branches: [alpha, beta, rc, main]
   pull_request:
     types: [opened, synchronize]
-    branches: [ alpha, beta, rc, main ]
+    branches: [alpha, beta, rc, main]
 
 jobs:
   test-e2e:
-    runs-on: ubuntu-latest
+    runs-on: windows-latest
     strategy:
       matrix:
         node-version: [18.x]
@@ -20,7 +20,12 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+
       - name: Install dependencies
         run: npm install
+
+      - name: Fix auto-fixable eslint issues
+        run: npm run eslint -- --fix
+
       - name: Run e2e tests
         run: npm run test:e2e

jest.config.electron.e2e.ts

@@ -5,7 +5,15 @@ const jestConfig = {
 	roots: ["<rootDir>/src/electron"],
 	testMatch: ["**/*.test.e2e.ts"],
 	transform: {
-		"^.+\\.(ts|js)$": ["@swc/jest"],
+		"^.+\\.(ts)$": ["@swc/jest"],
+		"^.+\\.(js)$": [
+			"@swc/jest",
+			{
+				jsc: {
+					target: "es5",
+				},
+			},
+		],
 	},
 	moduleNameMapper: {
 		"@/(.*)": "<rootDir>/src/electron/future/$1",

package.json

@@ -21,7 +21,7 @@
     "spj": "npx sort-package-json@latest",
     "test:client": "jest --runInBand --config jest.config.client.ts --verbose",
     "pretest:e2e": "nextron build --no-pack",
-    "test:e2e": "xvfb-run --auto-servernum --server-args=\"-screen 0 1280x960x24\" -- cross-env TEST_ENV=test npx playwright test",
+    "test:e2e": "cross-env TEST_ENV=test npx playwright test",
     "test:e2e:local": "cross-env TEST_ENV=local npx playwright test",
     "test:electron": "jest --runInBand --config jest.config.electron.ts --verbose",
     "test:electron:e2e": "cross-env TEST_ENV=local jest --runInBand --config jest.config.electron.e2e.ts --verbose",
@@ -34,6 +34,7 @@
     "electron-dl": "3.5.2",
     "electron-serve": "1.3.0",
     "electron-store": "^8.2.0",
+    "onnxruntime-node": "^1.17.0",
     "sharp": "0.33.2"
   },
   "devDependencies": {
@@ -81,6 +82,7 @@
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^7.1.1",
     "@typescript-eslint/parser": "^7.1.1",
+    "@xenova/transformers": "github:xenova/transformers.js#v3",
     "axios": "^1.6.7",
     "codemirror": "^5.65.16",
     "conventional-changelog-cli": "^4.1.0",
@@ -151,8 +153,8 @@
     "tslib": "2.6.2",
     "type-fest": "4.12.0",
     "typescript": "^5.4.2",
-    "uuid": "^9.0.1",
     "use-debounce": "^10.0.0",
+    "uuid": "^9.0.1",
     "webpack": "^5.90.3"
   },
   "optionalDependencies": {