From 7758eda4b199de66ce9d85e20eff60ff5865b84c Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Thu, 18 Jan 2024 06:33:55 +0700 Subject: [PATCH 01/49] [janhq/jan#1635] feat(WIP): decouple nitro engine into a library - The library can be built, but with local dependency on janhq/jan as submodule - Next step is update janhq/core to npmjs and use the resolved package there instead of submodule - TODO: update GitHub-action workflow to build nitro-node for different OSes and add the tarballs of npm packages to release - TODO: from janhq/jan build step for extensions, simply download nitro-node from github release --- .github/scripts/auto-sign.sh | 10 + nitro-node/.gitignore | 23 ++ nitro-node/.npmrc | 2 + nitro-node/.yarnrc.yml | 1 + nitro-node/Makefile | 46 +++ nitro-node/README.md | 78 +++++ nitro-node/download.bat | 3 + nitro-node/jan | 1 + nitro-node/package.json | 57 ++++ nitro-node/src/@types/global.d.ts | 33 ++ nitro-node/src/helpers/sse.ts | 65 ++++ nitro-node/src/index.ts | 279 ++++++++++++++++ nitro-node/src/module.ts | 514 ++++++++++++++++++++++++++++++ nitro-node/tsconfig.json | 15 + nitro-node/webpack.config.js | 43 +++ yarn.lock | 4 - 16 files changed, 1170 insertions(+), 4 deletions(-) create mode 100755 .github/scripts/auto-sign.sh create mode 100644 nitro-node/.gitignore create mode 100644 nitro-node/.npmrc create mode 100644 nitro-node/.yarnrc.yml create mode 100644 nitro-node/Makefile create mode 100644 nitro-node/README.md create mode 100644 nitro-node/download.bat create mode 160000 nitro-node/jan create mode 100644 nitro-node/package.json create mode 100644 nitro-node/src/@types/global.d.ts create mode 100644 nitro-node/src/helpers/sse.ts create mode 100644 nitro-node/src/index.ts create mode 100644 nitro-node/src/module.ts create mode 100644 nitro-node/tsconfig.json create mode 100644 nitro-node/webpack.config.js delete mode 100644 yarn.lock diff --git a/.github/scripts/auto-sign.sh b/.github/scripts/auto-sign.sh new file mode 100755 index 000000000..5e6ef9750 --- /dev/null +++ b/.github/scripts/auto-sign.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Check if both APP_PATH and DEVELOPER_ID environment variables are set +if [[ -z "$APP_PATH" ]] || [[ -z "$DEVELOPER_ID" ]]; then + echo "Either APP_PATH or DEVELOPER_ID is not set. Skipping script execution." + exit 0 +fi + +# If both variables are set, execute the following commands +find "$APP_PATH" \( -type f -perm +111 -o -name "*.node" \) -exec codesign -s "$DEVELOPER_ID" --options=runtime {} \; diff --git a/nitro-node/.gitignore b/nitro-node/.gitignore new file mode 100644 index 000000000..56913fe29 --- /dev/null +++ b/nitro-node/.gitignore @@ -0,0 +1,23 @@ +.vscode +.env + +# Jan inference +error.log +.yarn +node_modules +*.tgz +yarn.lock +dist +build +.DS_Store +package-lock.json + +*.log + +# Nitro binary files +bin/*/nitro +bin/*/*.metal +bin/*/*.exe +bin/*/*.dll +bin/*/*.exp +bin/*/*.lib \ No newline at end of file diff --git a/nitro-node/.npmrc b/nitro-node/.npmrc new file mode 100644 index 000000000..d84128b58 --- /dev/null +++ b/nitro-node/.npmrc @@ -0,0 +1,2 @@ +scripts-prepend-node-path=true +engine-strict=true diff --git a/nitro-node/.yarnrc.yml b/nitro-node/.yarnrc.yml new file mode 100644 index 000000000..3186f3f07 --- /dev/null +++ b/nitro-node/.yarnrc.yml @@ -0,0 +1 @@ +nodeLinker: node-modules diff --git a/nitro-node/Makefile b/nitro-node/Makefile new file mode 100644 index 000000000..c39b2af53 --- /dev/null +++ b/nitro-node/Makefile @@ -0,0 +1,46 @@ +# Makefile for Nitro-node - Build and Clean + +# Default target, build all +.PHONY: all + +all: publish + +# Build jan/core +build-core: + git submodule update --init +ifeq ($(OS),Windows_NT) + type NUL > jan/yarn.lock +else + touch jan/yarn.lock +endif + cd jan && yarn install + cd jan/core && yarn build + +# Installs yarn dependencies +install: build-core +ifeq ($(OS),Windows_NT) + yarn config set network-timeout 300000 +endif + yarn install + +# Build +build: install + yarn run build + +# Download Nitro +download-nitro: install + yarn run downloadnitro + +# Builds and publishes the extension +publish: build download-nitro + yarn run publish + +clean: +ifeq ($(OS),Windows_NT) + del /S *.tgz + powershell -Command "Get-ChildItem -Path . -Include node_modules, dist -Recurse -Directory | Remove-Item -Recurse -Force" +else + rm -f *.tgz + find . -name "node_modules" -type d -prune -exec rm -rf '{}' + + find . -name "dist" -type d -exec rm -rf '{}' + +endif diff --git a/nitro-node/README.md b/nitro-node/README.md new file mode 100644 index 000000000..455783efb --- /dev/null +++ b/nitro-node/README.md @@ -0,0 +1,78 @@ +# Jan inference plugin + +Created using Jan app example + +# Create a Jan Plugin using Typescript + +Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 + +## Create Your Own Plugin + +To create your own plugin, you can use this repository as a template! Just follow the below instructions: + +1. Click the Use this template button at the top of the repository +2. Select Create a new repository +3. Select an owner and name for your new repository +4. Click Create repository +5. Clone your new repository + +## Initial Setup + +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. + +> [!NOTE] +> +> You'll need to have a reasonably modern version of +> [Node.js](https://nodejs.org) handy. If you are using a version manager like +> [`nodenv`](https://github.com/nodenv/nodenv) or +> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the +> root of your repository to install the version specified in +> [`package.json`](./package.json). Otherwise, 20.x or later should work! + +1. :hammer_and_wrench: Install the dependencies + + ```bash + npm install + ``` + +1. :building_construction: Package the TypeScript for distribution + + ```bash + npm run bundle + ``` + +1. :white_check_mark: Check your artifact + + There will be a tgz file in your plugin directory now + +## Update the Plugin Metadata + +The [`package.json`](package.json) file defines metadata about your plugin, such as +plugin name, main entry, description and version. + +When you copy this repository, update `package.json` with the name, description for your plugin. + +## Update the Plugin Code + +The [`src/`](./src/) directory is the heart of your plugin! This contains the +source code that will be run when your plugin extension functions are invoked. You can replace the +contents of this directory with your own code. + +There are a few things to keep in mind when writing your plugin code: + +- Most Jan Plugin Extension functions are processed asynchronously. + In `index.ts`, you will see that the extension function will return a `Promise`. + + ```typescript + import { core } from "@janhq/core"; + + function onStart(): Promise { + return core.invokePluginFunc(MODULE_PATH, "run", 0); + } + ``` + + For more information about the Jan Plugin Core module, see the + [documentation](https://github.com/janhq/jan/blob/main/core/README.md). + +So, what are you waiting for? Go ahead and start customizing your plugin! + diff --git a/nitro-node/download.bat b/nitro-node/download.bat new file mode 100644 index 000000000..22e1c85b3 --- /dev/null +++ b/nitro-node/download.bat @@ -0,0 +1,3 @@ +@echo off +set /p NITRO_VERSION=<./bin/version.txt +.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu diff --git a/nitro-node/jan b/nitro-node/jan new file mode 160000 index 000000000..42a86e9cb --- /dev/null +++ b/nitro-node/jan @@ -0,0 +1 @@ +Subproject commit 42a86e9cb71a195a58d4050b6f8649d080155382 diff --git a/nitro-node/package.json b/nitro-node/package.json new file mode 100644 index 000000000..afa029e1f --- /dev/null +++ b/nitro-node/package.json @@ -0,0 +1,57 @@ +{ + "name": "@janhq/nitro-node", + "version": "1.0.0", + "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See nitro.jan.ai", + "main": "dist/index.js", + "module": "dist/module.js", + "author": "Jan ", + "license": "AGPL-3.0", + "scripts": { + "build": "tsc -b . && webpack --config webpack.config.js", + "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro", + "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro", + "downloadnitro:win32": "download.bat", + "downloadnitro": "run-script-os", + "publish:darwin": "../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack", + "publish:win32": "cpx \"bin/**\" \"dist/bin\" && npm pack", + "publish:linux": "cpx \"bin/**\" \"dist/bin\" && npm pack", + "publish": "run-script-os" + }, + "exports": { + ".": "./dist/index.js", + "./main": "./dist/module.js" + }, + "devDependencies": { + "cpx": "^1.5.0", + "run-script-os": "^1.1.6", + "typescript": "^5.3.3", + "webpack": "^5.88.2", + "webpack-cli": "^5.1.4" + }, + "dependencies": { + "@janhq/core": "file:jan/core", + "download-cli": "^1.1.1", + "fetch-retry": "^5.0.6", + "os-utils": "^0.0.14", + "path-browserify": "^1.0.1", + "rxjs": "^7.8.1", + "tcp-port-used": "^1.0.2", + "ts-loader": "^9.5.0", + "ulid": "^2.3.0" + }, + "engines": { + "node": ">=18.12.0" + }, + "files": [ + "dist/*", + "package.json", + "README.md" + ], + "bundleDependencies": [ + "tcp-port-used", + "fetch-retry", + "os-utils", + "@janhq/core" + ], + "packageManager": "yarn@1.22.21" +} diff --git a/nitro-node/src/@types/global.d.ts b/nitro-node/src/@types/global.d.ts new file mode 100644 index 000000000..6bcdc4adc --- /dev/null +++ b/nitro-node/src/@types/global.d.ts @@ -0,0 +1,33 @@ +declare const MODULE: string; +declare const INFERENCE_URL: string; +declare const TROUBLESHOOTING_URL: string; + +/** + * The parameters for the initModel function. + * @property settings - The settings for the machine learning model. + * @property settings.ctx_len - The context length. + * @property settings.ngl - The number of generated tokens. + * @property settings.cont_batching - Whether to use continuous batching. + * @property settings.embedding - Whether to use embedding. + */ +interface EngineSettings { + ctx_len: number; + ngl: number; + cpu_threads: number; + cont_batching: boolean; + embedding: boolean; +} + +/** + * The response from the initModel function. + * @property error - An error message if the model fails to load. + */ +interface ModelOperationResponse { + error?: any; + modelFile?: string; +} + +interface ResourcesInfo { + numCpuPhysicalCore: number; + memAvailable: number; +} \ No newline at end of file diff --git a/nitro-node/src/helpers/sse.ts b/nitro-node/src/helpers/sse.ts new file mode 100644 index 000000000..c6352383d --- /dev/null +++ b/nitro-node/src/helpers/sse.ts @@ -0,0 +1,65 @@ +import { Model } from "@janhq/core"; +import { Observable } from "rxjs"; +/** + * Sends a request to the inference server to generate a response based on the recent messages. + * @param recentMessages - An array of recent messages to use as context for the inference. + * @returns An Observable that emits the generated response as a string. + */ +export function requestInference( + recentMessages: any[], + model: Model, + controller?: AbortController +): Observable { + return new Observable((subscriber) => { + const requestBody = JSON.stringify({ + messages: recentMessages, + model: model.id, + stream: true, + ...model.parameters, + }); + fetch(INFERENCE_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + Accept: model.parameters.stream + ? "text/event-stream" + : "application/json", + }, + body: requestBody, + signal: controller?.signal, + }) + .then(async (response) => { + if (model.parameters.stream === false) { + const data = await response.json(); + subscriber.next(data.choices[0]?.message?.content ?? ""); + } else { + const stream = response.body; + const decoder = new TextDecoder("utf-8"); + const reader = stream?.getReader(); + let content = ""; + + while (true && reader) { + const { done, value } = await reader.read(); + if (done) { + break; + } + const text = decoder.decode(value); + const lines = text.trim().split("\n"); + for (const line of lines) { + if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { + const data = JSON.parse(line.replace("data: ", "")); + content += data.choices[0]?.delta?.content ?? ""; + if (content.startsWith("assistant: ")) { + content = content.replace("assistant: ", ""); + } + subscriber.next(content); + } + } + } + } + subscriber.complete(); + }) + .catch((err) => subscriber.error(err)); + }); +} diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts new file mode 100644 index 000000000..b6c63f59a --- /dev/null +++ b/nitro-node/src/index.ts @@ -0,0 +1,279 @@ +/** + * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. + * The class provides methods for initializing and stopping a model, and for making inference requests. + * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + * @version 1.0.0 + * @module inference-extension/src/index + */ + +import { + ChatCompletionRole, + ContentType, + EventName, + MessageRequest, + MessageStatus, + ExtensionType, + ThreadContent, + ThreadMessage, + events, + executeOnMain, + fs, + Model, + joinPath, + InferenceExtension, + log, + InferenceEngine, +} from "@janhq/core"; +import { requestInference } from "./helpers/sse"; +import { ulid } from "ulid"; +import { join } from "path"; + +/** + * A class that implements the InferenceExtension interface from the @janhq/core package. + * The class provides methods for initializing and stopping a model, and for making inference requests. + * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + */ +export default class JanInferenceNitroExtension implements InferenceExtension { + private static readonly _homeDir = "file://engines"; + private static readonly _settingsDir = "file://settings"; + private static readonly _engineMetadataFileName = "nitro.json"; + + /** + * Checking the health for Nitro's process each 5 secs. + */ + private static readonly _intervalHealthCheck = 5 * 1000; + + private _currentModel: Model; + + private _engineSettings: EngineSettings = { + ctx_len: 2048, + ngl: 100, + cpu_threads: 1, + cont_batching: false, + embedding: false, + }; + + controller = new AbortController(); + isCancelled = false; + + /** + * The interval id for the health check. Used to stop the health check. + */ + private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = + undefined; + + /** + * Tracking the current state of nitro process. + */ + private nitroProcessInfo: any = undefined; + + /** + * Returns the type of the extension. + * @returns {ExtensionType} The type of the extension. + */ + type(): ExtensionType { + return ExtensionType.Inference; + } + + /** + * Subscribes to events emitted by the @janhq/core package. + */ + async onLoad() { + if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) { + await fs + .mkdirSync(JanInferenceNitroExtension._homeDir) + .catch((err) => console.debug(err)); + } + + if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir))) + await fs.mkdirSync(JanInferenceNitroExtension._settingsDir); + this.writeDefaultEngineSettings(); + + // Events subscription + events.on(EventName.OnMessageSent, (data) => this.onMessageRequest(data)); + + events.on(EventName.OnModelInit, (model: Model) => this.onModelInit(model)); + + events.on(EventName.OnModelStop, (model: Model) => this.onModelStop(model)); + + events.on(EventName.OnInferenceStopped, () => this.onInferenceStopped()); + + // Attempt to fetch nvidia info + await executeOnMain(MODULE, "updateNvidiaInfo", {}); + } + + /** + * Stops the model inference. + */ + onUnload(): void {} + + private async writeDefaultEngineSettings() { + try { + const engineFile = join( + JanInferenceNitroExtension._homeDir, + JanInferenceNitroExtension._engineMetadataFileName + ); + if (await fs.existsSync(engineFile)) { + const engine = await fs.readFileSync(engineFile, "utf-8"); + this._engineSettings = + typeof engine === "object" ? engine : JSON.parse(engine); + } else { + await fs.writeFileSync( + engineFile, + JSON.stringify(this._engineSettings, null, 2) + ); + } + } catch (err) { + console.error(err); + } + } + + private async onModelInit(model: Model) { + if (model.engine !== InferenceEngine.nitro) return; + + const modelFullPath = await joinPath(["models", model.id]); + + const nitroInitResult = await executeOnMain(MODULE, "initModel", { + modelFullPath: modelFullPath, + model: model, + }); + + if (nitroInitResult.error === null) { + events.emit(EventName.OnModelFail, model); + return; + } + + this._currentModel = model; + events.emit(EventName.OnModelReady, model); + + this.getNitroProcesHealthIntervalId = setInterval( + () => this.periodicallyGetNitroHealth(), + JanInferenceNitroExtension._intervalHealthCheck + ); + } + + private async onModelStop(model: Model) { + if (model.engine !== "nitro") return; + + await executeOnMain(MODULE, "stopModel"); + events.emit(EventName.OnModelStopped, {}); + + // stop the periocally health check + if (this.getNitroProcesHealthIntervalId) { + console.debug("Stop calling Nitro process health check"); + clearInterval(this.getNitroProcesHealthIntervalId); + this.getNitroProcesHealthIntervalId = undefined; + } + } + + /** + * Periodically check for nitro process's health. + */ + private async periodicallyGetNitroHealth(): Promise { + const health = await executeOnMain(MODULE, "getCurrentNitroProcessInfo"); + + const isRunning = this.nitroProcessInfo?.isRunning ?? false; + if (isRunning && health.isRunning === false) { + console.debug("Nitro process is stopped"); + events.emit(EventName.OnModelStopped, {}); + } + this.nitroProcessInfo = health; + } + + private async onInferenceStopped() { + this.isCancelled = true; + this.controller?.abort(); + } + + /** + * Makes a single response inference request. + * @param {MessageRequest} data - The data for the inference request. + * @returns {Promise} A promise that resolves with the inference response. + */ + async inference(data: MessageRequest): Promise { + const timestamp = Date.now(); + const message: ThreadMessage = { + thread_id: data.threadId, + created: timestamp, + updated: timestamp, + status: MessageStatus.Ready, + id: "", + role: ChatCompletionRole.Assistant, + object: "thread.message", + content: [], + }; + + return new Promise(async (resolve, reject) => { + requestInference(data.messages ?? [], this._currentModel).subscribe({ + next: (_content) => {}, + complete: async () => { + resolve(message); + }, + error: async (err) => { + reject(err); + }, + }); + }); + } + + /** + * Handles a new message request by making an inference request and emitting events. + * Function registered in event manager, should be static to avoid binding issues. + * Pass instance as a reference. + * @param {MessageRequest} data - The data for the new message request. + */ + private async onMessageRequest(data: MessageRequest) { + if (data.model.engine !== "nitro") return; + + const timestamp = Date.now(); + const message: ThreadMessage = { + id: ulid(), + thread_id: data.threadId, + assistant_id: data.assistantId, + role: ChatCompletionRole.Assistant, + content: [], + status: MessageStatus.Pending, + created: timestamp, + updated: timestamp, + object: "thread.message", + }; + events.emit(EventName.OnMessageResponse, message); + + this.isCancelled = false; + this.controller = new AbortController(); + + requestInference( + data.messages ?? [], + { ...this._currentModel, ...data.model }, + this.controller + ).subscribe({ + next: (content) => { + const messageContent: ThreadContent = { + type: ContentType.Text, + text: { + value: content.trim(), + annotations: [], + }, + }; + message.content = [messageContent]; + events.emit(EventName.OnMessageUpdate, message); + }, + complete: async () => { + message.status = message.content.length + ? MessageStatus.Ready + : MessageStatus.Error; + events.emit(EventName.OnMessageUpdate, message); + }, + error: async (err) => { + if (this.isCancelled || message.content.length) { + message.status = MessageStatus.Stopped; + events.emit(EventName.OnMessageUpdate, message); + return; + } + message.status = MessageStatus.Error; + events.emit(EventName.OnMessageUpdate, message); + log(`[APP]::Error: ${err.message}`); + }, + }); + } +} diff --git a/nitro-node/src/module.ts b/nitro-node/src/module.ts new file mode 100644 index 000000000..6907f244a --- /dev/null +++ b/nitro-node/src/module.ts @@ -0,0 +1,514 @@ +const fs = require("fs"); +const path = require("path"); +const { exec, spawn } = require("child_process"); +const tcpPortUsed = require("tcp-port-used"); +const fetchRetry = require("fetch-retry")(global.fetch); +const osUtils = require("os-utils"); +const { readFileSync, writeFileSync, existsSync } = require("fs"); +const { log } = require("@janhq/core/node"); + +// The PORT to use for the Nitro subprocess +const PORT = 3928; +const LOCAL_HOST = "127.0.0.1"; +const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`; +const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`; +const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; +const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; +const SUPPORTED_MODEL_FORMAT = ".gguf"; +const NVIDIA_INFO_FILE = path.join( + require("os").homedir(), + "jan", + "settings", + "settings.json" +); + +// The subprocess instance for Nitro +let subprocess = undefined; +let currentModelFile: string = undefined; +let currentSettings = undefined; + +let nitroProcessInfo = undefined; + +/** + * Default GPU settings + **/ +const DEFALT_SETTINGS = { + notify: true, + run_mode: "cpu", + nvidia_driver: { + exist: false, + version: "", + }, + cuda: { + exist: false, + version: "", + }, + gpus: [], + gpu_highest_vram: "", +}; + +/** + * Stops a Nitro subprocess. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +function stopModel(): Promise { + return killSubprocess(); +} + +/** + * Initializes a Nitro subprocess to load a machine learning model. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package + * TODO: Should it be startModel instead? + */ +async function initModel(wrapper: any): Promise { + currentModelFile = wrapper.modelFullPath; + const janRoot = path.join(require("os").homedir(), "jan"); + if (!currentModelFile.includes(janRoot)) { + currentModelFile = path.join(janRoot, currentModelFile); + } + const files: string[] = fs.readdirSync(currentModelFile); + + // Look for GGUF model file + const ggufBinFile = files.find( + (file) => + file === path.basename(currentModelFile) || + file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT) + ); + + currentModelFile = path.join(currentModelFile, ggufBinFile); + + if (wrapper.model.engine !== "nitro") { + return Promise.resolve({ error: "Not a nitro model" }); + } else { + const nitroResourceProbe = await getResourcesInfo(); + // Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt + if (wrapper.model.settings.prompt_template) { + const promptTemplate = wrapper.model.settings.prompt_template; + const prompt = promptTemplateConverter(promptTemplate); + if (prompt.error) { + return Promise.resolve({ error: prompt.error }); + } + wrapper.model.settings.system_prompt = prompt.system_prompt; + wrapper.model.settings.user_prompt = prompt.user_prompt; + wrapper.model.settings.ai_prompt = prompt.ai_prompt; + } + + currentSettings = { + llama_model_path: currentModelFile, + ...wrapper.model.settings, + // This is critical and requires real system information + cpu_threads: nitroResourceProbe.numCpuPhysicalCore, + }; + return loadModel(nitroResourceProbe); + } +} + +async function loadModel(nitroResourceProbe: any | undefined) { + // Gather system information for CPU physical cores and memory + if (!nitroResourceProbe) nitroResourceProbe = await getResourcesInfo(); + return killSubprocess() + .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) + .then(() => { + /** + * There is a problem with Windows process manager + * Should wait for awhile to make sure the port is free and subprocess is killed + * The tested threshold is 500ms + **/ + if (process.platform === "win32") { + return new Promise((resolve) => setTimeout(resolve, 500)); + } else { + return Promise.resolve(); + } + }) + .then(() => spawnNitroProcess(nitroResourceProbe)) + .then(() => loadLLMModel(currentSettings)) + .then(validateModelStatus) + .catch((err) => { + log(`[NITRO]::Error: ${err}`); + // TODO: Broadcast error so app could display proper error message + return { error: err, currentModelFile }; + }); +} + +function promptTemplateConverter(promptTemplate) { + // Split the string using the markers + const systemMarker = "{system_message}"; + const promptMarker = "{prompt}"; + + if ( + promptTemplate.includes(systemMarker) && + promptTemplate.includes(promptMarker) + ) { + // Find the indices of the markers + const systemIndex = promptTemplate.indexOf(systemMarker); + const promptIndex = promptTemplate.indexOf(promptMarker); + + // Extract the parts of the string + const system_prompt = promptTemplate.substring(0, systemIndex); + const user_prompt = promptTemplate.substring( + systemIndex + systemMarker.length, + promptIndex + ); + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length + ); + + // Return the split parts + return { system_prompt, user_prompt, ai_prompt }; + } else if (promptTemplate.includes(promptMarker)) { + // Extract the parts of the string for the case where only promptMarker is present + const promptIndex = promptTemplate.indexOf(promptMarker); + const user_prompt = promptTemplate.substring(0, promptIndex); + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length + ); + const system_prompt = ""; + + // Return the split parts + return { system_prompt, user_prompt, ai_prompt }; + } + + // Return an error if none of the conditions are met + return { error: "Cannot split prompt template" }; +} + +/** + * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + */ +function loadLLMModel(settings): Promise { + log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`); + return fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(settings), + retries: 3, + retryDelay: 500, + }).catch((err) => { + log(`[NITRO]::Error: Load model failed with error ${err}`); + }); +} + +/** + * Validates the status of a model. + * @returns {Promise} A promise that resolves to an object. + * If the model is loaded successfully, the object is empty. + * If the model is not loaded successfully, the object contains an error message. + */ +async function validateModelStatus(): Promise { + // Send a GET request to the validation URL. + // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. + return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + retries: 5, + retryDelay: 500, + }).then(async (res: Response) => { + // If the response is OK, check model_loaded status. + if (res.ok) { + const body = await res.json(); + // If the model is loaded, return an empty object. + // Otherwise, return an object with an error message. + if (body.model_loaded) { + return { error: undefined }; + } + } + return { error: "Model loading failed" }; + }); +} + +/** + * Terminates the Nitro subprocess. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +async function killSubprocess(): Promise { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 5000); + log(`[NITRO]::Debug: Request to kill Nitro`); + + return fetch(NITRO_HTTP_KILL_URL, { + method: "DELETE", + signal: controller.signal, + }) + .then(() => { + subprocess?.kill(); + subprocess = undefined; + }) + .catch(() => {}) + .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) + .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)); +} + +/** + * Spawns a Nitro subprocess. + * @param nitroResourceProbe - The Nitro resource probe. + * @returns A promise that resolves when the Nitro subprocess is started. + */ +function spawnNitroProcess(nitroResourceProbe: any): Promise { + log(`[NITRO]::Debug: Spawning Nitro subprocess...`); + + return new Promise(async (resolve, reject) => { + let binaryFolder = path.join(__dirname, "bin"); // Current directory by default + let cudaVisibleDevices = ""; + let binaryName; + if (process.platform === "win32") { + let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + if (nvidiaInfo["run_mode"] === "cpu") { + binaryFolder = path.join(binaryFolder, "win-cpu"); + } else { + if (nvidiaInfo["cuda"].version === "12") { + binaryFolder = path.join(binaryFolder, "win-cuda-12-0"); + } else { + binaryFolder = path.join(binaryFolder, "win-cuda-11-7"); + } + cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; + } + binaryName = "nitro.exe"; + } else if (process.platform === "darwin") { + if (process.arch === "arm64") { + binaryFolder = path.join(binaryFolder, "mac-arm64"); + } else { + binaryFolder = path.join(binaryFolder, "mac-x64"); + } + binaryName = "nitro"; + } else { + let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + if (nvidiaInfo["run_mode"] === "cpu") { + binaryFolder = path.join(binaryFolder, "linux-cpu"); + } else { + if (nvidiaInfo["cuda"].version === "12") { + binaryFolder = path.join(binaryFolder, "linux-cuda-12-0"); + } else { + binaryFolder = path.join(binaryFolder, "linux-cuda-11-7"); + } + cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; + } + binaryName = "nitro"; + } + + const binaryPath = path.join(binaryFolder, binaryName); + // Execute the binary + subprocess = spawn(binaryPath, ["1", LOCAL_HOST, PORT.toString()], { + cwd: binaryFolder, + env: { + ...process.env, + CUDA_VISIBLE_DEVICES: cudaVisibleDevices, + }, + }); + + // Handle subprocess output + subprocess.stdout.on("data", (data) => { + log(`[NITRO]::Debug: ${data}`); + }); + + subprocess.stderr.on("data", (data) => { + log(`[NITRO]::Error: ${data}`); + }); + + subprocess.on("close", (code) => { + log(`[NITRO]::Debug: Nitro exited with code: ${code}`); + subprocess = null; + reject(`child process exited with code ${code}`); + }); + + tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { + resolve(nitroResourceProbe); + }); + }); +} + +/** + * Get the system resources information + * TODO: Move to Core so that it can be reused + */ +function getResourcesInfo(): Promise { + return new Promise(async (resolve) => { + const cpu = await osUtils.cpuCount(); + log(`[NITRO]::CPU informations - ${cpu}`); + const response: ResourcesInfo = { + numCpuPhysicalCore: cpu, + memAvailable: 0, + }; + resolve(response); + }); +} + +/** + * This will retrive GPU informations and persist settings.json + * Will be called when the extension is loaded to turn on GPU acceleration if supported + */ +async function updateNvidiaInfo() { + if (process.platform !== "darwin") { + await Promise.all([ + updateNvidiaDriverInfo(), + updateCudaExistence(), + updateGpuInfo(), + ]); + } +} + +/** + * Retrieve current nitro process + */ +const getCurrentNitroProcessInfo = (): Promise => { + nitroProcessInfo = { + isRunning: subprocess != null, + }; + return nitroProcessInfo; +}; + +/** + * Every module should have a dispose function + * This will be called when the extension is unloaded and should clean up any resources + * Also called when app is closed + */ +function dispose() { + // clean other registered resources here + killSubprocess(); +} + +/** + * Validate nvidia and cuda for linux and windows + */ +async function updateNvidiaDriverInfo(): Promise { + exec( + "nvidia-smi --query-gpu=driver_version --format=csv,noheader", + (error, stdout) => { + let data; + try { + data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + } catch (error) { + data = DEFALT_SETTINGS; + } + + if (!error) { + const firstLine = stdout.split("\n")[0].trim(); + data["nvidia_driver"].exist = true; + data["nvidia_driver"].version = firstLine; + } else { + data["nvidia_driver"].exist = false; + } + + writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); + Promise.resolve(); + } + ); +} + +/** + * Check if file exists in paths + */ +function checkFileExistenceInPaths(file: string, paths: string[]): boolean { + return paths.some((p) => existsSync(path.join(p, file))); +} + +/** + * Validate cuda for linux and windows + */ +function updateCudaExistence() { + let filesCuda12: string[]; + let filesCuda11: string[]; + let paths: string[]; + let cudaVersion: string = ""; + + if (process.platform === "win32") { + filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; + filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; + paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : []; + } else { + filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; + filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; + paths = process.env.LD_LIBRARY_PATH + ? process.env.LD_LIBRARY_PATH.split(path.delimiter) + : []; + paths.push("/usr/lib/x86_64-linux-gnu/"); + } + + let cudaExists = filesCuda12.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ); + + if (!cudaExists) { + cudaExists = filesCuda11.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ); + if (cudaExists) { + cudaVersion = "11"; + } + } else { + cudaVersion = "12"; + } + + let data; + try { + data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + } catch (error) { + data = DEFALT_SETTINGS; + } + + data["cuda"].exist = cudaExists; + data["cuda"].version = cudaVersion; + if (cudaExists) { + data.run_mode = "gpu"; + } + writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); +} + +/** + * Get GPU information + */ +async function updateGpuInfo(): Promise { + exec( + "nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", + (error, stdout) => { + let data; + try { + data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + } catch (error) { + data = DEFALT_SETTINGS; + } + + if (!error) { + // Get GPU info and gpu has higher memory first + let highestVram = 0; + let highestVramId = "0"; + let gpus = stdout + .trim() + .split("\n") + .map((line) => { + let [id, vram] = line.split(", "); + vram = vram.replace(/\r/g, ""); + if (parseFloat(vram) > highestVram) { + highestVram = parseFloat(vram); + highestVramId = id; + } + return { id, vram }; + }); + + data["gpus"] = gpus; + data["gpu_highest_vram"] = highestVramId; + } else { + data["gpus"] = []; + } + + writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); + Promise.resolve(); + } + ); +} + +module.exports = { + initModel, + stopModel, + killSubprocess, + dispose, + updateNvidiaInfo, + getCurrentNitroProcessInfo, +}; diff --git a/nitro-node/tsconfig.json b/nitro-node/tsconfig.json new file mode 100644 index 000000000..b48175a16 --- /dev/null +++ b/nitro-node/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es2016", + "module": "ES6", + "moduleResolution": "node", + + "outDir": "./dist", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": false, + "skipLibCheck": true, + "rootDir": "./src" + }, + "include": ["./src"] +} diff --git a/nitro-node/webpack.config.js b/nitro-node/webpack.config.js new file mode 100644 index 000000000..2927affbc --- /dev/null +++ b/nitro-node/webpack.config.js @@ -0,0 +1,43 @@ +const path = require("path"); +const webpack = require("webpack"); +const packageJson = require("./package.json"); + +module.exports = { + experiments: { outputModule: true }, + entry: "./src/index.ts", // Adjust the entry point to match your project's main file + mode: "production", + module: { + rules: [ + { + test: /\.tsx?$/, + use: "ts-loader", + exclude: /node_modules/, + }, + ], + }, + plugins: [ + new webpack.DefinePlugin({ + MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), + INFERENCE_URL: JSON.stringify( + process.env.INFERENCE_URL || + "http://127.0.0.1:3928/inferences/llamacpp/chat_completion" + ), + TROUBLESHOOTING_URL: JSON.stringify("https://jan.ai/guides/troubleshooting") + }), + ], + output: { + filename: "index.js", // Adjust the output file name as needed + path: path.resolve(__dirname, "dist"), + library: { type: "module" }, // Specify ESM output format + }, + resolve: { + extensions: [".ts", ".js"], + fallback: { + path: require.resolve("path-browserify"), + }, + }, + optimization: { + minimize: false, + }, + // Add loaders and other configuration as needed for your project +}; diff --git a/yarn.lock b/yarn.lock deleted file mode 100644 index fb57ccd13..000000000 --- a/yarn.lock +++ /dev/null @@ -1,4 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - From ffcd05d21abdd656de05d68175bc61749415441d Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Mon, 22 Jan 2024 14:03:48 +0700 Subject: [PATCH 02/49] fix: exclude nitro-node/Makefile from root .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 13a9b93d2..be1237faa 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ CMakeFiles CMakeScripts Testing Makefile +!nitro-node/Makefile cmake_install.cmake install_manifest.txt compile_commands.json From 81ee8f373b5d847786725dc306f56f7806511c97 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Mon, 22 Jan 2024 15:08:15 +0700 Subject: [PATCH 03/49] chore(rebase): rebase nitro-node from 'dev' branch of janhq/jan - Update nitro-node source code - Relax implicit type rules in tsconfig.json - Dirty fix for wrong typing in src/module.tsconfig - Update build to use @janhq/core from npmjs - Tested build with yarn@1 --- nitro-node/Makefile | 20 +- nitro-node/jan | 1 - nitro-node/package.json | 32 ++- nitro-node/rollup.config.ts | 77 ++++++ nitro-node/src/@types/global.d.ts | 2 +- nitro-node/src/index.ts | 86 ++++--- nitro-node/src/module.ts | 8 +- nitro-node/src/node/execute.ts | 65 +++++ nitro-node/src/node/index.ts | 379 ++++++++++++++++++++++++++++++ nitro-node/src/node/nvidia.ts | 201 ++++++++++++++++ nitro-node/tsconfig.json | 34 ++- 11 files changed, 818 insertions(+), 87 deletions(-) delete mode 160000 nitro-node/jan create mode 100644 nitro-node/rollup.config.ts create mode 100644 nitro-node/src/node/execute.ts create mode 100644 nitro-node/src/node/index.ts create mode 100644 nitro-node/src/node/nvidia.ts diff --git a/nitro-node/Makefile b/nitro-node/Makefile index c39b2af53..f862fa8fa 100644 --- a/nitro-node/Makefile +++ b/nitro-node/Makefile @@ -3,21 +3,11 @@ # Default target, build all .PHONY: all -all: publish - -# Build jan/core -build-core: - git submodule update --init -ifeq ($(OS),Windows_NT) - type NUL > jan/yarn.lock -else - touch jan/yarn.lock -endif - cd jan && yarn install - cd jan/core && yarn build +all: clean publish # Installs yarn dependencies -install: build-core +#install: build-core +install: ifeq ($(OS),Windows_NT) yarn config set network-timeout 300000 endif @@ -37,10 +27,10 @@ publish: build download-nitro clean: ifeq ($(OS),Windows_NT) - del /S *.tgz + del /F /S *.tgz .yarn yarn.lock powershell -Command "Get-ChildItem -Path . -Include node_modules, dist -Recurse -Directory | Remove-Item -Recurse -Force" else - rm -f *.tgz + rm -rf *.tgz .yarn yarn.lock find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + endif diff --git a/nitro-node/jan b/nitro-node/jan deleted file mode 160000 index 42a86e9cb..000000000 --- a/nitro-node/jan +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 42a86e9cb71a195a58d4050b6f8649d080155382 diff --git a/nitro-node/package.json b/nitro-node/package.json index afa029e1f..d386f7250 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -3,11 +3,11 @@ "version": "1.0.0", "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See nitro.jan.ai", "main": "dist/index.js", - "module": "dist/module.js", + "node": "dist/node/index.cjs.js", "author": "Jan ", "license": "AGPL-3.0", "scripts": { - "build": "tsc -b . && webpack --config webpack.config.js", + "build": "tsc --module commonjs && rollup -c rollup.config.ts", "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro", "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro", "downloadnitro:win32": "download.bat", @@ -19,28 +19,37 @@ }, "exports": { ".": "./dist/index.js", - "./main": "./dist/module.js" + "./main": "./dist/node/index.cjs.js" }, "devDependencies": { + "@rollup/plugin-commonjs": "^25.0.7", + "@rollup/plugin-json": "^6.1.0", + "@rollup/plugin-node-resolve": "^15.2.3", + "@types/node": "^20.11.4", + "@types/tcp-port-used": "^1.0.4", "cpx": "^1.5.0", + "download-cli": "^1.1.1", + "rimraf": "^3.0.2", + "rollup": "^2.38.5", + "rollup-plugin-define": "^1.0.1", + "rollup-plugin-sourcemaps": "^0.6.3", + "rollup-plugin-typescript2": "^0.36.0", "run-script-os": "^1.1.6", - "typescript": "^5.3.3", - "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "typescript": "^5.3.3" }, "dependencies": { - "@janhq/core": "file:jan/core", - "download-cli": "^1.1.1", + "@janhq/core": "^0.1.11", + "@rollup/plugin-replace": "^5.0.5", + "@types/os-utils": "^0.0.4", "fetch-retry": "^5.0.6", "os-utils": "^0.0.14", "path-browserify": "^1.0.1", "rxjs": "^7.8.1", "tcp-port-used": "^1.0.2", - "ts-loader": "^9.5.0", "ulid": "^2.3.0" }, "engines": { - "node": ">=18.12.0" + "node": ">=18.0.0" }, "files": [ "dist/*", @@ -52,6 +61,5 @@ "fetch-retry", "os-utils", "@janhq/core" - ], - "packageManager": "yarn@1.22.21" + ] } diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts new file mode 100644 index 000000000..374a054cd --- /dev/null +++ b/nitro-node/rollup.config.ts @@ -0,0 +1,77 @@ +import resolve from "@rollup/plugin-node-resolve"; +import commonjs from "@rollup/plugin-commonjs"; +import sourceMaps from "rollup-plugin-sourcemaps"; +import typescript from "rollup-plugin-typescript2"; +import json from "@rollup/plugin-json"; +import replace from "@rollup/plugin-replace"; +const packageJson = require("./package.json"); + +const pkg = require("./package.json"); + +export default [ + { + input: `src/index.ts`, + output: [{ file: pkg.main, format: "es", sourcemap: true }], + // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') + external: [], + watch: { + include: "src/**", + }, + plugins: [ + replace({ + NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), + INFERENCE_URL: JSON.stringify( + process.env.INFERENCE_URL || + "http://127.0.0.1:3928/inferences/llamacpp/chat_completion" + ), + TROUBLESHOOTING_URL: JSON.stringify( + "https://jan.ai/guides/troubleshooting" + ), + }), + // Allow json resolution + json(), + // Compile TypeScript files + typescript({ useTsconfigDeclarationDir: true }), + // Compile TypeScript files + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + commonjs(), + // Allow node_modules resolution, so you can use 'external' to control + // which external modules to include in the bundle + // https://github.com/rollup/rollup-plugin-node-resolve#usage + resolve({ + extensions: [".js", ".ts", ".svelte"], + }), + + // Resolve source maps to the original source + sourceMaps(), + ], + }, + { + input: `src/node/index.ts`, + output: [ + { file: "dist/node/index.cjs.js", format: "cjs", sourcemap: true }, + ], + // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') + external: ["@janhq/core/node"], + watch: { + include: "src/node/**", + }, + plugins: [ + // Allow json resolution + json(), + // Compile TypeScript files + typescript({ useTsconfigDeclarationDir: true }), + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + commonjs(), + // Allow node_modules resolution, so you can use 'external' to control + // which external modules to include in the bundle + // https://github.com/rollup/rollup-plugin-node-resolve#usage + resolve({ + extensions: [".ts", ".js", ".json"], + }), + + // Resolve source maps to the original source + sourceMaps(), + ], + }, +]; diff --git a/nitro-node/src/@types/global.d.ts b/nitro-node/src/@types/global.d.ts index 6bcdc4adc..5fb41f0f8 100644 --- a/nitro-node/src/@types/global.d.ts +++ b/nitro-node/src/@types/global.d.ts @@ -1,4 +1,4 @@ -declare const MODULE: string; +declare const NODE: string; declare const INFERENCE_URL: string; declare const TROUBLESHOOTING_URL: string; diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index b6c63f59a..ba3705fde 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -9,10 +9,8 @@ import { ChatCompletionRole, ContentType, - EventName, MessageRequest, MessageStatus, - ExtensionType, ThreadContent, ThreadMessage, events, @@ -23,17 +21,19 @@ import { InferenceExtension, log, InferenceEngine, + MessageEvent, + ModelEvent, + InferenceEvent, } from "@janhq/core"; import { requestInference } from "./helpers/sse"; import { ulid } from "ulid"; -import { join } from "path"; /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ -export default class JanInferenceNitroExtension implements InferenceExtension { +export default class JanInferenceNitroExtension extends InferenceExtension { private static readonly _homeDir = "file://engines"; private static readonly _settingsDir = "file://settings"; private static readonly _engineMetadataFileName = "nitro.json"; @@ -43,7 +43,7 @@ export default class JanInferenceNitroExtension implements InferenceExtension { */ private static readonly _intervalHealthCheck = 5 * 1000; - private _currentModel: Model; + private _currentModel: Model | undefined; private _engineSettings: EngineSettings = { ctx_len: 2048, @@ -67,14 +67,6 @@ export default class JanInferenceNitroExtension implements InferenceExtension { */ private nitroProcessInfo: any = undefined; - /** - * Returns the type of the extension. - * @returns {ExtensionType} The type of the extension. - */ - type(): ExtensionType { - return ExtensionType.Inference; - } - /** * Subscribes to events emitted by the @janhq/core package. */ @@ -82,7 +74,7 @@ export default class JanInferenceNitroExtension implements InferenceExtension { if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) { await fs .mkdirSync(JanInferenceNitroExtension._homeDir) - .catch((err) => console.debug(err)); + .catch((err: Error) => console.debug(err)); } if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir))) @@ -90,16 +82,18 @@ export default class JanInferenceNitroExtension implements InferenceExtension { this.writeDefaultEngineSettings(); // Events subscription - events.on(EventName.OnMessageSent, (data) => this.onMessageRequest(data)); + events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.onMessageRequest(data) + ); - events.on(EventName.OnModelInit, (model: Model) => this.onModelInit(model)); + events.on(ModelEvent.OnModelInit, (model: Model) => this.onModelInit(model)); - events.on(EventName.OnModelStop, (model: Model) => this.onModelStop(model)); + events.on(ModelEvent.OnModelStop, (model: Model) => this.onModelStop(model)); - events.on(EventName.OnInferenceStopped, () => this.onInferenceStopped()); + events.on(InferenceEvent.OnInferenceStopped, () => this.onInferenceStopped()); // Attempt to fetch nvidia info - await executeOnMain(MODULE, "updateNvidiaInfo", {}); + await executeOnMain(NODE, "updateNvidiaInfo", {}); } /** @@ -109,10 +103,10 @@ export default class JanInferenceNitroExtension implements InferenceExtension { private async writeDefaultEngineSettings() { try { - const engineFile = join( + const engineFile = await joinPath([ JanInferenceNitroExtension._homeDir, - JanInferenceNitroExtension._engineMetadataFileName - ); + JanInferenceNitroExtension._engineMetadataFileName, + ]); if (await fs.existsSync(engineFile)) { const engine = await fs.readFileSync(engineFile, "utf-8"); this._engineSettings = @@ -133,18 +127,18 @@ export default class JanInferenceNitroExtension implements InferenceExtension { const modelFullPath = await joinPath(["models", model.id]); - const nitroInitResult = await executeOnMain(MODULE, "initModel", { - modelFullPath: modelFullPath, - model: model, + const nitroInitResult = await executeOnMain(NODE, "runModel", { + modelFullPath, + model, }); - if (nitroInitResult.error === null) { - events.emit(EventName.OnModelFail, model); + if (nitroInitResult?.error) { + events.emit(ModelEvent.OnModelFail, model); return; } this._currentModel = model; - events.emit(EventName.OnModelReady, model); + events.emit(ModelEvent.OnModelReady, model); this.getNitroProcesHealthIntervalId = setInterval( () => this.periodicallyGetNitroHealth(), @@ -155,12 +149,11 @@ export default class JanInferenceNitroExtension implements InferenceExtension { private async onModelStop(model: Model) { if (model.engine !== "nitro") return; - await executeOnMain(MODULE, "stopModel"); - events.emit(EventName.OnModelStopped, {}); + await executeOnMain(NODE, "stopModel"); + events.emit(ModelEvent.OnModelStopped, {}); // stop the periocally health check if (this.getNitroProcesHealthIntervalId) { - console.debug("Stop calling Nitro process health check"); clearInterval(this.getNitroProcesHealthIntervalId); this.getNitroProcesHealthIntervalId = undefined; } @@ -170,12 +163,12 @@ export default class JanInferenceNitroExtension implements InferenceExtension { * Periodically check for nitro process's health. */ private async periodicallyGetNitroHealth(): Promise { - const health = await executeOnMain(MODULE, "getCurrentNitroProcessInfo"); + const health = await executeOnMain(NODE, "getCurrentNitroProcessInfo"); const isRunning = this.nitroProcessInfo?.isRunning ?? false; if (isRunning && health.isRunning === false) { console.debug("Nitro process is stopped"); - events.emit(EventName.OnModelStopped, {}); + events.emit(ModelEvent.OnModelStopped, {}); } this.nitroProcessInfo = health; } @@ -204,6 +197,8 @@ export default class JanInferenceNitroExtension implements InferenceExtension { }; return new Promise(async (resolve, reject) => { + if (!this._currentModel) return Promise.reject("No model loaded"); + requestInference(data.messages ?? [], this._currentModel).subscribe({ next: (_content) => {}, complete: async () => { @@ -223,7 +218,9 @@ export default class JanInferenceNitroExtension implements InferenceExtension { * @param {MessageRequest} data - The data for the new message request. */ private async onMessageRequest(data: MessageRequest) { - if (data.model.engine !== "nitro") return; + if (data.model?.engine !== InferenceEngine.nitro || !this._currentModel) { + return; + } const timestamp = Date.now(); const message: ThreadMessage = { @@ -237,16 +234,17 @@ export default class JanInferenceNitroExtension implements InferenceExtension { updated: timestamp, object: "thread.message", }; - events.emit(EventName.OnMessageResponse, message); + events.emit(MessageEvent.OnMessageResponse, message); this.isCancelled = false; this.controller = new AbortController(); - requestInference( - data.messages ?? [], - { ...this._currentModel, ...data.model }, - this.controller - ).subscribe({ + // @ts-ignore + const model: Model = { + ...(this._currentModel || {}), + ...(data.model || {}), + }; + requestInference(data.messages ?? [], model, this.controller).subscribe({ next: (content) => { const messageContent: ThreadContent = { type: ContentType.Text, @@ -256,22 +254,22 @@ export default class JanInferenceNitroExtension implements InferenceExtension { }, }; message.content = [messageContent]; - events.emit(EventName.OnMessageUpdate, message); + events.emit(MessageEvent.OnMessageUpdate, message); }, complete: async () => { message.status = message.content.length ? MessageStatus.Ready : MessageStatus.Error; - events.emit(EventName.OnMessageUpdate, message); + events.emit(MessageEvent.OnMessageUpdate, message); }, error: async (err) => { if (this.isCancelled || message.content.length) { message.status = MessageStatus.Stopped; - events.emit(EventName.OnMessageUpdate, message); + events.emit(MessageEvent.OnMessageUpdate, message); return; } message.status = MessageStatus.Error; - events.emit(EventName.OnMessageUpdate, message); + events.emit(MessageEvent.OnMessageUpdate, message); log(`[APP]::Error: ${err.message}`); }, }); diff --git a/nitro-node/src/module.ts b/nitro-node/src/module.ts index 6907f244a..c80d9c39c 100644 --- a/nitro-node/src/module.ts +++ b/nitro-node/src/module.ts @@ -23,11 +23,11 @@ const NVIDIA_INFO_FILE = path.join( ); // The subprocess instance for Nitro -let subprocess = undefined; -let currentModelFile: string = undefined; +let subprocess: any | undefined = undefined; +let currentModelFile: string = ''; let currentSettings = undefined; -let nitroProcessInfo = undefined; +let nitroProcessInfo: any | undefined = undefined; /** * Default GPU settings @@ -241,7 +241,7 @@ async function killSubprocess(): Promise { subprocess?.kill(); subprocess = undefined; }) - .catch(() => {}) + .catch(() => { }) .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)); } diff --git a/nitro-node/src/node/execute.ts b/nitro-node/src/node/execute.ts new file mode 100644 index 000000000..ca266639c --- /dev/null +++ b/nitro-node/src/node/execute.ts @@ -0,0 +1,65 @@ +import { readFileSync } from "fs"; +import * as path from "path"; +import { NVIDIA_INFO_FILE } from "./nvidia"; + +export interface NitroExecutableOptions { + executablePath: string; + cudaVisibleDevices: string; +} +/** + * Find which executable file to run based on the current platform. + * @returns The name of the executable file to run. + */ +export const executableNitroFile = (): NitroExecutableOptions => { + let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default + let cudaVisibleDevices = ""; + let binaryName = "nitro"; + /** + * The binary folder is different for each platform. + */ + if (process.platform === "win32") { + /** + * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 + */ + let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + if (nvidiaInfo["run_mode"] === "cpu") { + binaryFolder = path.join(binaryFolder, "win-cpu"); + } else { + if (nvidiaInfo["cuda"].version === "12") { + binaryFolder = path.join(binaryFolder, "win-cuda-12-0"); + } else { + binaryFolder = path.join(binaryFolder, "win-cuda-11-7"); + } + cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; + } + binaryName = "nitro.exe"; + } else if (process.platform === "darwin") { + /** + * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) + */ + if (process.arch === "arm64") { + binaryFolder = path.join(binaryFolder, "mac-arm64"); + } else { + binaryFolder = path.join(binaryFolder, "mac-x64"); + } + } else { + /** + * For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0 + */ + let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + if (nvidiaInfo["run_mode"] === "cpu") { + binaryFolder = path.join(binaryFolder, "linux-cpu"); + } else { + if (nvidiaInfo["cuda"].version === "12") { + binaryFolder = path.join(binaryFolder, "linux-cuda-12-0"); + } else { + binaryFolder = path.join(binaryFolder, "linux-cuda-11-7"); + } + cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; + } + } + return { + executablePath: path.join(binaryFolder, binaryName), + cudaVisibleDevices, + }; +}; diff --git a/nitro-node/src/node/index.ts b/nitro-node/src/node/index.ts new file mode 100644 index 000000000..765b2240f --- /dev/null +++ b/nitro-node/src/node/index.ts @@ -0,0 +1,379 @@ +import fs from "fs"; +import path from "path"; +import { ChildProcessWithoutNullStreams, spawn } from "child_process"; +import tcpPortUsed from "tcp-port-used"; +import fetchRT from "fetch-retry"; +import osUtils from "os-utils"; +import { log } from "@janhq/core/node"; +import { getNitroProcessInfo, updateNvidiaInfo } from "./nvidia"; +import { Model, InferenceEngine, ModelSettingParams } from "@janhq/core"; +import { executableNitroFile } from "./execute"; +import { homedir } from "os"; +// Polyfill fetch with retry +const fetchRetry = fetchRT(fetch); + +/** + * The response object for model init operation. + */ +interface ModelInitOptions { + modelFullPath: string; + model: Model; +} + +/** + * The response object of Prompt Template parsing. + */ +interface PromptTemplate { + system_prompt?: string; + ai_prompt?: string; + user_prompt?: string; + error?: string; +} + +/** + * Model setting args for Nitro model load. + */ +interface ModelSettingArgs extends ModelSettingParams { + llama_model_path: string; + cpu_threads: number; +} + +// The PORT to use for the Nitro subprocess +const PORT = 3928; +// The HOST address to use for the Nitro subprocess +const LOCAL_HOST = "127.0.0.1"; +// The URL for the Nitro subprocess +const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`; +// The URL for the Nitro subprocess to load a model +const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`; +// The URL for the Nitro subprocess to validate a model +const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; +// The URL for the Nitro subprocess to kill itself +const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; + +// The supported model format +// TODO: Should be an array to support more models +const SUPPORTED_MODEL_FORMAT = ".gguf"; + +// The subprocess instance for Nitro +let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; +// The current model file url +let currentModelFile: string = ""; +// The current model settings +let currentSettings: ModelSettingArgs | undefined = undefined; + +/** + * Stops a Nitro subprocess. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +function stopModel(): Promise { + return killSubprocess(); +} + +/** + * Initializes a Nitro subprocess to load a machine learning model. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package + */ +async function runModel( + wrapper: ModelInitOptions +): Promise { + if (wrapper.model.engine !== InferenceEngine.nitro) { + // Not a nitro model + return Promise.resolve(); + } + + currentModelFile = wrapper.modelFullPath; + const janRoot = path.join(homedir(), "jan"); + if (!currentModelFile.includes(janRoot)) { + currentModelFile = path.join(janRoot, currentModelFile); + } + const files: string[] = fs.readdirSync(currentModelFile); + + // Look for GGUF model file + const ggufBinFile = files.find( + (file) => + file === path.basename(currentModelFile) || + file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT) + ); + + if (!ggufBinFile) return Promise.reject("No GGUF model file found"); + + currentModelFile = path.join(currentModelFile, ggufBinFile); + + if (wrapper.model.engine !== InferenceEngine.nitro) { + return Promise.reject("Not a nitro model"); + } else { + const nitroResourceProbe = await getResourcesInfo(); + // Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt + if (wrapper.model.settings.prompt_template) { + const promptTemplate = wrapper.model.settings.prompt_template; + const prompt = promptTemplateConverter(promptTemplate); + if (prompt?.error) { + return Promise.reject(prompt.error); + } + wrapper.model.settings.system_prompt = prompt.system_prompt; + wrapper.model.settings.user_prompt = prompt.user_prompt; + wrapper.model.settings.ai_prompt = prompt.ai_prompt; + } + + currentSettings = { + llama_model_path: currentModelFile, + ...wrapper.model.settings, + // This is critical and requires real system information + cpu_threads: Math.max(1, Math.round(nitroResourceProbe.numCpuPhysicalCore / 2)), + }; + return runNitroAndLoadModel(); + } +} + +/** + * 1. Spawn Nitro process + * 2. Load model into Nitro subprocess + * 3. Validate model status + * @returns + */ +async function runNitroAndLoadModel() { + // Gather system information for CPU physical cores and memory + return killSubprocess() + .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) + .then(() => { + /** + * There is a problem with Windows process manager + * Should wait for awhile to make sure the port is free and subprocess is killed + * The tested threshold is 500ms + **/ + if (process.platform === "win32") { + return new Promise((resolve) => setTimeout(resolve, 500)); + } else { + return Promise.resolve(); + } + }) + .then(spawnNitroProcess) + .then(() => loadLLMModel(currentSettings)) + .then(validateModelStatus) + .catch((err) => { + // TODO: Broadcast error so app could display proper error message + log(`[NITRO]::Error: ${err}`); + return { error: err }; + }); +} + +/** + * Parse prompt template into agrs settings + * @param promptTemplate Template as string + * @returns + */ +function promptTemplateConverter(promptTemplate: string): PromptTemplate { + // Split the string using the markers + const systemMarker = "{system_message}"; + const promptMarker = "{prompt}"; + + if ( + promptTemplate.includes(systemMarker) && + promptTemplate.includes(promptMarker) + ) { + // Find the indices of the markers + const systemIndex = promptTemplate.indexOf(systemMarker); + const promptIndex = promptTemplate.indexOf(promptMarker); + + // Extract the parts of the string + const system_prompt = promptTemplate.substring(0, systemIndex); + const user_prompt = promptTemplate.substring( + systemIndex + systemMarker.length, + promptIndex + ); + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length + ); + + // Return the split parts + return { system_prompt, user_prompt, ai_prompt }; + } else if (promptTemplate.includes(promptMarker)) { + // Extract the parts of the string for the case where only promptMarker is present + const promptIndex = promptTemplate.indexOf(promptMarker); + const user_prompt = promptTemplate.substring(0, promptIndex); + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length + ); + + // Return the split parts + return { user_prompt, ai_prompt }; + } + + // Return an error if none of the conditions are met + return { error: "Cannot split prompt template" }; +} + +/** + * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + */ +function loadLLMModel(settings: any): Promise { + log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`); + return fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(settings), + retries: 3, + retryDelay: 500, + }) + .then((res) => { + log( + `[NITRO]::Debug: Load model success with response ${JSON.stringify( + res + )}` + ); + return Promise.resolve(res); + }) + .catch((err) => { + log(`[NITRO]::Error: Load model failed with error ${err}`); + return Promise.reject(); + }); +} + +/** + * Validates the status of a model. + * @returns {Promise} A promise that resolves to an object. + * If the model is loaded successfully, the object is empty. + * If the model is not loaded successfully, the object contains an error message. + */ +async function validateModelStatus(): Promise { + // Send a GET request to the validation URL. + // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. + return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + retries: 5, + retryDelay: 500, + }).then(async (res: Response) => { + log( + `[NITRO]::Debug: Validate model state success with response ${JSON.stringify( + res + )}` + ); + // If the response is OK, check model_loaded status. + if (res.ok) { + const body = await res.json(); + // If the model is loaded, return an empty object. + // Otherwise, return an object with an error message. + if (body.model_loaded) { + return Promise.resolve(); + } + } + return Promise.reject("Validate model status failed"); + }); +} + +/** + * Terminates the Nitro subprocess. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +async function killSubprocess(): Promise { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 5000); + log(`[NITRO]::Debug: Request to kill Nitro`); + + return fetch(NITRO_HTTP_KILL_URL, { + method: "DELETE", + signal: controller.signal, + }) + .then(() => { + subprocess?.kill(); + subprocess = undefined; + }) + .catch(() => {}) + .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) + .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)); +} + +/** + * Spawns a Nitro subprocess. + * @returns A promise that resolves when the Nitro subprocess is started. + */ +function spawnNitroProcess(): Promise { + log(`[NITRO]::Debug: Spawning Nitro subprocess...`); + + return new Promise(async (resolve, reject) => { + let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default + let executableOptions = executableNitroFile(); + + const args: string[] = ["1", LOCAL_HOST, PORT.toString()]; + // Execute the binary + log( + `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}` + ); + subprocess = spawn( + executableOptions.executablePath, + ["1", LOCAL_HOST, PORT.toString()], + { + cwd: binaryFolder, + env: { + ...process.env, + CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices, + }, + } + ); + + // Handle subprocess output + subprocess.stdout.on("data", (data: any) => { + log(`[NITRO]::Debug: ${data}`); + }); + + subprocess.stderr.on("data", (data: any) => { + log(`[NITRO]::Error: ${data}`); + }); + + subprocess.on("close", (code: any) => { + log(`[NITRO]::Debug: Nitro exited with code: ${code}`); + subprocess = undefined; + reject(`child process exited with code ${code}`); + }); + + tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { + log(`[NITRO]::Debug: Nitro is ready`); + resolve(); + }); + }); +} + +/** + * Get the system resources information + * TODO: Move to Core so that it can be reused + */ +function getResourcesInfo(): Promise { + return new Promise(async (resolve) => { + const cpu = await osUtils.cpuCount(); + log(`[NITRO]::CPU informations - ${cpu}`); + const response: ResourcesInfo = { + numCpuPhysicalCore: cpu, + memAvailable: 0, + }; + resolve(response); + }); +} + +/** + * Every module should have a dispose function + * This will be called when the extension is unloaded and should clean up any resources + * Also called when app is closed + */ +function dispose() { + // clean other registered resources here + killSubprocess(); +} + +export default { + runModel, + stopModel, + killSubprocess, + dispose, + updateNvidiaInfo, + getCurrentNitroProcessInfo: () => getNitroProcessInfo(subprocess), +}; diff --git a/nitro-node/src/node/nvidia.ts b/nitro-node/src/node/nvidia.ts new file mode 100644 index 000000000..ddd5719e1 --- /dev/null +++ b/nitro-node/src/node/nvidia.ts @@ -0,0 +1,201 @@ +import { writeFileSync, existsSync, readFileSync } from "fs"; +import { exec } from "child_process"; +import path from "path"; +import { homedir } from "os"; + +/** + * Default GPU settings + **/ +const DEFALT_SETTINGS = { + notify: true, + run_mode: "cpu", + nvidia_driver: { + exist: false, + version: "", + }, + cuda: { + exist: false, + version: "", + }, + gpus: [], + gpu_highest_vram: "", +}; + +/** + * Path to the settings file + **/ +export const NVIDIA_INFO_FILE = path.join( + homedir(), + "jan", + "settings", + "settings.json" +); + +/** + * Current nitro process + */ +let nitroProcessInfo: NitroProcessInfo | undefined = undefined; + +/** + * Nitro process info + */ +export interface NitroProcessInfo { + isRunning: boolean +} + +/** + * This will retrive GPU informations and persist settings.json + * Will be called when the extension is loaded to turn on GPU acceleration if supported + */ +export async function updateNvidiaInfo() { + if (process.platform !== "darwin") { + await Promise.all([ + updateNvidiaDriverInfo(), + updateCudaExistence(), + updateGpuInfo(), + ]); + } +} + +/** + * Retrieve current nitro process + */ +export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { + nitroProcessInfo = { + isRunning: subprocess != null, + }; + return nitroProcessInfo; +}; + +/** + * Validate nvidia and cuda for linux and windows + */ +export async function updateNvidiaDriverInfo(): Promise { + exec( + "nvidia-smi --query-gpu=driver_version --format=csv,noheader", + (error, stdout) => { + let data; + try { + data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + } catch (error) { + data = DEFALT_SETTINGS; + } + + if (!error) { + const firstLine = stdout.split("\n")[0].trim(); + data["nvidia_driver"].exist = true; + data["nvidia_driver"].version = firstLine; + } else { + data["nvidia_driver"].exist = false; + } + + writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); + Promise.resolve(); + } + ); +} + +/** + * Check if file exists in paths + */ +export function checkFileExistenceInPaths( + file: string, + paths: string[] +): boolean { + return paths.some((p) => existsSync(path.join(p, file))); +} + +/** + * Validate cuda for linux and windows + */ +export function updateCudaExistence() { + let filesCuda12: string[]; + let filesCuda11: string[]; + let paths: string[]; + let cudaVersion: string = ""; + + if (process.platform === "win32") { + filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; + filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; + paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : []; + } else { + filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; + filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; + paths = process.env.LD_LIBRARY_PATH + ? process.env.LD_LIBRARY_PATH.split(path.delimiter) + : []; + paths.push("/usr/lib/x86_64-linux-gnu/"); + } + + let cudaExists = filesCuda12.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ); + + if (!cudaExists) { + cudaExists = filesCuda11.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ); + if (cudaExists) { + cudaVersion = "11"; + } + } else { + cudaVersion = "12"; + } + + let data; + try { + data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + } catch (error) { + data = DEFALT_SETTINGS; + } + + data["cuda"].exist = cudaExists; + data["cuda"].version = cudaVersion; + if (cudaExists) { + data.run_mode = "gpu"; + } + writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); +} + +/** + * Get GPU information + */ +export async function updateGpuInfo(): Promise { + exec( + "nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", + (error, stdout) => { + let data; + try { + data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); + } catch (error) { + data = DEFALT_SETTINGS; + } + + if (!error) { + // Get GPU info and gpu has higher memory first + let highestVram = 0; + let highestVramId = "0"; + let gpus = stdout + .trim() + .split("\n") + .map((line) => { + let [id, vram] = line.split(", "); + vram = vram.replace(/\r/g, ""); + if (parseFloat(vram) > highestVram) { + highestVram = parseFloat(vram); + highestVramId = id; + } + return { id, vram }; + }); + + data["gpus"] = gpus; + data["gpu_highest_vram"] = highestVramId; + } else { + data["gpus"] = []; + } + + writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); + Promise.resolve(); + } + ); +} diff --git a/nitro-node/tsconfig.json b/nitro-node/tsconfig.json index b48175a16..a2cee893d 100644 --- a/nitro-node/tsconfig.json +++ b/nitro-node/tsconfig.json @@ -1,15 +1,29 @@ { "compilerOptions": { - "target": "es2016", - "module": "ES6", "moduleResolution": "node", - - "outDir": "./dist", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": false, - "skipLibCheck": true, - "rootDir": "./src" + "target": "es5", + "module": "ES2020", + "lib": [ + "es2015", + "es2016", + "es2017", + "dom" + ], + "strict": true, + "noImplicitAny": false, // FIXME: some code lines still use wrong types or no explicit types + "sourceMap": true, + "declaration": true, + "allowSyntheticDefaultImports": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "declarationDir": "dist/types", + "outDir": "dist", + "importHelpers": true, + "typeRoots": [ + "node_modules/@types" + ] }, - "include": ["./src"] + "include": [ + "src" + ] } From 85e3189733e1e1f9d96dbe2065531bfd19af3fe0 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Mon, 22 Jan 2024 15:24:53 +0700 Subject: [PATCH 04/49] chore(build): make clean all cached packages and yarn versions --- nitro-node/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nitro-node/Makefile b/nitro-node/Makefile index f862fa8fa..f5e2c3f7e 100644 --- a/nitro-node/Makefile +++ b/nitro-node/Makefile @@ -3,7 +3,7 @@ # Default target, build all .PHONY: all -all: clean publish +all: publish # Installs yarn dependencies #install: build-core @@ -27,10 +27,10 @@ publish: build download-nitro clean: ifeq ($(OS),Windows_NT) - del /F /S *.tgz .yarn yarn.lock + del /F /S *.tgz .yarn yarn.lock package-lock.json powershell -Command "Get-ChildItem -Path . -Include node_modules, dist -Recurse -Directory | Remove-Item -Recurse -Force" else - rm -rf *.tgz .yarn yarn.lock + rm -rf *.tgz .yarn yarn.lock package-lock.json find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + endif From 55c540dfb368c5939b161e9da206ac6bc6fce5f7 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Mon, 22 Jan 2024 20:19:52 +0700 Subject: [PATCH 05/49] chore(build): download nitro before packing nitro-node - Remove download script for windows - Add node-js script to help download nitro binary --- nitro-node/Makefile | 4 +- nitro-node/download-nitro.js | 129 +++++++++++++++++++++++++++++++++++ nitro-node/download.bat | 3 - nitro-node/package.json | 10 +-- 4 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 nitro-node/download-nitro.js delete mode 100644 nitro-node/download.bat diff --git a/nitro-node/Makefile b/nitro-node/Makefile index f5e2c3f7e..13a45e73b 100644 --- a/nitro-node/Makefile +++ b/nitro-node/Makefile @@ -22,15 +22,17 @@ download-nitro: install yarn run downloadnitro # Builds and publishes the extension -publish: build download-nitro +publish: build yarn run publish clean: ifeq ($(OS),Windows_NT) del /F /S *.tgz .yarn yarn.lock package-lock.json powershell -Command "Get-ChildItem -Path . -Include node_modules, dist -Recurse -Directory | Remove-Item -Recurse -Force" + powershell -Command "Get-ChildItem -Path .\bin -Recurse -Directory | Remove-Item -Recurse -Force" else rm -rf *.tgz .yarn yarn.lock package-lock.json find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + + find ./bin -type d -mindepth 1 -maxdepth 1 -exec rm -rf '{}' + endif diff --git a/nitro-node/download-nitro.js b/nitro-node/download-nitro.js new file mode 100644 index 000000000..35ab191f3 --- /dev/null +++ b/nitro-node/download-nitro.js @@ -0,0 +1,129 @@ +const https = require('https'); +const url = require('url'); +const fs = require('fs'); +const path = require('path'); +const zlib = require('zlib'); +const EventEmitter = require('events'); + +// Define nitro version to download in this file +const VERSION_TXT = path.join('.', 'bin', 'version.txt'); +// Read nitro version +const NITRO_VERSION = fs.readFileSync(VERSION_TXT, 'utf8').trim(); +// The platform OS to download nitro for +const PLATFORM = process.env.npm_config_platform || process.platform; +// The platform architecture +//const ARCH = process.env.npm_config_arch || process.arch; + +const linuxVariants = { + 'linux-amd64': path.join('.', 'bin', 'linux-cpu'), + 'linux-amd64-cuda-12-0': path.join('.', 'bin', 'linux-cuda-12-0', 'nitro'), + 'linux-amd64-cuda-11-7': path.join('.', 'bin', 'linux-cuda-11-7', 'nitro'), +} + +const darwinVariants = { + 'mac-arm64': path.join('.', 'bin', 'mac-arm64', 'nitro'), + 'mac-amd64': path.join('.', 'bin', 'mac-x64', 'nitro'), +} + +const win32Variants = { + 'win-amd64-cuda-12-0': path.join('.', 'bin', 'win-cuda-12-0', 'nitro.exe'), + 'win-amd64-cuda-11-7': path.join('.', 'bin', 'win-cuda-11-7', 'nitro.exe'), + 'win-amd64': path.join('.', 'bin', 'win-cpu', 'nitro.exe'), +} + +// Mapping to installation variants +const variantMapping = { + 'darwin': darwinVariants, + 'linux': linuxVariants, + 'win32': win32Variants, +} + +if (!(PLATFORM in variantMapping)) { + throw Error(`Invalid platform: ${PLATFORM}`); +} +// Get download config for this platform +const variantConfig = variantMapping[PLATFORM]; + +// Generate download link for each tarball +const getTarUrl = (version, suffix) => `https://github.com/janhq/nitro/releases/download/v${version}/nitro-${version}-${suffix}.tar.gz` + +// Create filestream to write the binary +const createOutputFileStream = async (filePath) => { + // Create immediate directories if they do not exist + const dir = path.dirname(filePath); + const createDir = await fs.mkdirSync(dir, { recursive: true }); + if (createDir) console.log(`created ${createDir}`); + return fs.createWriteStream(filePath); +} + +// Extract tarball and write to file from the response content +const writeToFile = (fileStream, variant = '?') => (response) => { + const totalLength = parseInt(response.headers['content-length'], 10); + console.log(`[${variant}] Need to download ${totalLength} bytes...`); + + // TODO: promises in for loop does not actually wait so output will be gobbled + //let currentLength = 0; + //response.on('data', (chunk) => { + // currentLength += chunk.length; + // const percentage = Math.floor(100.0 * currentLength / totalLength); + // if (percentage % 10) { + // process.stdout.write(`\r[${variant}] ${percentage}%...`); + // } + //}); + + response.on('end', () => { + console.log(`[${variant}] Finished downloading!`); + }); + + return response.pipe(zlib.createUnzip()).pipe(fileStream); +} + +const promisifyWriter = (writer) => { + const eventEmitter = new EventEmitter(); + return ({ + p: new Promise((resolve, reject) => { + eventEmitter.on('end', resolve); + eventEmitter.on('error', reject); + }), + callback(response) { + writer(response).on('error', (err) => eventEmitter.emit('error', err)).on('end', () => eventEmitter.emit('end')); + } + }); +} + +// Download single binary +const downloadBinary = async (version, suffix, filePath) => { + const tarUrl = getTarUrl(version, suffix); + console.log(`Downloading ${tarUrl} to ${filePath}`); + const fileStream = await createOutputFileStream(filePath); + const waitable = promisifyWriter(writeToFile(fileStream, suffix)); + const writer = waitable.callback; + https.get(tarUrl, (response) => { + if (response.statusCode > 300 && response.statusCode < 400 && response.headers.location) { + if (new URL(response.headers.location).hostname) { + https.get(response.headers.location, writer); + } else { + https.get( + url.resolve( + new URL(tarUrl).hostname, + response.headers.location + ), + writer + ); + } + } else { + writer(response); + } + }); + await waitable.p; +} + +// Download the binaries +const downloadBinaries = async (version, config) => { + await Promise.all(Object.entries(config).map(async ([k, v]) => { + await downloadBinary(version, k, v); + })); +} + +// Call the download function with version and config +downloadBinaries(NITRO_VERSION, variantConfig); diff --git a/nitro-node/download.bat b/nitro-node/download.bat deleted file mode 100644 index 22e1c85b3..000000000 --- a/nitro-node/download.bat +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -set /p NITRO_VERSION=<./bin/version.txt -.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu diff --git a/nitro-node/package.json b/nitro-node/package.json index d386f7250..95b278f51 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -8,14 +8,13 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro", - "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro", - "downloadnitro:win32": "download.bat", - "downloadnitro": "run-script-os", + "downloadnitro": "node download-nitro.js", "publish:darwin": "../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack", "publish:win32": "cpx \"bin/**\" \"dist/bin\" && npm pack", "publish:linux": "cpx \"bin/**\" \"dist/bin\" && npm pack", - "publish": "run-script-os" + "publish": "run-script-os", + "postinstall": "yarn run downloadnitro", + "prepack": "yarn run downloadnitro" }, "exports": { ".": "./dist/index.js", @@ -52,6 +51,7 @@ "node": ">=18.0.0" }, "files": [ + "download-nitro.js", "dist/*", "package.json", "README.md" From 8ba9202e7a23264fbe3d17414ea64a5f95fd4245 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 23 Jan 2024 02:59:07 +0700 Subject: [PATCH 06/49] chore(build): hook for installing nitro when using as dependency - Cleanup webpack config - Fix script download nitro - Add postinstall script - Fix prepack script - Also cleanup ./bin on make clean --- nitro-node/Makefile | 8 +-- nitro-node/download-nitro.js | 126 ++++++++++++----------------------- nitro-node/package.json | 17 ++--- nitro-node/postinstall.js | 6 ++ nitro-node/webpack.config.js | 43 ------------ 5 files changed, 61 insertions(+), 139 deletions(-) create mode 100644 nitro-node/postinstall.js delete mode 100644 nitro-node/webpack.config.js diff --git a/nitro-node/Makefile b/nitro-node/Makefile index 13a45e73b..e2d9195bb 100644 --- a/nitro-node/Makefile +++ b/nitro-node/Makefile @@ -23,16 +23,14 @@ download-nitro: install # Builds and publishes the extension publish: build - yarn run publish + yarn run build:publish clean: ifeq ($(OS),Windows_NT) - del /F /S *.tgz .yarn yarn.lock package-lock.json + del /F /S *.tgz .yarn yarn.lock package-lock.json bin powershell -Command "Get-ChildItem -Path . -Include node_modules, dist -Recurse -Directory | Remove-Item -Recurse -Force" - powershell -Command "Get-ChildItem -Path .\bin -Recurse -Directory | Remove-Item -Recurse -Force" else - rm -rf *.tgz .yarn yarn.lock package-lock.json + rm -rf *.tgz .yarn yarn.lock package-lock.json bin find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + - find ./bin -type d -mindepth 1 -maxdepth 1 -exec rm -rf '{}' + endif diff --git a/nitro-node/download-nitro.js b/nitro-node/download-nitro.js index 35ab191f3..50482251b 100644 --- a/nitro-node/download-nitro.js +++ b/nitro-node/download-nitro.js @@ -1,14 +1,11 @@ -const https = require('https'); -const url = require('url'); const fs = require('fs'); const path = require('path'); -const zlib = require('zlib'); -const EventEmitter = require('events'); +const download = require('download'); +const stream = require('stream'); +const pipeline = stream.promises.pipeline; -// Define nitro version to download in this file -const VERSION_TXT = path.join('.', 'bin', 'version.txt'); -// Read nitro version -const NITRO_VERSION = fs.readFileSync(VERSION_TXT, 'utf8').trim(); +// Define nitro version to download in env variable +const NITRO_VERSION = process.env.NITRO_VERSION || '0.2.11'; // The platform OS to download nitro for const PLATFORM = process.env.npm_config_platform || process.platform; // The platform architecture @@ -16,19 +13,19 @@ const PLATFORM = process.env.npm_config_platform || process.platform; const linuxVariants = { 'linux-amd64': path.join('.', 'bin', 'linux-cpu'), - 'linux-amd64-cuda-12-0': path.join('.', 'bin', 'linux-cuda-12-0', 'nitro'), - 'linux-amd64-cuda-11-7': path.join('.', 'bin', 'linux-cuda-11-7', 'nitro'), + 'linux-amd64-cuda-12-0': path.join('.', 'bin', 'linux-cuda-12-0'), + 'linux-amd64-cuda-11-7': path.join('.', 'bin', 'linux-cuda-11-7'), } const darwinVariants = { - 'mac-arm64': path.join('.', 'bin', 'mac-arm64', 'nitro'), - 'mac-amd64': path.join('.', 'bin', 'mac-x64', 'nitro'), + 'mac-arm64': path.join('.', 'bin', 'mac-arm64'), + 'mac-amd64': path.join('.', 'bin', 'mac-x64'), } const win32Variants = { - 'win-amd64-cuda-12-0': path.join('.', 'bin', 'win-cuda-12-0', 'nitro.exe'), - 'win-amd64-cuda-11-7': path.join('.', 'bin', 'win-cuda-11-7', 'nitro.exe'), - 'win-amd64': path.join('.', 'bin', 'win-cpu', 'nitro.exe'), + 'win-amd64-cuda-12-0': path.join('.', 'bin', 'win-cuda-12-0'), + 'win-amd64-cuda-11-7': path.join('.', 'bin', 'win-cuda-11-7'), + 'win-amd64': path.join('.', 'bin', 'win-cpu'), } // Mapping to installation variants @@ -47,83 +44,46 @@ const variantConfig = variantMapping[PLATFORM]; // Generate download link for each tarball const getTarUrl = (version, suffix) => `https://github.com/janhq/nitro/releases/download/v${version}/nitro-${version}-${suffix}.tar.gz` -// Create filestream to write the binary -const createOutputFileStream = async (filePath) => { - // Create immediate directories if they do not exist - const dir = path.dirname(filePath); - const createDir = await fs.mkdirSync(dir, { recursive: true }); - if (createDir) console.log(`created ${createDir}`); - return fs.createWriteStream(filePath); -} - -// Extract tarball and write to file from the response content -const writeToFile = (fileStream, variant = '?') => (response) => { - const totalLength = parseInt(response.headers['content-length'], 10); - console.log(`[${variant}] Need to download ${totalLength} bytes...`); - - // TODO: promises in for loop does not actually wait so output will be gobbled - //let currentLength = 0; - //response.on('data', (chunk) => { - // currentLength += chunk.length; - // const percentage = Math.floor(100.0 * currentLength / totalLength); - // if (percentage % 10) { - // process.stdout.write(`\r[${variant}] ${percentage}%...`); - // } - //}); - - response.on('end', () => { +// Report download progress +const createProgressReporter = (variant) => (stream) => stream.on( + 'downloadProgress', + (progress) => { + process.stdout.write(`\r[${variant}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`); + }).on('finish', () => { console.log(`[${variant}] Finished downloading!`); - }); - - return response.pipe(zlib.createUnzip()).pipe(fileStream); -} - -const promisifyWriter = (writer) => { - const eventEmitter = new EventEmitter(); - return ({ - p: new Promise((resolve, reject) => { - eventEmitter.on('end', resolve); - eventEmitter.on('error', reject); - }), - callback(response) { - writer(response).on('error', (err) => eventEmitter.emit('error', err)).on('end', () => eventEmitter.emit('end')); - } - }); -} + }) // Download single binary -const downloadBinary = async (version, suffix, filePath) => { +const downloadBinary = (version, suffix, filePath) => { const tarUrl = getTarUrl(version, suffix); console.log(`Downloading ${tarUrl} to ${filePath}`); - const fileStream = await createOutputFileStream(filePath); - const waitable = promisifyWriter(writeToFile(fileStream, suffix)); - const writer = waitable.callback; - https.get(tarUrl, (response) => { - if (response.statusCode > 300 && response.statusCode < 400 && response.headers.location) { - if (new URL(response.headers.location).hostname) { - https.get(response.headers.location, writer); - } else { - https.get( - url.resolve( - new URL(tarUrl).hostname, - response.headers.location - ), - writer - ); - } - } else { - writer(response); - } - }); - await waitable.p; + const progressReporter = createProgressReporter(suffix); + return progressReporter( + download(tarUrl, filePath, { + strip: 1, + extract: true, + }) + ); } // Download the binaries const downloadBinaries = async (version, config) => { - await Promise.all(Object.entries(config).map(async ([k, v]) => { - await downloadBinary(version, k, v); - })); + await Object.entries(config).reduce( + async (p, [k, v]) => { + p.then(() => downloadBinary(version, k, v)); + }, + Promise.resolve(), + ); } // Call the download function with version and config -downloadBinaries(NITRO_VERSION, variantConfig); +const run = () => { + downloadBinaries(NITRO_VERSION, variantConfig); +} + +module.exports = run; + +// Run script if called directly instead of import as module +if (require.main === module) { + run(); +} diff --git a/nitro-node/package.json b/nitro-node/package.json index 95b278f51..ac3911e3e 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -9,12 +9,12 @@ "scripts": { "build": "tsc --module commonjs && rollup -c rollup.config.ts", "downloadnitro": "node download-nitro.js", - "publish:darwin": "../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack", - "publish:win32": "cpx \"bin/**\" \"dist/bin\" && npm pack", - "publish:linux": "cpx \"bin/**\" \"dist/bin\" && npm pack", - "publish": "run-script-os", - "postinstall": "yarn run downloadnitro", - "prepack": "yarn run downloadnitro" + "build:publish:darwin": "../.github/scripts/auto-sign.sh && npm pack", + "build:publish:win32": "npm pack", + "build:publish:linux": "npm pack", + "build:publish": "run-script-os", + "postinstall": "node postinstall.js", + "prepack": "cpx \"bin/**\" \"dist/bin\"" }, "exports": { ".": "./dist/index.js", @@ -26,8 +26,6 @@ "@rollup/plugin-node-resolve": "^15.2.3", "@types/node": "^20.11.4", "@types/tcp-port-used": "^1.0.4", - "cpx": "^1.5.0", - "download-cli": "^1.1.1", "rimraf": "^3.0.2", "rollup": "^2.38.5", "rollup-plugin-define": "^1.0.1", @@ -37,6 +35,8 @@ "typescript": "^5.3.3" }, "dependencies": { + "cpx": "^1.5.0", + "download": "^8.0.0", "@janhq/core": "^0.1.11", "@rollup/plugin-replace": "^5.0.5", "@types/os-utils": "^0.0.4", @@ -52,6 +52,7 @@ }, "files": [ "download-nitro.js", + "postinstall.js", "dist/*", "package.json", "README.md" diff --git a/nitro-node/postinstall.js b/nitro-node/postinstall.js new file mode 100644 index 000000000..18b9457af --- /dev/null +++ b/nitro-node/postinstall.js @@ -0,0 +1,6 @@ +// Only run if this package is installed as dependency +if (process.env.INIT_CWD === process.cwd()) + process.exit() + +const downloadNitro = require('./download-nitro'); +downloadNitro(); diff --git a/nitro-node/webpack.config.js b/nitro-node/webpack.config.js deleted file mode 100644 index 2927affbc..000000000 --- a/nitro-node/webpack.config.js +++ /dev/null @@ -1,43 +0,0 @@ -const path = require("path"); -const webpack = require("webpack"); -const packageJson = require("./package.json"); - -module.exports = { - experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", - module: { - rules: [ - { - test: /\.tsx?$/, - use: "ts-loader", - exclude: /node_modules/, - }, - ], - }, - plugins: [ - new webpack.DefinePlugin({ - MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - INFERENCE_URL: JSON.stringify( - process.env.INFERENCE_URL || - "http://127.0.0.1:3928/inferences/llamacpp/chat_completion" - ), - TROUBLESHOOTING_URL: JSON.stringify("https://jan.ai/guides/troubleshooting") - }), - ], - output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format - }, - resolve: { - extensions: [".ts", ".js"], - fallback: { - path: require.resolve("path-browserify"), - }, - }, - optimization: { - minimize: false, - }, - // Add loaders and other configuration as needed for your project -}; From 08657a1034cb29a01cd485b6dd80a7bce6b1db80 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 23 Jan 2024 12:10:02 +0700 Subject: [PATCH 07/49] refactor(WIP): decouple from @janhq/core --- nitro-node/README.md | 79 +--- nitro-node/download-nitro.js | 2 - nitro-node/package.json | 6 +- nitro-node/rollup.config.ts | 47 +-- nitro-node/src/@types/global.d.ts | 55 ++- nitro-node/src/{node => }/execute.ts | 2 + nitro-node/src/helpers/sse.ts | 65 --- nitro-node/src/index.ts | 565 +++++++++++++++------------ nitro-node/src/module.ts | 514 ------------------------ nitro-node/src/node/index.ts | 379 ------------------ nitro-node/src/{node => }/nvidia.ts | 0 nitro-node/tsconfig.json | 4 +- 12 files changed, 367 insertions(+), 1351 deletions(-) rename nitro-node/src/{node => }/execute.ts (99%) delete mode 100644 nitro-node/src/helpers/sse.ts delete mode 100644 nitro-node/src/module.ts delete mode 100644 nitro-node/src/node/index.ts rename nitro-node/src/{node => }/nvidia.ts (100%) diff --git a/nitro-node/README.md b/nitro-node/README.md index 455783efb..915057eb2 100644 --- a/nitro-node/README.md +++ b/nitro-node/README.md @@ -1,78 +1,3 @@ -# Jan inference plugin - -Created using Jan app example - -# Create a Jan Plugin using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 - -## Create Your Own Plugin - -To create your own plugin, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your plugin directory now - -## Update the Plugin Metadata - -The [`package.json`](package.json) file defines metadata about your plugin, such as -plugin name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your plugin. - -## Update the Plugin Code - -The [`src/`](./src/) directory is the heart of your plugin! This contains the -source code that will be run when your plugin extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your plugin code: - -- Most Jan Plugin Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { core } from "@janhq/core"; - - function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); - } - ``` - - For more information about the Jan Plugin Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your plugin! +# NodeJS wrapper for Nitro +**TODO** Documenting on usage diff --git a/nitro-node/download-nitro.js b/nitro-node/download-nitro.js index 50482251b..feb54251f 100644 --- a/nitro-node/download-nitro.js +++ b/nitro-node/download-nitro.js @@ -1,8 +1,6 @@ -const fs = require('fs'); const path = require('path'); const download = require('download'); const stream = require('stream'); -const pipeline = stream.promises.pipeline; // Define nitro version to download in env variable const NITRO_VERSION = process.env.NITRO_VERSION || '0.2.11'; diff --git a/nitro-node/package.json b/nitro-node/package.json index ac3911e3e..702a790bf 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -1,9 +1,9 @@ { "name": "@janhq/nitro-node", "version": "1.0.0", - "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See nitro.jan.ai", + "description": "This NodeJS library is a wrapper for Nitro, a lightweight (3mb) inference engine written in C++. See nitro.jan.ai", "main": "dist/index.js", - "node": "dist/node/index.cjs.js", + "node": "dist/index.cjs.js", "author": "Jan ", "license": "AGPL-3.0", "scripts": { @@ -18,7 +18,7 @@ }, "exports": { ".": "./dist/index.js", - "./main": "./dist/node/index.cjs.js" + "./main": "./dist/index.cjs.js" }, "devDependencies": { "@rollup/plugin-commonjs": "^25.0.7", diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts index 374a054cd..922a1fdfe 100644 --- a/nitro-node/rollup.config.ts +++ b/nitro-node/rollup.config.ts @@ -3,58 +3,17 @@ import commonjs from "@rollup/plugin-commonjs"; import sourceMaps from "rollup-plugin-sourcemaps"; import typescript from "rollup-plugin-typescript2"; import json from "@rollup/plugin-json"; -import replace from "@rollup/plugin-replace"; -const packageJson = require("./package.json"); - -const pkg = require("./package.json"); export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: "es", sourcemap: true }], - // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: [], - watch: { - include: "src/**", - }, - plugins: [ - replace({ - NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), - INFERENCE_URL: JSON.stringify( - process.env.INFERENCE_URL || - "http://127.0.0.1:3928/inferences/llamacpp/chat_completion" - ), - TROUBLESHOOTING_URL: JSON.stringify( - "https://jan.ai/guides/troubleshooting" - ), - }), - // Allow json resolution - json(), - // Compile TypeScript files - typescript({ useTsconfigDeclarationDir: true }), - // Compile TypeScript files - // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) - commonjs(), - // Allow node_modules resolution, so you can use 'external' to control - // which external modules to include in the bundle - // https://github.com/rollup/rollup-plugin-node-resolve#usage - resolve({ - extensions: [".js", ".ts", ".svelte"], - }), - - // Resolve source maps to the original source - sourceMaps(), - ], - }, - { - input: `src/node/index.ts`, output: [ - { file: "dist/node/index.cjs.js", format: "cjs", sourcemap: true }, + { file: "dist/index.cjs.js", format: "cjs", sourcemap: true }, ], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: ["@janhq/core/node"], + external: [], watch: { - include: "src/node/**", + include: "src/**", }, plugins: [ // Allow json resolution diff --git a/nitro-node/src/@types/global.d.ts b/nitro-node/src/@types/global.d.ts index 5fb41f0f8..bc49ddc6b 100644 --- a/nitro-node/src/@types/global.d.ts +++ b/nitro-node/src/@types/global.d.ts @@ -1,28 +1,8 @@ -declare const NODE: string; -declare const INFERENCE_URL: string; -declare const TROUBLESHOOTING_URL: string; - -/** - * The parameters for the initModel function. - * @property settings - The settings for the machine learning model. - * @property settings.ctx_len - The context length. - * @property settings.ngl - The number of generated tokens. - * @property settings.cont_batching - Whether to use continuous batching. - * @property settings.embedding - Whether to use embedding. - */ -interface EngineSettings { - ctx_len: number; - ngl: number; - cpu_threads: number; - cont_batching: boolean; - embedding: boolean; -} - /** * The response from the initModel function. * @property error - An error message if the model fails to load. */ -interface ModelOperationResponse { +interface NitroModelOperationResponse { error?: any; modelFile?: string; } @@ -30,4 +10,37 @@ interface ModelOperationResponse { interface ResourcesInfo { numCpuPhysicalCore: number; memAvailable: number; +} + +/** + * Setting for prompts when inferencing with Nitro + */ +interface NitroPromptSetting { + prompt_template?: string; + system_prompt?: string; + ai_prompt?: string; + user_prompt?: string; +} + +/** + * The available model settings + */ +interface NitroModelSetting extends NitroPromptSetting { + llama_model_path: string; + cpu_threads: number; +} + +/** + * The response object for model init operation. + */ +interface NitroModelInitOptions { + modelFullPath: string; + settings: NitroPromptSetting; +} + +/** + * Logging interface for passing custom logger to nitro-node + */ +interface NitroLogger { + (message: string, fileName?: string): void; } \ No newline at end of file diff --git a/nitro-node/src/node/execute.ts b/nitro-node/src/execute.ts similarity index 99% rename from nitro-node/src/node/execute.ts rename to nitro-node/src/execute.ts index ca266639c..6eb11b36d 100644 --- a/nitro-node/src/node/execute.ts +++ b/nitro-node/src/execute.ts @@ -2,10 +2,12 @@ import { readFileSync } from "fs"; import * as path from "path"; import { NVIDIA_INFO_FILE } from "./nvidia"; + export interface NitroExecutableOptions { executablePath: string; cudaVisibleDevices: string; } + /** * Find which executable file to run based on the current platform. * @returns The name of the executable file to run. diff --git a/nitro-node/src/helpers/sse.ts b/nitro-node/src/helpers/sse.ts deleted file mode 100644 index c6352383d..000000000 --- a/nitro-node/src/helpers/sse.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { Model } from "@janhq/core"; -import { Observable } from "rxjs"; -/** - * Sends a request to the inference server to generate a response based on the recent messages. - * @param recentMessages - An array of recent messages to use as context for the inference. - * @returns An Observable that emits the generated response as a string. - */ -export function requestInference( - recentMessages: any[], - model: Model, - controller?: AbortController -): Observable { - return new Observable((subscriber) => { - const requestBody = JSON.stringify({ - messages: recentMessages, - model: model.id, - stream: true, - ...model.parameters, - }); - fetch(INFERENCE_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - Accept: model.parameters.stream - ? "text/event-stream" - : "application/json", - }, - body: requestBody, - signal: controller?.signal, - }) - .then(async (response) => { - if (model.parameters.stream === false) { - const data = await response.json(); - subscriber.next(data.choices[0]?.message?.content ?? ""); - } else { - const stream = response.body; - const decoder = new TextDecoder("utf-8"); - const reader = stream?.getReader(); - let content = ""; - - while (true && reader) { - const { done, value } = await reader.read(); - if (done) { - break; - } - const text = decoder.decode(value); - const lines = text.trim().split("\n"); - for (const line of lines) { - if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { - const data = JSON.parse(line.replace("data: ", "")); - content += data.choices[0]?.delta?.content ?? ""; - if (content.startsWith("assistant: ")) { - content = content.replace("assistant: ", ""); - } - subscriber.next(content); - } - } - } - } - subscriber.complete(); - }) - .catch((err) => subscriber.error(err)); - }); -} diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index ba3705fde..cd90177e4 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -1,277 +1,356 @@ +import fs from "fs"; +import path from "path"; +import { ChildProcessWithoutNullStreams, spawn } from "child_process"; +import tcpPortUsed from "tcp-port-used"; +import fetchRT from "fetch-retry"; +import osUtils from "os-utils"; +import { getNitroProcessInfo, updateNvidiaInfo } from "./nvidia"; +import { executableNitroFile } from "./execute"; +// Polyfill fetch with retry +const fetchRetry = fetchRT(fetch); + /** - * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - * @version 1.0.0 - * @module inference-extension/src/index + * The response object of Prompt Template parsing. */ +type PromptTemplate = Omit & { + error?: string; +} -import { - ChatCompletionRole, - ContentType, - MessageRequest, - MessageStatus, - ThreadContent, - ThreadMessage, - events, - executeOnMain, - fs, - Model, - joinPath, - InferenceExtension, - log, - InferenceEngine, - MessageEvent, - ModelEvent, - InferenceEvent, -} from "@janhq/core"; -import { requestInference } from "./helpers/sse"; -import { ulid } from "ulid"; +// The PORT to use for the Nitro subprocess +const PORT = 3928; +// The HOST address to use for the Nitro subprocess +const LOCAL_HOST = "127.0.0.1"; +// The URL for the Nitro subprocess +const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`; +// The URL for the Nitro subprocess to load a model +const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`; +// The URL for the Nitro subprocess to validate a model +const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; +// The URL for the Nitro subprocess to kill itself +const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; + +// The supported model format +// TODO: Should be an array to support more models +const SUPPORTED_MODEL_FORMAT = ".gguf"; + +// The subprocess instance for Nitro +let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; +// The current model file url +let currentModelFile: string = ""; +// The current model settings +let currentSettings: NitroModelSetting | undefined = undefined; +// The logger to use, default to console.log +let log: NitroLogger = (message, ..._) => console.log(message); /** - * A class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + * Set logger before running nitro */ -export default class JanInferenceNitroExtension extends InferenceExtension { - private static readonly _homeDir = "file://engines"; - private static readonly _settingsDir = "file://settings"; - private static readonly _engineMetadataFileName = "nitro.json"; - - /** - * Checking the health for Nitro's process each 5 secs. - */ - private static readonly _intervalHealthCheck = 5 * 1000; - - private _currentModel: Model | undefined; - - private _engineSettings: EngineSettings = { - ctx_len: 2048, - ngl: 100, - cpu_threads: 1, - cont_batching: false, - embedding: false, - }; - - controller = new AbortController(); - isCancelled = false; - - /** - * The interval id for the health check. Used to stop the health check. - */ - private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = - undefined; - - /** - * Tracking the current state of nitro process. - */ - private nitroProcessInfo: any = undefined; - - /** - * Subscribes to events emitted by the @janhq/core package. - */ - async onLoad() { - if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) { - await fs - .mkdirSync(JanInferenceNitroExtension._homeDir) - .catch((err: Error) => console.debug(err)); - } - - if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir))) - await fs.mkdirSync(JanInferenceNitroExtension._settingsDir); - this.writeDefaultEngineSettings(); - - // Events subscription - events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.onMessageRequest(data) - ); - - events.on(ModelEvent.OnModelInit, (model: Model) => this.onModelInit(model)); - - events.on(ModelEvent.OnModelStop, (model: Model) => this.onModelStop(model)); - - events.on(InferenceEvent.OnInferenceStopped, () => this.onInferenceStopped()); +function setLogger(logger: NitroLogger) { + log = logger; +} - // Attempt to fetch nvidia info - await executeOnMain(NODE, "updateNvidiaInfo", {}); - } +/** + * Stops a Nitro subprocess. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +function stopModel(): Promise { + return killSubprocess(); +} - /** - * Stops the model inference. - */ - onUnload(): void {} - - private async writeDefaultEngineSettings() { - try { - const engineFile = await joinPath([ - JanInferenceNitroExtension._homeDir, - JanInferenceNitroExtension._engineMetadataFileName, - ]); - if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, "utf-8"); - this._engineSettings = - typeof engine === "object" ? engine : JSON.parse(engine); - } else { - await fs.writeFileSync( - engineFile, - JSON.stringify(this._engineSettings, null, 2) - ); - } - } catch (err) { - console.error(err); +/** + * Initializes a Nitro subprocess to load a machine learning model. + * @param modelFullPath - The absolute full path to model directory. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package + */ +async function runModel( + { + modelFullPath, + settings, + }: NitroModelInitOptions +): Promise { + const files: string[] = fs.readdirSync(modelFullPath); + + // Look for GGUF model file + const ggufBinFile = files.find( + (file) => + file === path.basename(modelFullPath) || + file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT) + ); + + if (!ggufBinFile) return Promise.reject("No GGUF model file found"); + + currentModelFile = path.join(modelFullPath, ggufBinFile); + + const nitroResourceProbe = await getResourcesInfo(); + // Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt + if (settings.prompt_template) { + const promptTemplate = settings.prompt_template; + const prompt = promptTemplateConverter(promptTemplate); + if (prompt?.error) { + return Promise.reject(prompt.error); } + settings.system_prompt = prompt.system_prompt; + settings.user_prompt = prompt.user_prompt; + settings.ai_prompt = prompt.ai_prompt; } - private async onModelInit(model: Model) { - if (model.engine !== InferenceEngine.nitro) return; - - const modelFullPath = await joinPath(["models", model.id]); + currentSettings = { + llama_model_path: currentModelFile, + ...settings, + // This is critical and requires real system information + cpu_threads: Math.max(1, Math.round(nitroResourceProbe.numCpuPhysicalCore / 2)), + }; + return runNitroAndLoadModel(); +} - const nitroInitResult = await executeOnMain(NODE, "runModel", { - modelFullPath, - model, +/** + * 1. Spawn Nitro process + * 2. Load model into Nitro subprocess + * 3. Validate model status + * @returns + */ +async function runNitroAndLoadModel(): Promise { + // Gather system information for CPU physical cores and memory + return killSubprocess() + .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) + .then(() => { + /** + * There is a problem with Windows process manager + * Should wait for awhile to make sure the port is free and subprocess is killed + * The tested threshold is 500ms + **/ + if (process.platform === "win32") { + return new Promise((resolve) => setTimeout(resolve, 500)); + } else { + return Promise.resolve(); + } + }) + .then(spawnNitroProcess) + .then(() => loadLLMModel(currentSettings)) + .then(validateModelStatus) + .catch((err) => { + // TODO: Broadcast error so app could display proper error message + log(`[NITRO]::Error: ${err}`); + return { error: err }; }); +} - if (nitroInitResult?.error) { - events.emit(ModelEvent.OnModelFail, model); - return; - } - - this._currentModel = model; - events.emit(ModelEvent.OnModelReady, model); +/** + * Parse prompt template into agrs settings + * @param promptTemplate Template as string + * @returns + */ +function promptTemplateConverter(promptTemplate: string): PromptTemplate { + // Split the string using the markers + const systemMarker = "{system_message}"; + const promptMarker = "{prompt}"; + + if ( + promptTemplate.includes(systemMarker) && + promptTemplate.includes(promptMarker) + ) { + // Find the indices of the markers + const systemIndex = promptTemplate.indexOf(systemMarker); + const promptIndex = promptTemplate.indexOf(promptMarker); + + // Extract the parts of the string + const system_prompt = promptTemplate.substring(0, systemIndex); + const user_prompt = promptTemplate.substring( + systemIndex + systemMarker.length, + promptIndex + ); + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length + ); - this.getNitroProcesHealthIntervalId = setInterval( - () => this.periodicallyGetNitroHealth(), - JanInferenceNitroExtension._intervalHealthCheck + // Return the split parts + return { system_prompt, user_prompt, ai_prompt }; + } else if (promptTemplate.includes(promptMarker)) { + // Extract the parts of the string for the case where only promptMarker is present + const promptIndex = promptTemplate.indexOf(promptMarker); + const user_prompt = promptTemplate.substring(0, promptIndex); + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length ); - } - private async onModelStop(model: Model) { - if (model.engine !== "nitro") return; + // Return the split parts + return { user_prompt, ai_prompt }; + } - await executeOnMain(NODE, "stopModel"); - events.emit(ModelEvent.OnModelStopped, {}); + // Return an error if none of the conditions are met + return { error: "Cannot split prompt template" }; +} - // stop the periocally health check - if (this.getNitroProcesHealthIntervalId) { - clearInterval(this.getNitroProcesHealthIntervalId); - this.getNitroProcesHealthIntervalId = undefined; - } +/** + * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + */ +async function loadLLMModel(settings: any): Promise { + log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`); + try { + const res = await fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(settings), + retries: 3, + retryDelay: 500, + }); + log( + `[NITRO]::Debug: Load model success with response ${JSON.stringify( + res + )}` + ); + return await Promise.resolve(res); + } catch (err) { + log(`[NITRO]::Error: Load model failed with error ${err}`); + return await Promise.reject(); } +} - /** - * Periodically check for nitro process's health. - */ - private async periodicallyGetNitroHealth(): Promise { - const health = await executeOnMain(NODE, "getCurrentNitroProcessInfo"); - - const isRunning = this.nitroProcessInfo?.isRunning ?? false; - if (isRunning && health.isRunning === false) { - console.debug("Nitro process is stopped"); - events.emit(ModelEvent.OnModelStopped, {}); +/** + * Validates the status of a model. + * @returns {Promise} A promise that resolves to an object. + * If the model is loaded successfully, the object is empty. + * If the model is not loaded successfully, the object contains an error message. + */ +async function validateModelStatus(): Promise { + // Send a GET request to the validation URL. + // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. + return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + retries: 5, + retryDelay: 500, + }).then(async (res: Response) => { + log( + `[NITRO]::Debug: Validate model state success with response ${JSON.stringify( + res + )}` + ); + // If the response is OK, check model_loaded status. + if (res.ok) { + const body = await res.json(); + // If the model is loaded, return an empty object. + // Otherwise, return an object with an error message. + if (body.model_loaded) { + return Promise.resolve({}); + } } - this.nitroProcessInfo = health; - } + return Promise.reject("Validate model status failed"); + }); +} - private async onInferenceStopped() { - this.isCancelled = true; - this.controller?.abort(); - } +/** + * Terminates the Nitro subprocess. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +async function killSubprocess(): Promise { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 5000); + log(`[NITRO]::Debug: Request to kill Nitro`); + + return fetch(NITRO_HTTP_KILL_URL, { + method: "DELETE", + signal: controller.signal, + }) + .then(() => { + subprocess?.kill(); + subprocess = undefined; + }) + .catch(() => { }) + .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) + .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)); +} - /** - * Makes a single response inference request. - * @param {MessageRequest} data - The data for the inference request. - * @returns {Promise} A promise that resolves with the inference response. - */ - async inference(data: MessageRequest): Promise { - const timestamp = Date.now(); - const message: ThreadMessage = { - thread_id: data.threadId, - created: timestamp, - updated: timestamp, - status: MessageStatus.Ready, - id: "", - role: ChatCompletionRole.Assistant, - object: "thread.message", - content: [], - }; +/** + * Spawns a Nitro subprocess. + * @returns A promise that resolves when the Nitro subprocess is started. + */ +function spawnNitroProcess(): Promise { + log(`[NITRO]::Debug: Spawning Nitro subprocess...`); - return new Promise(async (resolve, reject) => { - if (!this._currentModel) return Promise.reject("No model loaded"); + return new Promise(async (resolve, reject) => { + const binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default + const executableOptions = executableNitroFile(); - requestInference(data.messages ?? [], this._currentModel).subscribe({ - next: (_content) => {}, - complete: async () => { - resolve(message); - }, - error: async (err) => { - reject(err); + const args: string[] = ["1", LOCAL_HOST, PORT.toString()]; + // Execute the binary + log( + `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}` + ); + subprocess = spawn( + executableOptions.executablePath, + ["1", LOCAL_HOST, PORT.toString()], + { + cwd: binaryFolder, + env: { + ...process.env, + CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices, }, - }); + } + ); + + // Handle subprocess output + subprocess.stdout.on("data", (data: any) => { + log(`[NITRO]::Debug: ${data}`); }); - } - /** - * Handles a new message request by making an inference request and emitting events. - * Function registered in event manager, should be static to avoid binding issues. - * Pass instance as a reference. - * @param {MessageRequest} data - The data for the new message request. - */ - private async onMessageRequest(data: MessageRequest) { - if (data.model?.engine !== InferenceEngine.nitro || !this._currentModel) { - return; - } + subprocess.stderr.on("data", (data: any) => { + log(`[NITRO]::Error: ${data}`); + }); - const timestamp = Date.now(); - const message: ThreadMessage = { - id: ulid(), - thread_id: data.threadId, - assistant_id: data.assistantId, - role: ChatCompletionRole.Assistant, - content: [], - status: MessageStatus.Pending, - created: timestamp, - updated: timestamp, - object: "thread.message", - }; - events.emit(MessageEvent.OnMessageResponse, message); + subprocess.on("close", (code: any) => { + log(`[NITRO]::Debug: Nitro exited with code: ${code}`); + subprocess = undefined; + reject(`child process exited with code ${code}`); + }); - this.isCancelled = false; - this.controller = new AbortController(); + tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { + log(`[NITRO]::Debug: Nitro is ready`); + resolve(); + }); + }); +} - // @ts-ignore - const model: Model = { - ...(this._currentModel || {}), - ...(data.model || {}), +/** + * Get the system resources information + * TODO: Move to Core so that it can be reused + */ +function getResourcesInfo(): Promise { + return new Promise(async (resolve) => { + const cpu = osUtils.cpuCount(); + log(`[NITRO]::CPU informations - ${cpu}`); + const response: ResourcesInfo = { + numCpuPhysicalCore: cpu, + memAvailable: 0, }; - requestInference(data.messages ?? [], model, this.controller).subscribe({ - next: (content) => { - const messageContent: ThreadContent = { - type: ContentType.Text, - text: { - value: content.trim(), - annotations: [], - }, - }; - message.content = [messageContent]; - events.emit(MessageEvent.OnMessageUpdate, message); - }, - complete: async () => { - message.status = message.content.length - ? MessageStatus.Ready - : MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); - }, - error: async (err) => { - if (this.isCancelled || message.content.length) { - message.status = MessageStatus.Stopped; - events.emit(MessageEvent.OnMessageUpdate, message); - return; - } - message.status = MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); - log(`[APP]::Error: ${err.message}`); - }, - }); - } + resolve(response); + }); } + +/** + * Every module should have a dispose function + * This will be called when the extension is unloaded and should clean up any resources + * Also called when app is closed + */ +function dispose() { + // clean other registered resources here + killSubprocess(); +} + +export default { + setLogger, + runModel, + stopModel, + killSubprocess, + dispose, + updateNvidiaInfo, + getCurrentNitroProcessInfo: () => getNitroProcessInfo(subprocess), +}; diff --git a/nitro-node/src/module.ts b/nitro-node/src/module.ts deleted file mode 100644 index c80d9c39c..000000000 --- a/nitro-node/src/module.ts +++ /dev/null @@ -1,514 +0,0 @@ -const fs = require("fs"); -const path = require("path"); -const { exec, spawn } = require("child_process"); -const tcpPortUsed = require("tcp-port-used"); -const fetchRetry = require("fetch-retry")(global.fetch); -const osUtils = require("os-utils"); -const { readFileSync, writeFileSync, existsSync } = require("fs"); -const { log } = require("@janhq/core/node"); - -// The PORT to use for the Nitro subprocess -const PORT = 3928; -const LOCAL_HOST = "127.0.0.1"; -const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`; -const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`; -const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; -const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; -const SUPPORTED_MODEL_FORMAT = ".gguf"; -const NVIDIA_INFO_FILE = path.join( - require("os").homedir(), - "jan", - "settings", - "settings.json" -); - -// The subprocess instance for Nitro -let subprocess: any | undefined = undefined; -let currentModelFile: string = ''; -let currentSettings = undefined; - -let nitroProcessInfo: any | undefined = undefined; - -/** - * Default GPU settings - **/ -const DEFALT_SETTINGS = { - notify: true, - run_mode: "cpu", - nvidia_driver: { - exist: false, - version: "", - }, - cuda: { - exist: false, - version: "", - }, - gpus: [], - gpu_highest_vram: "", -}; - -/** - * Stops a Nitro subprocess. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -function stopModel(): Promise { - return killSubprocess(); -} - -/** - * Initializes a Nitro subprocess to load a machine learning model. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package - * TODO: Should it be startModel instead? - */ -async function initModel(wrapper: any): Promise { - currentModelFile = wrapper.modelFullPath; - const janRoot = path.join(require("os").homedir(), "jan"); - if (!currentModelFile.includes(janRoot)) { - currentModelFile = path.join(janRoot, currentModelFile); - } - const files: string[] = fs.readdirSync(currentModelFile); - - // Look for GGUF model file - const ggufBinFile = files.find( - (file) => - file === path.basename(currentModelFile) || - file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT) - ); - - currentModelFile = path.join(currentModelFile, ggufBinFile); - - if (wrapper.model.engine !== "nitro") { - return Promise.resolve({ error: "Not a nitro model" }); - } else { - const nitroResourceProbe = await getResourcesInfo(); - // Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt - if (wrapper.model.settings.prompt_template) { - const promptTemplate = wrapper.model.settings.prompt_template; - const prompt = promptTemplateConverter(promptTemplate); - if (prompt.error) { - return Promise.resolve({ error: prompt.error }); - } - wrapper.model.settings.system_prompt = prompt.system_prompt; - wrapper.model.settings.user_prompt = prompt.user_prompt; - wrapper.model.settings.ai_prompt = prompt.ai_prompt; - } - - currentSettings = { - llama_model_path: currentModelFile, - ...wrapper.model.settings, - // This is critical and requires real system information - cpu_threads: nitroResourceProbe.numCpuPhysicalCore, - }; - return loadModel(nitroResourceProbe); - } -} - -async function loadModel(nitroResourceProbe: any | undefined) { - // Gather system information for CPU physical cores and memory - if (!nitroResourceProbe) nitroResourceProbe = await getResourcesInfo(); - return killSubprocess() - .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => { - /** - * There is a problem with Windows process manager - * Should wait for awhile to make sure the port is free and subprocess is killed - * The tested threshold is 500ms - **/ - if (process.platform === "win32") { - return new Promise((resolve) => setTimeout(resolve, 500)); - } else { - return Promise.resolve(); - } - }) - .then(() => spawnNitroProcess(nitroResourceProbe)) - .then(() => loadLLMModel(currentSettings)) - .then(validateModelStatus) - .catch((err) => { - log(`[NITRO]::Error: ${err}`); - // TODO: Broadcast error so app could display proper error message - return { error: err, currentModelFile }; - }); -} - -function promptTemplateConverter(promptTemplate) { - // Split the string using the markers - const systemMarker = "{system_message}"; - const promptMarker = "{prompt}"; - - if ( - promptTemplate.includes(systemMarker) && - promptTemplate.includes(promptMarker) - ) { - // Find the indices of the markers - const systemIndex = promptTemplate.indexOf(systemMarker); - const promptIndex = promptTemplate.indexOf(promptMarker); - - // Extract the parts of the string - const system_prompt = promptTemplate.substring(0, systemIndex); - const user_prompt = promptTemplate.substring( - systemIndex + systemMarker.length, - promptIndex - ); - const ai_prompt = promptTemplate.substring( - promptIndex + promptMarker.length - ); - - // Return the split parts - return { system_prompt, user_prompt, ai_prompt }; - } else if (promptTemplate.includes(promptMarker)) { - // Extract the parts of the string for the case where only promptMarker is present - const promptIndex = promptTemplate.indexOf(promptMarker); - const user_prompt = promptTemplate.substring(0, promptIndex); - const ai_prompt = promptTemplate.substring( - promptIndex + promptMarker.length - ); - const system_prompt = ""; - - // Return the split parts - return { system_prompt, user_prompt, ai_prompt }; - } - - // Return an error if none of the conditions are met - return { error: "Cannot split prompt template" }; -} - -/** - * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - */ -function loadLLMModel(settings): Promise { - log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`); - return fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(settings), - retries: 3, - retryDelay: 500, - }).catch((err) => { - log(`[NITRO]::Error: Load model failed with error ${err}`); - }); -} - -/** - * Validates the status of a model. - * @returns {Promise} A promise that resolves to an object. - * If the model is loaded successfully, the object is empty. - * If the model is not loaded successfully, the object contains an error message. - */ -async function validateModelStatus(): Promise { - // Send a GET request to the validation URL. - // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. - return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { - method: "GET", - headers: { - "Content-Type": "application/json", - }, - retries: 5, - retryDelay: 500, - }).then(async (res: Response) => { - // If the response is OK, check model_loaded status. - if (res.ok) { - const body = await res.json(); - // If the model is loaded, return an empty object. - // Otherwise, return an object with an error message. - if (body.model_loaded) { - return { error: undefined }; - } - } - return { error: "Model loading failed" }; - }); -} - -/** - * Terminates the Nitro subprocess. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -async function killSubprocess(): Promise { - const controller = new AbortController(); - setTimeout(() => controller.abort(), 5000); - log(`[NITRO]::Debug: Request to kill Nitro`); - - return fetch(NITRO_HTTP_KILL_URL, { - method: "DELETE", - signal: controller.signal, - }) - .then(() => { - subprocess?.kill(); - subprocess = undefined; - }) - .catch(() => { }) - .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)); -} - -/** - * Spawns a Nitro subprocess. - * @param nitroResourceProbe - The Nitro resource probe. - * @returns A promise that resolves when the Nitro subprocess is started. - */ -function spawnNitroProcess(nitroResourceProbe: any): Promise { - log(`[NITRO]::Debug: Spawning Nitro subprocess...`); - - return new Promise(async (resolve, reject) => { - let binaryFolder = path.join(__dirname, "bin"); // Current directory by default - let cudaVisibleDevices = ""; - let binaryName; - if (process.platform === "win32") { - let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvidiaInfo["run_mode"] === "cpu") { - binaryFolder = path.join(binaryFolder, "win-cpu"); - } else { - if (nvidiaInfo["cuda"].version === "12") { - binaryFolder = path.join(binaryFolder, "win-cuda-12-0"); - } else { - binaryFolder = path.join(binaryFolder, "win-cuda-11-7"); - } - cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; - } - binaryName = "nitro.exe"; - } else if (process.platform === "darwin") { - if (process.arch === "arm64") { - binaryFolder = path.join(binaryFolder, "mac-arm64"); - } else { - binaryFolder = path.join(binaryFolder, "mac-x64"); - } - binaryName = "nitro"; - } else { - let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvidiaInfo["run_mode"] === "cpu") { - binaryFolder = path.join(binaryFolder, "linux-cpu"); - } else { - if (nvidiaInfo["cuda"].version === "12") { - binaryFolder = path.join(binaryFolder, "linux-cuda-12-0"); - } else { - binaryFolder = path.join(binaryFolder, "linux-cuda-11-7"); - } - cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; - } - binaryName = "nitro"; - } - - const binaryPath = path.join(binaryFolder, binaryName); - // Execute the binary - subprocess = spawn(binaryPath, ["1", LOCAL_HOST, PORT.toString()], { - cwd: binaryFolder, - env: { - ...process.env, - CUDA_VISIBLE_DEVICES: cudaVisibleDevices, - }, - }); - - // Handle subprocess output - subprocess.stdout.on("data", (data) => { - log(`[NITRO]::Debug: ${data}`); - }); - - subprocess.stderr.on("data", (data) => { - log(`[NITRO]::Error: ${data}`); - }); - - subprocess.on("close", (code) => { - log(`[NITRO]::Debug: Nitro exited with code: ${code}`); - subprocess = null; - reject(`child process exited with code ${code}`); - }); - - tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { - resolve(nitroResourceProbe); - }); - }); -} - -/** - * Get the system resources information - * TODO: Move to Core so that it can be reused - */ -function getResourcesInfo(): Promise { - return new Promise(async (resolve) => { - const cpu = await osUtils.cpuCount(); - log(`[NITRO]::CPU informations - ${cpu}`); - const response: ResourcesInfo = { - numCpuPhysicalCore: cpu, - memAvailable: 0, - }; - resolve(response); - }); -} - -/** - * This will retrive GPU informations and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported - */ -async function updateNvidiaInfo() { - if (process.platform !== "darwin") { - await Promise.all([ - updateNvidiaDriverInfo(), - updateCudaExistence(), - updateGpuInfo(), - ]); - } -} - -/** - * Retrieve current nitro process - */ -const getCurrentNitroProcessInfo = (): Promise => { - nitroProcessInfo = { - isRunning: subprocess != null, - }; - return nitroProcessInfo; -}; - -/** - * Every module should have a dispose function - * This will be called when the extension is unloaded and should clean up any resources - * Also called when app is closed - */ -function dispose() { - // clean other registered resources here - killSubprocess(); -} - -/** - * Validate nvidia and cuda for linux and windows - */ -async function updateNvidiaDriverInfo(): Promise { - exec( - "nvidia-smi --query-gpu=driver_version --format=csv,noheader", - (error, stdout) => { - let data; - try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - } catch (error) { - data = DEFALT_SETTINGS; - } - - if (!error) { - const firstLine = stdout.split("\n")[0].trim(); - data["nvidia_driver"].exist = true; - data["nvidia_driver"].version = firstLine; - } else { - data["nvidia_driver"].exist = false; - } - - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - Promise.resolve(); - } - ); -} - -/** - * Check if file exists in paths - */ -function checkFileExistenceInPaths(file: string, paths: string[]): boolean { - return paths.some((p) => existsSync(path.join(p, file))); -} - -/** - * Validate cuda for linux and windows - */ -function updateCudaExistence() { - let filesCuda12: string[]; - let filesCuda11: string[]; - let paths: string[]; - let cudaVersion: string = ""; - - if (process.platform === "win32") { - filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; - filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; - paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : []; - } else { - filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; - filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; - paths = process.env.LD_LIBRARY_PATH - ? process.env.LD_LIBRARY_PATH.split(path.delimiter) - : []; - paths.push("/usr/lib/x86_64-linux-gnu/"); - } - - let cudaExists = filesCuda12.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ); - - if (!cudaExists) { - cudaExists = filesCuda11.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ); - if (cudaExists) { - cudaVersion = "11"; - } - } else { - cudaVersion = "12"; - } - - let data; - try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - } catch (error) { - data = DEFALT_SETTINGS; - } - - data["cuda"].exist = cudaExists; - data["cuda"].version = cudaVersion; - if (cudaExists) { - data.run_mode = "gpu"; - } - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); -} - -/** - * Get GPU information - */ -async function updateGpuInfo(): Promise { - exec( - "nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", - (error, stdout) => { - let data; - try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - } catch (error) { - data = DEFALT_SETTINGS; - } - - if (!error) { - // Get GPU info and gpu has higher memory first - let highestVram = 0; - let highestVramId = "0"; - let gpus = stdout - .trim() - .split("\n") - .map((line) => { - let [id, vram] = line.split(", "); - vram = vram.replace(/\r/g, ""); - if (parseFloat(vram) > highestVram) { - highestVram = parseFloat(vram); - highestVramId = id; - } - return { id, vram }; - }); - - data["gpus"] = gpus; - data["gpu_highest_vram"] = highestVramId; - } else { - data["gpus"] = []; - } - - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - Promise.resolve(); - } - ); -} - -module.exports = { - initModel, - stopModel, - killSubprocess, - dispose, - updateNvidiaInfo, - getCurrentNitroProcessInfo, -}; diff --git a/nitro-node/src/node/index.ts b/nitro-node/src/node/index.ts deleted file mode 100644 index 765b2240f..000000000 --- a/nitro-node/src/node/index.ts +++ /dev/null @@ -1,379 +0,0 @@ -import fs from "fs"; -import path from "path"; -import { ChildProcessWithoutNullStreams, spawn } from "child_process"; -import tcpPortUsed from "tcp-port-used"; -import fetchRT from "fetch-retry"; -import osUtils from "os-utils"; -import { log } from "@janhq/core/node"; -import { getNitroProcessInfo, updateNvidiaInfo } from "./nvidia"; -import { Model, InferenceEngine, ModelSettingParams } from "@janhq/core"; -import { executableNitroFile } from "./execute"; -import { homedir } from "os"; -// Polyfill fetch with retry -const fetchRetry = fetchRT(fetch); - -/** - * The response object for model init operation. - */ -interface ModelInitOptions { - modelFullPath: string; - model: Model; -} - -/** - * The response object of Prompt Template parsing. - */ -interface PromptTemplate { - system_prompt?: string; - ai_prompt?: string; - user_prompt?: string; - error?: string; -} - -/** - * Model setting args for Nitro model load. - */ -interface ModelSettingArgs extends ModelSettingParams { - llama_model_path: string; - cpu_threads: number; -} - -// The PORT to use for the Nitro subprocess -const PORT = 3928; -// The HOST address to use for the Nitro subprocess -const LOCAL_HOST = "127.0.0.1"; -// The URL for the Nitro subprocess -const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`; -// The URL for the Nitro subprocess to load a model -const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`; -// The URL for the Nitro subprocess to validate a model -const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; -// The URL for the Nitro subprocess to kill itself -const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; - -// The supported model format -// TODO: Should be an array to support more models -const SUPPORTED_MODEL_FORMAT = ".gguf"; - -// The subprocess instance for Nitro -let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; -// The current model file url -let currentModelFile: string = ""; -// The current model settings -let currentSettings: ModelSettingArgs | undefined = undefined; - -/** - * Stops a Nitro subprocess. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -function stopModel(): Promise { - return killSubprocess(); -} - -/** - * Initializes a Nitro subprocess to load a machine learning model. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package - */ -async function runModel( - wrapper: ModelInitOptions -): Promise { - if (wrapper.model.engine !== InferenceEngine.nitro) { - // Not a nitro model - return Promise.resolve(); - } - - currentModelFile = wrapper.modelFullPath; - const janRoot = path.join(homedir(), "jan"); - if (!currentModelFile.includes(janRoot)) { - currentModelFile = path.join(janRoot, currentModelFile); - } - const files: string[] = fs.readdirSync(currentModelFile); - - // Look for GGUF model file - const ggufBinFile = files.find( - (file) => - file === path.basename(currentModelFile) || - file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT) - ); - - if (!ggufBinFile) return Promise.reject("No GGUF model file found"); - - currentModelFile = path.join(currentModelFile, ggufBinFile); - - if (wrapper.model.engine !== InferenceEngine.nitro) { - return Promise.reject("Not a nitro model"); - } else { - const nitroResourceProbe = await getResourcesInfo(); - // Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt - if (wrapper.model.settings.prompt_template) { - const promptTemplate = wrapper.model.settings.prompt_template; - const prompt = promptTemplateConverter(promptTemplate); - if (prompt?.error) { - return Promise.reject(prompt.error); - } - wrapper.model.settings.system_prompt = prompt.system_prompt; - wrapper.model.settings.user_prompt = prompt.user_prompt; - wrapper.model.settings.ai_prompt = prompt.ai_prompt; - } - - currentSettings = { - llama_model_path: currentModelFile, - ...wrapper.model.settings, - // This is critical and requires real system information - cpu_threads: Math.max(1, Math.round(nitroResourceProbe.numCpuPhysicalCore / 2)), - }; - return runNitroAndLoadModel(); - } -} - -/** - * 1. Spawn Nitro process - * 2. Load model into Nitro subprocess - * 3. Validate model status - * @returns - */ -async function runNitroAndLoadModel() { - // Gather system information for CPU physical cores and memory - return killSubprocess() - .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => { - /** - * There is a problem with Windows process manager - * Should wait for awhile to make sure the port is free and subprocess is killed - * The tested threshold is 500ms - **/ - if (process.platform === "win32") { - return new Promise((resolve) => setTimeout(resolve, 500)); - } else { - return Promise.resolve(); - } - }) - .then(spawnNitroProcess) - .then(() => loadLLMModel(currentSettings)) - .then(validateModelStatus) - .catch((err) => { - // TODO: Broadcast error so app could display proper error message - log(`[NITRO]::Error: ${err}`); - return { error: err }; - }); -} - -/** - * Parse prompt template into agrs settings - * @param promptTemplate Template as string - * @returns - */ -function promptTemplateConverter(promptTemplate: string): PromptTemplate { - // Split the string using the markers - const systemMarker = "{system_message}"; - const promptMarker = "{prompt}"; - - if ( - promptTemplate.includes(systemMarker) && - promptTemplate.includes(promptMarker) - ) { - // Find the indices of the markers - const systemIndex = promptTemplate.indexOf(systemMarker); - const promptIndex = promptTemplate.indexOf(promptMarker); - - // Extract the parts of the string - const system_prompt = promptTemplate.substring(0, systemIndex); - const user_prompt = promptTemplate.substring( - systemIndex + systemMarker.length, - promptIndex - ); - const ai_prompt = promptTemplate.substring( - promptIndex + promptMarker.length - ); - - // Return the split parts - return { system_prompt, user_prompt, ai_prompt }; - } else if (promptTemplate.includes(promptMarker)) { - // Extract the parts of the string for the case where only promptMarker is present - const promptIndex = promptTemplate.indexOf(promptMarker); - const user_prompt = promptTemplate.substring(0, promptIndex); - const ai_prompt = promptTemplate.substring( - promptIndex + promptMarker.length - ); - - // Return the split parts - return { user_prompt, ai_prompt }; - } - - // Return an error if none of the conditions are met - return { error: "Cannot split prompt template" }; -} - -/** - * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - */ -function loadLLMModel(settings: any): Promise { - log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`); - return fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(settings), - retries: 3, - retryDelay: 500, - }) - .then((res) => { - log( - `[NITRO]::Debug: Load model success with response ${JSON.stringify( - res - )}` - ); - return Promise.resolve(res); - }) - .catch((err) => { - log(`[NITRO]::Error: Load model failed with error ${err}`); - return Promise.reject(); - }); -} - -/** - * Validates the status of a model. - * @returns {Promise} A promise that resolves to an object. - * If the model is loaded successfully, the object is empty. - * If the model is not loaded successfully, the object contains an error message. - */ -async function validateModelStatus(): Promise { - // Send a GET request to the validation URL. - // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. - return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { - method: "GET", - headers: { - "Content-Type": "application/json", - }, - retries: 5, - retryDelay: 500, - }).then(async (res: Response) => { - log( - `[NITRO]::Debug: Validate model state success with response ${JSON.stringify( - res - )}` - ); - // If the response is OK, check model_loaded status. - if (res.ok) { - const body = await res.json(); - // If the model is loaded, return an empty object. - // Otherwise, return an object with an error message. - if (body.model_loaded) { - return Promise.resolve(); - } - } - return Promise.reject("Validate model status failed"); - }); -} - -/** - * Terminates the Nitro subprocess. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -async function killSubprocess(): Promise { - const controller = new AbortController(); - setTimeout(() => controller.abort(), 5000); - log(`[NITRO]::Debug: Request to kill Nitro`); - - return fetch(NITRO_HTTP_KILL_URL, { - method: "DELETE", - signal: controller.signal, - }) - .then(() => { - subprocess?.kill(); - subprocess = undefined; - }) - .catch(() => {}) - .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)); -} - -/** - * Spawns a Nitro subprocess. - * @returns A promise that resolves when the Nitro subprocess is started. - */ -function spawnNitroProcess(): Promise { - log(`[NITRO]::Debug: Spawning Nitro subprocess...`); - - return new Promise(async (resolve, reject) => { - let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default - let executableOptions = executableNitroFile(); - - const args: string[] = ["1", LOCAL_HOST, PORT.toString()]; - // Execute the binary - log( - `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}` - ); - subprocess = spawn( - executableOptions.executablePath, - ["1", LOCAL_HOST, PORT.toString()], - { - cwd: binaryFolder, - env: { - ...process.env, - CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices, - }, - } - ); - - // Handle subprocess output - subprocess.stdout.on("data", (data: any) => { - log(`[NITRO]::Debug: ${data}`); - }); - - subprocess.stderr.on("data", (data: any) => { - log(`[NITRO]::Error: ${data}`); - }); - - subprocess.on("close", (code: any) => { - log(`[NITRO]::Debug: Nitro exited with code: ${code}`); - subprocess = undefined; - reject(`child process exited with code ${code}`); - }); - - tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { - log(`[NITRO]::Debug: Nitro is ready`); - resolve(); - }); - }); -} - -/** - * Get the system resources information - * TODO: Move to Core so that it can be reused - */ -function getResourcesInfo(): Promise { - return new Promise(async (resolve) => { - const cpu = await osUtils.cpuCount(); - log(`[NITRO]::CPU informations - ${cpu}`); - const response: ResourcesInfo = { - numCpuPhysicalCore: cpu, - memAvailable: 0, - }; - resolve(response); - }); -} - -/** - * Every module should have a dispose function - * This will be called when the extension is unloaded and should clean up any resources - * Also called when app is closed - */ -function dispose() { - // clean other registered resources here - killSubprocess(); -} - -export default { - runModel, - stopModel, - killSubprocess, - dispose, - updateNvidiaInfo, - getCurrentNitroProcessInfo: () => getNitroProcessInfo(subprocess), -}; diff --git a/nitro-node/src/node/nvidia.ts b/nitro-node/src/nvidia.ts similarity index 100% rename from nitro-node/src/node/nvidia.ts rename to nitro-node/src/nvidia.ts diff --git a/nitro-node/tsconfig.json b/nitro-node/tsconfig.json index a2cee893d..8ad3ae0ea 100644 --- a/nitro-node/tsconfig.json +++ b/nitro-node/tsconfig.json @@ -6,11 +6,9 @@ "lib": [ "es2015", "es2016", - "es2017", - "dom" + "es2017" ], "strict": true, - "noImplicitAny": false, // FIXME: some code lines still use wrong types or no explicit types "sourceMap": true, "declaration": true, "allowSyntheticDefaultImports": true, From 6f625d94fa3e35f1dc605a573afc4d9f987df19e Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Wed, 24 Jan 2024 16:41:53 +0700 Subject: [PATCH 08/49] fix(postinstall): download path for nitro binary The path should always be relative to this library's directory --- nitro-node/download-nitro.js | 16 ++++++++-------- nitro-node/package.json | 11 +++-------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/nitro-node/download-nitro.js b/nitro-node/download-nitro.js index feb54251f..968896d6f 100644 --- a/nitro-node/download-nitro.js +++ b/nitro-node/download-nitro.js @@ -10,20 +10,20 @@ const PLATFORM = process.env.npm_config_platform || process.platform; //const ARCH = process.env.npm_config_arch || process.arch; const linuxVariants = { - 'linux-amd64': path.join('.', 'bin', 'linux-cpu'), - 'linux-amd64-cuda-12-0': path.join('.', 'bin', 'linux-cuda-12-0'), - 'linux-amd64-cuda-11-7': path.join('.', 'bin', 'linux-cuda-11-7'), + 'linux-amd64': path.join(__dirname, 'bin', 'linux-cpu'), + 'linux-amd64-cuda-12-0': path.join(__dirname, 'bin', 'linux-cuda-12-0'), + 'linux-amd64-cuda-11-7': path.join(__dirname, 'bin', 'linux-cuda-11-7'), } const darwinVariants = { - 'mac-arm64': path.join('.', 'bin', 'mac-arm64'), - 'mac-amd64': path.join('.', 'bin', 'mac-x64'), + 'mac-arm64': path.join(__dirname, 'bin', 'mac-arm64'), + 'mac-amd64': path.join(__dirname, 'bin', 'mac-x64'), } const win32Variants = { - 'win-amd64-cuda-12-0': path.join('.', 'bin', 'win-cuda-12-0'), - 'win-amd64-cuda-11-7': path.join('.', 'bin', 'win-cuda-11-7'), - 'win-amd64': path.join('.', 'bin', 'win-cpu'), + 'win-amd64-cuda-12-0': path.join(__dirname, 'bin', 'win-cuda-12-0'), + 'win-amd64-cuda-11-7': path.join(__dirname, 'bin', 'win-cuda-11-7'), + 'win-amd64': path.join(__dirname, 'bin', 'win-cpu'), } // Mapping to installation variants diff --git a/nitro-node/package.json b/nitro-node/package.json index 702a790bf..8d63bc529 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -13,8 +13,7 @@ "build:publish:win32": "npm pack", "build:publish:linux": "npm pack", "build:publish": "run-script-os", - "postinstall": "node postinstall.js", - "prepack": "cpx \"bin/**\" \"dist/bin\"" + "postinstall": "node postinstall.js" }, "exports": { ".": "./dist/index.js", @@ -35,14 +34,11 @@ "typescript": "^5.3.3" }, "dependencies": { - "cpx": "^1.5.0", - "download": "^8.0.0", - "@janhq/core": "^0.1.11", "@rollup/plugin-replace": "^5.0.5", "@types/os-utils": "^0.0.4", + "download": "^8.0.0", "fetch-retry": "^5.0.6", "os-utils": "^0.0.14", - "path-browserify": "^1.0.1", "rxjs": "^7.8.1", "tcp-port-used": "^1.0.2", "ulid": "^2.3.0" @@ -60,7 +56,6 @@ "bundleDependencies": [ "tcp-port-used", "fetch-retry", - "os-utils", - "@janhq/core" + "os-utils" ] } From 8c742cf32a0d09e29657ccb96458e0adf60ec2a9 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Wed, 24 Jan 2024 16:54:14 +0700 Subject: [PATCH 09/49] fix(downloadnitro): log messages for downloading nitro variants --- nitro-node/download-nitro.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nitro-node/download-nitro.js b/nitro-node/download-nitro.js index 968896d6f..432195c5f 100644 --- a/nitro-node/download-nitro.js +++ b/nitro-node/download-nitro.js @@ -1,6 +1,5 @@ const path = require('path'); const download = require('download'); -const stream = require('stream'); // Define nitro version to download in env variable const NITRO_VERSION = process.env.NITRO_VERSION || '0.2.11'; @@ -46,8 +45,11 @@ const getTarUrl = (version, suffix) => `https://github.com/janhq/nitro/releases/ const createProgressReporter = (variant) => (stream) => stream.on( 'downloadProgress', (progress) => { + // Print and update progress on a single line of terminal process.stdout.write(`\r[${variant}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`); - }).on('finish', () => { + }).on('end', () => { + // Jump to new line to log next message + console.log(); console.log(`[${variant}] Finished downloading!`); }) @@ -65,11 +67,9 @@ const downloadBinary = (version, suffix, filePath) => { } // Download the binaries -const downloadBinaries = async (version, config) => { - await Object.entries(config).reduce( - async (p, [k, v]) => { - p.then(() => downloadBinary(version, k, v)); - }, +const downloadBinaries = (version, config) => { + return Object.entries(config).reduce( + (p, [k, v]) => p.then(() => downloadBinary(version, k, v)), Promise.resolve(), ); } From 5f462203975c3bac7ffe189a859928775f184a92 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Wed, 24 Jan 2024 17:27:12 +0700 Subject: [PATCH 10/49] chore(nitro-node): cleanup dependencies --- nitro-node/package.json | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/nitro-node/package.json b/nitro-node/package.json index 8d63bc529..9fe633753 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -9,10 +9,7 @@ "scripts": { "build": "tsc --module commonjs && rollup -c rollup.config.ts", "downloadnitro": "node download-nitro.js", - "build:publish:darwin": "../.github/scripts/auto-sign.sh && npm pack", - "build:publish:win32": "npm pack", - "build:publish:linux": "npm pack", - "build:publish": "run-script-os", + "build:publish": "npm pack", "postinstall": "node postinstall.js" }, "exports": { @@ -23,25 +20,23 @@ "@rollup/plugin-commonjs": "^25.0.7", "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-replace": "^5.0.5", "@types/node": "^20.11.4", + "@types/os-utils": "^0.0.4", "@types/tcp-port-used": "^1.0.4", + "jest": "^29.7.0", "rimraf": "^3.0.2", "rollup": "^2.38.5", "rollup-plugin-define": "^1.0.1", "rollup-plugin-sourcemaps": "^0.6.3", "rollup-plugin-typescript2": "^0.36.0", - "run-script-os": "^1.1.6", "typescript": "^5.3.3" }, "dependencies": { - "@rollup/plugin-replace": "^5.0.5", - "@types/os-utils": "^0.0.4", "download": "^8.0.0", "fetch-retry": "^5.0.6", "os-utils": "^0.0.14", - "rxjs": "^7.8.1", - "tcp-port-used": "^1.0.2", - "ulid": "^2.3.0" + "tcp-port-used": "^1.0.2" }, "engines": { "node": ">=18.0.0" From 9e2dd61d1fd9a1ef6a36d75b27385641c9ce0857 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Wed, 24 Jan 2024 21:35:09 +0700 Subject: [PATCH 11/49] test(nitro-node): First test case for nitro-node - Use jest to test - Using model `tinyllama-1.1b` as test model - Test start/stop nitro via nitro-node wrapper --- nitro-node/download-nitro.js | 1 + nitro-node/jest.config.ts | 9 ++ nitro-node/package.json | 5 ++ nitro-node/src/@types/global.d.ts | 3 +- nitro-node/src/execute.ts | 4 +- nitro-node/src/index.ts | 46 ++++------ nitro-node/src/nvidia.ts | 8 +- nitro-node/test/model.json | 28 ++++++ nitro-node/test/nitro-process.test.ts | 124 ++++++++++++++++++++++++++ nitro-node/tsconfig.json | 1 + 10 files changed, 194 insertions(+), 35 deletions(-) create mode 100644 nitro-node/jest.config.ts create mode 100644 nitro-node/test/model.json create mode 100644 nitro-node/test/nitro-process.test.ts diff --git a/nitro-node/download-nitro.js b/nitro-node/download-nitro.js index 432195c5f..dd4cb8e5b 100644 --- a/nitro-node/download-nitro.js +++ b/nitro-node/download-nitro.js @@ -80,6 +80,7 @@ const run = () => { } module.exports = run; +module.exports.createProgressReporter = createProgressReporter // Run script if called directly instead of import as module if (require.main === module) { diff --git a/nitro-node/jest.config.ts b/nitro-node/jest.config.ts new file mode 100644 index 000000000..9dbaf2efa --- /dev/null +++ b/nitro-node/jest.config.ts @@ -0,0 +1,9 @@ +import type { JestConfigWithTsJest } from 'ts-jest' + +const jestConfig: JestConfigWithTsJest = { + preset: 'ts-jest', + testEnvironment: 'node', + transformIgnorePatterns: ['/node_modules/'] +} + +export default jestConfig diff --git a/nitro-node/package.json b/nitro-node/package.json index 9fe633753..3cfbd795f 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -7,6 +7,8 @@ "author": "Jan ", "license": "AGPL-3.0", "scripts": { + "test": "jest --verbose --detectOpenHandles", + "pretest": "node download-nitro.js", "build": "tsc --module commonjs && rollup -c rollup.config.ts", "downloadnitro": "node download-nitro.js", "build:publish": "npm pack", @@ -21,6 +23,8 @@ "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.2.3", "@rollup/plugin-replace": "^5.0.5", + "@types/download": "^8.0.5", + "@types/jest": "^29.5.11", "@types/node": "^20.11.4", "@types/os-utils": "^0.0.4", "@types/tcp-port-used": "^1.0.4", @@ -30,6 +34,7 @@ "rollup-plugin-define": "^1.0.1", "rollup-plugin-sourcemaps": "^0.6.3", "rollup-plugin-typescript2": "^0.36.0", + "ts-jest": "^29.1.2", "typescript": "^5.3.3" }, "dependencies": { diff --git a/nitro-node/src/@types/global.d.ts b/nitro-node/src/@types/global.d.ts index bc49ddc6b..3be794d55 100644 --- a/nitro-node/src/@types/global.d.ts +++ b/nitro-node/src/@types/global.d.ts @@ -16,7 +16,6 @@ interface ResourcesInfo { * Setting for prompts when inferencing with Nitro */ interface NitroPromptSetting { - prompt_template?: string; system_prompt?: string; ai_prompt?: string; user_prompt?: string; @@ -35,7 +34,7 @@ interface NitroModelSetting extends NitroPromptSetting { */ interface NitroModelInitOptions { modelFullPath: string; - settings: NitroPromptSetting; + promptTemplate?: string; } /** diff --git a/nitro-node/src/execute.ts b/nitro-node/src/execute.ts index 6eb11b36d..c190c678f 100644 --- a/nitro-node/src/execute.ts +++ b/nitro-node/src/execute.ts @@ -1,5 +1,5 @@ -import { readFileSync } from "fs"; -import * as path from "path"; +import { readFileSync } from "node:fs"; +import path from "node:path"; import { NVIDIA_INFO_FILE } from "./nvidia"; diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index cd90177e4..33d73fbf4 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -1,6 +1,6 @@ -import fs from "fs"; -import path from "path"; -import { ChildProcessWithoutNullStreams, spawn } from "child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; import tcpPortUsed from "tcp-port-used"; import fetchRT from "fetch-retry"; import osUtils from "os-utils"; @@ -9,13 +9,6 @@ import { executableNitroFile } from "./execute"; // Polyfill fetch with retry const fetchRetry = fetchRT(fetch); -/** - * The response object of Prompt Template parsing. - */ -type PromptTemplate = Omit & { - error?: string; -} - // The PORT to use for the Nitro subprocess const PORT = 3928; // The HOST address to use for the Nitro subprocess @@ -68,9 +61,9 @@ function stopModel(): Promise { async function runModel( { modelFullPath, - settings, + promptTemplate, }: NitroModelInitOptions -): Promise { +): Promise { const files: string[] = fs.readdirSync(modelFullPath); // Look for GGUF model file @@ -85,21 +78,19 @@ async function runModel( currentModelFile = path.join(modelFullPath, ggufBinFile); const nitroResourceProbe = await getResourcesInfo(); - // Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt - if (settings.prompt_template) { - const promptTemplate = settings.prompt_template; - const prompt = promptTemplateConverter(promptTemplate); - if (prompt?.error) { - return Promise.reject(prompt.error); + // Convert promptTemplate to system_prompt, user_prompt, ai_prompt + const prompt: NitroPromptSetting = {}; + if (promptTemplate) { + try { + Object.assign(prompt, promptTemplateConverter(promptTemplate)); + } catch (e: any) { + return Promise.reject(e); } - settings.system_prompt = prompt.system_prompt; - settings.user_prompt = prompt.user_prompt; - settings.ai_prompt = prompt.ai_prompt; } currentSettings = { + ...prompt, llama_model_path: currentModelFile, - ...settings, // This is critical and requires real system information cpu_threads: Math.max(1, Math.round(nitroResourceProbe.numCpuPhysicalCore / 2)), }; @@ -140,10 +131,11 @@ async function runNitroAndLoadModel(): Promise\n{system_message}<|user|>\n{prompt}<|assistant|>" + }, + "parameters": { + "temperature": 0.7, + "top_p": 0.95, + "stream": true, + "max_tokens": 2048, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "TinyLlama", + "tags": ["Tiny", "Foundation Model"], + "size": 669000000 + }, + "engine": "nitro" +} \ No newline at end of file diff --git a/nitro-node/test/nitro-process.test.ts b/nitro-node/test/nitro-process.test.ts new file mode 100644 index 000000000..27e407f5e --- /dev/null +++ b/nitro-node/test/nitro-process.test.ts @@ -0,0 +1,124 @@ +import { jest, describe, test } from '@jest/globals' + +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +import download from 'download' + +import { default as nitro } from '../src/index' +import { Duplex } from 'node:stream' +const { runModel, stopModel } = nitro + +// FIXME: Shorthand only possible for es6 targets and up +//import * as model from './model.json' assert {type: 'json'} + +// Get model config +const getModelConfigHook = (callback: (modelCfg: any) => void) => () => { + const modelJson = fs.readFileSync(path.join(__dirname, 'model.json'), { + encoding: 'utf8', + }) + const modelCfg = JSON.parse(modelJson) + callback(modelCfg) +} + +// Report download progress +const createProgressReporter = (name: string) => (stream: Promise & Duplex) => stream.on( + 'downloadProgress', + (progress: { transferred: any; total: any; percent: number }) => { + // Print and update progress on a single line of terminal + process.stdout.write(`\r[${name}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`); + }).on('end', () => { + // Jump to new line to log next message + process.stdout.write(`\n[${name}] Finished downloading!`); + }) + +// Download model file +const downloadModelHook = (modelCfg: any, targetDir: string) => async () => { + const fileName = modelCfg.source_url.split('/')?.pop() ?? 'model.gguf' + const progressReporter = createProgressReporter(modelCfg.name) + await progressReporter( + download( + modelCfg.source_url, + targetDir, + { + filename: fileName, + strip: 1, + extract: true, + }, + ) + ) + console.log(`Downloaded model ${modelCfg.name} at path ${path.join(targetDir, fileName)}`) +} + +// Cleanup tmp directory that is used during tests +const cleanupTargetDirHook = (targetDir: string) => () => { + fs.rmSync(targetDir, { + recursive: true, // Remove whole directory + maxRetries: 3, // Retry 3 times on error + retryDelay: 250, // Back-off with 250ms delay + }) +} + +/** + * Sleep for the specified milliseconds + * @param {number} ms milliseconds to sleep for + * @returns {Promise} + */ +const sleep = async (ms: number): Promise => Promise.resolve().then(() => setTimeout(() => void (0), ms)) + +/** + * Basic test suite + */ +describe('Manage nitro process', () => { + /// BEGIN SUITE CONFIG + const modelFullPath = fs.mkdtempSync(path.join(os.tmpdir(), 'nitro-node-test')); + let modelCfg: any = {} + + // Setup steps before running the suite + const setupHooks = [ + // Get model config from json + getModelConfigHook((cfg) => Object.assign(modelCfg, cfg)), + // Download model before starting tests + downloadModelHook(modelCfg, modelFullPath), + ] + // Teardown steps after running the suite + const teardownHooks = [ + // On teardown, cleanup tmp directory that was created earlier + cleanupTargetDirHook(modelFullPath), + ] + /// END SUITE CONFIG + + /// BEGIN HOOKS REGISTERING + beforeAll( + // Run all the hooks sequentially + async () => setupHooks.reduce((p, fn) => p.then(fn), Promise.resolve()), + // Set timeout for tests to wait for downloading model before run + 10 * 60 * 1000, + ) + afterAll( + // Run all the hooks sequentially + async () => teardownHooks.reduce((p, fn) => p.then(fn), Promise.resolve()), + // Set timeout for cleaning up + 10 * 60 * 1000, + ) + /// END HOOKS REGISTERING + + /// BEGIN TESTS + test('start/stop nitro process normally', + async () => { + // Start nitro + await runModel({ + modelFullPath, + promptTemplate: modelCfg.settings.prompt_template, + }) + // Wait 5s for nitro to start + await sleep(5 * 1000) + // Stop nitro + await stopModel() + }, + // Set default timeout to 1 minutes + 1 * 60 * 1000, + ) + /// END TESTS +}) diff --git a/nitro-node/tsconfig.json b/nitro-node/tsconfig.json index 8ad3ae0ea..c4ccd8b7f 100644 --- a/nitro-node/tsconfig.json +++ b/nitro-node/tsconfig.json @@ -17,6 +17,7 @@ "declarationDir": "dist/types", "outDir": "dist", "importHelpers": true, + "esModuleInterop": true, "typeRoots": [ "node_modules/@types" ] From 33293ec93edbac36c06359ee41b74ee0803575fb Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Thu, 25 Jan 2024 12:34:34 +0700 Subject: [PATCH 12/49] feat(nitro-node): chat completion function - Add chat completion function - Add simple test case for chat completion - Add GitHub action to build and test nitro node --- .github/scripts/auto-sign.sh | 10 - .github/workflows/build-nitro-node.yml | 369 +++++++++++++++++++ nitro-node/.gitignore | 12 +- nitro-node/Makefile | 10 +- nitro-node/download-nitro.js | 3 +- nitro-node/package.json | 1 + nitro-node/src/index.ts | 75 +++- nitro-node/test/nitro-process.test.ts | 101 ++++- nitro-node/test/{ => test_assets}/model.json | 0 9 files changed, 541 insertions(+), 40 deletions(-) delete mode 100755 .github/scripts/auto-sign.sh create mode 100644 .github/workflows/build-nitro-node.yml rename nitro-node/test/{ => test_assets}/model.json (100%) diff --git a/.github/scripts/auto-sign.sh b/.github/scripts/auto-sign.sh deleted file mode 100755 index 5e6ef9750..000000000 --- a/.github/scripts/auto-sign.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Check if both APP_PATH and DEVELOPER_ID environment variables are set -if [[ -z "$APP_PATH" ]] || [[ -z "$DEVELOPER_ID" ]]; then - echo "Either APP_PATH or DEVELOPER_ID is not set. Skipping script execution." - exit 0 -fi - -# If both variables are set, execute the following commands -find "$APP_PATH" \( -type f -perm +111 -o -name "*.node" \) -exec codesign -s "$DEVELOPER_ID" --options=runtime {} \; diff --git a/.github/workflows/build-nitro-node.yml b/.github/workflows/build-nitro-node.yml new file mode 100644 index 000000000..ebb5a169f --- /dev/null +++ b/.github/workflows/build-nitro-node.yml @@ -0,0 +1,369 @@ +name: Build nitro-node + +on: + schedule: + - cron: "0 20 * * *" # At 8 PM UTC, which is 3 AM UTC+7 + push: + branches: + - main + - feat/1635/decouple-nitro-inference-engine-into-a-library + #tags: ["v[0-9]+.[0-9]+.[0-9]+"] + paths: + [ + ".github/scripts/**", + ".github/workflows/build-nitro-node.yml", + "nitro-node", + ] + pull_request: + types: [opened, synchronize, reopened] + paths: + [ + ".github/scripts/**", + ".github/workflows/build-nitro-node.yml", + "nitro-node", + ] + workflow_dispatch: + +env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + +jobs: + ubuntu-amd64-non-cuda-build: + runs-on: ubuntu-latest + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Restore cached model file + id: cache-model-restore + uses: actions/cache/restore@v4 + with: + path: | + test/test_assets/*.gguf + key: ${{ runner.os }}-model-gguf + + - uses: suisei-cn/actions-download-file@v1.4.0 + id: download-model-file + name: Download the file + with: + url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + target: test/test_assets/ + auto-match: true + retry-times: 3 + + - name: Save downloaded model file to cache + id: cache-model-save + uses: actions/cache/save@v4 + with: + path: | + test/test_assets/*.gguf + key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + - name: Run tests + id: test_nitro_node + run: | + cd nitro-node + make clean test + + #ubuntu-amd64-build: + # runs-on: ubuntu-18-04-cuda-11-7 + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + + # - name: Restore cached model file + # id: cache-model-restore + # uses: actions/cache/restore@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ runner.os }}-model-gguf + + # - uses: suisei-cn/actions-download-file@v1.4.0 + # id: download-model-file + # name: Download the file + # with: + # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + # target: test/test_assets/ + # auto-match: true + # retry-times: 3 + + # - name: Save downloaded model file to cache + # id: cache-model-save + # uses: actions/cache/save@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + # - name: Run tests + # id: test_nitro_node + # run: | + # cd nitro-node + # make clean test + + #ubuntu-amd64-cuda-build: + # runs-on: ubuntu-18-04-cuda-${{ matrix.cuda }} + # strategy: + # matrix: + # cuda: ["12-0", "11-7"] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + + # - name: Restore cached model file + # id: cache-model-restore + # uses: actions/cache/restore@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ runner.os }}-model-gguf + + # - uses: suisei-cn/actions-download-file@v1.4.0 + # id: download-model-file + # name: Download the file + # with: + # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + # target: test/test_assets/ + # auto-match: true + # retry-times: 3 + + # - name: Save downloaded model file to cache + # id: cache-model-save + # uses: actions/cache/save@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + # - name: Run tests + # id: test_nitro_node + # run: | + # cd nitro-node + # make clean test + + #macOS-M-build: + # runs-on: mac-silicon + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + + # - name: Restore cached model file + # id: cache-model-restore + # uses: actions/cache/restore@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ runner.os }}-model-gguf + + # - uses: suisei-cn/actions-download-file@v1.4.0 + # id: download-model-file + # name: Download the file + # with: + # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + # target: test/test_assets/ + # auto-match: true + # retry-times: 3 + + # - name: Save downloaded model file to cache + # id: cache-model-save + # uses: actions/cache/save@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + # - name: Run tests + # id: test_nitro_node + # run: | + # cd nitro-node + # make clean test + + #macOS-Intel-build: + # runs-on: macos-latest + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + + # - name: Restore cached model file + # id: cache-model-restore + # uses: actions/cache/restore@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ runner.os }}-model-gguf + + # - uses: suisei-cn/actions-download-file@v1.4.0 + # id: download-model-file + # name: Download the file + # with: + # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + # target: test/test_assets/ + # auto-match: true + # retry-times: 3 + + # - name: Save downloaded model file to cache + # id: cache-model-save + # uses: actions/cache/save@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + # - name: Run tests + # id: test_nitro_node + # run: | + # cd nitro-node + # make clean test + + #windows-amd64-build: + # runs-on: windows-latest + # steps: + # - name: Clone + + # id: checkout + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + + # - name: Setup VSWhere.exe + # uses: warrenbuckley/Setup-VSWhere@v1 + # with: + # version: latest + # silent: true + # env: + # ACTIONS_ALLOW_UNSECURE_COMMANDS: true + + # - name: Restore cached model file + # id: cache-model-restore + # uses: actions/cache/restore@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ runner.os }}-model-gguf + + # - uses: suisei-cn/actions-download-file@v1.4.0 + # id: download-model-file + # name: Download the file + # with: + # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + # target: test/test_assets/ + # auto-match: true + # retry-times: 3 + + # - name: Save downloaded model file to cache + # id: cache-model-save + # uses: actions/cache/save@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + # - name: Run tests + # id: test_nitro_node + # run: | + # cd nitro-node + # make clean test + + #windows-amd64-cuda-build: + # runs-on: windows-cuda-${{ matrix.cuda }} + # strategy: + # matrix: + # cuda: ["12-0", "11-7"] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + + # - name: actions-setup-cmake + # uses: jwlawson/actions-setup-cmake@v1.14.1 + + # - name: Setup VSWhere.exe + # uses: warrenbuckley/Setup-VSWhere@v1 + # with: + # version: latest + # silent: true + # env: + # ACTIONS_ALLOW_UNSECURE_COMMANDS: true + + # - uses: actions/setup-dotnet@v3 + # with: + # dotnet-version: "6.0.x" + + # - name: Restore cached model file + # id: cache-model-restore + # uses: actions/cache/restore@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ runner.os }}-model-gguf + + # - uses: suisei-cn/actions-download-file@v1.4.0 + # id: download-model-file + # name: Download the file + # with: + # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + # target: test/test_assets/ + # auto-match: true + # retry-times: 3 + + # - name: Save downloaded model file to cache + # id: cache-model-save + # uses: actions/cache/save@v4 + # with: + # path: | + # test/test_assets/*.gguf + # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + # - name: Run tests + # id: test_nitro_node + # run: | + # cd nitro-node + # make clean test diff --git a/nitro-node/.gitignore b/nitro-node/.gitignore index 56913fe29..446285591 100644 --- a/nitro-node/.gitignore +++ b/nitro-node/.gitignore @@ -2,7 +2,6 @@ .env # Jan inference -error.log .yarn node_modules *.tgz @@ -14,10 +13,7 @@ package-lock.json *.log -# Nitro binary files -bin/*/nitro -bin/*/*.metal -bin/*/*.exe -bin/*/*.dll -bin/*/*.exp -bin/*/*.lib \ No newline at end of file +# Nitro binary directory +bin/ +# Locally downloaded model for testing +test/test_assets/*.gguf \ No newline at end of file diff --git a/nitro-node/Makefile b/nitro-node/Makefile index e2d9195bb..6827c2536 100644 --- a/nitro-node/Makefile +++ b/nitro-node/Makefile @@ -21,13 +21,21 @@ build: install download-nitro: install yarn run downloadnitro +test-ci: install + yarn test + +# Note, this make target is just for testing on *NIX systems +test: install + @test -e test/test_assets/model.gguf && echo "test/test_assets/model.gguf is already downloaded" || (mkdir -p test/test_assets && curl -JLo test/test_assets/model.gguf "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf") + yarn test + # Builds and publishes the extension publish: build yarn run build:publish clean: ifeq ($(OS),Windows_NT) - del /F /S *.tgz .yarn yarn.lock package-lock.json bin + powershell -Command "Remove-Item -Recurse -Force -Path *.tgz, .yarn, yarn.lock, package-lock.json, bin" powershell -Command "Get-ChildItem -Path . -Include node_modules, dist -Recurse -Directory | Remove-Item -Recurse -Force" else rm -rf *.tgz .yarn yarn.lock package-lock.json bin diff --git a/nitro-node/download-nitro.js b/nitro-node/download-nitro.js index dd4cb8e5b..34de656b3 100644 --- a/nitro-node/download-nitro.js +++ b/nitro-node/download-nitro.js @@ -1,3 +1,4 @@ +const os = require('os'); const path = require('path'); const download = require('download'); @@ -46,7 +47,7 @@ const createProgressReporter = (variant) => (stream) => stream.on( 'downloadProgress', (progress) => { // Print and update progress on a single line of terminal - process.stdout.write(`\r[${variant}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`); + process.stdout.write(`\r\x1b[K[${variant}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`); }).on('end', () => { // Jump to new line to log next message console.log(); diff --git a/nitro-node/package.json b/nitro-node/package.json index 3cfbd795f..137db29f6 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -35,6 +35,7 @@ "rollup-plugin-sourcemaps": "^0.6.3", "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^29.1.2", + "ts-node": "^10.9.2", "typescript": "^5.3.3" }, "dependencies": { diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index 33d73fbf4..5c068a3e4 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -1,3 +1,4 @@ +import os from 'node:os'; import fs from "node:fs"; import path from "node:path"; import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; @@ -21,6 +22,8 @@ const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/ const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; // The URL for the Nitro subprocess to kill itself const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; +// The URL for the Nitro subprocess to run chat completion +const NITRO_HTTP_CHAT_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/chat_completion` // The supported model format // TODO: Should be an array to support more models @@ -33,7 +36,7 @@ let currentModelFile: string = ""; // The current model settings let currentSettings: NitroModelSetting | undefined = undefined; // The logger to use, default to console.log -let log: NitroLogger = (message, ..._) => console.log(message); +let log: NitroLogger = (message, ..._) => process.stdout.write(message + os.EOL); /** * Set logger before running nitro @@ -47,7 +50,7 @@ function setLogger(logger: NitroLogger) { * @param wrapper - The model wrapper. * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. */ -function stopModel(): Promise { +function stopModel(): Promise { return killSubprocess(); } @@ -114,9 +117,9 @@ async function runNitroAndLoadModel(): Promise setTimeout(resolve, 500)); + return new Promise((resolve) => setTimeout(() => resolve({}), 500)); } else { - return Promise.resolve(); + return Promise.resolve({}); } }) .then(spawnNitroProcess) @@ -192,6 +195,7 @@ async function loadLLMModel(settings: any): Promise { retries: 3, retryDelay: 500, }); + // FIXME: Actually check response, as the model directory might not exist log( `[NITRO]::Debug: Load model success with response ${JSON.stringify( res @@ -204,6 +208,51 @@ async function loadLLMModel(settings: any): Promise { } } +/** + * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified + * @param {any} request The request that is then sent to nitro + * @param {WritableStream} outStream Optional stream that consume the response body + * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. + * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data + */ +async function chatCompletion(request: any, outStream?: WritableStream): Promise { + if (outStream) { + // Add stream option if there is an outStream specified when calling this function + Object.assign(request, { + stream: true, + }) + } + log(`[NITRO]::Debug: Running chat completion with request ${JSON.stringify(request)}`); + return fetchRetry(NITRO_HTTP_CHAT_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + 'Accept': 'text/event-stream', + 'Access-Control-Allow-Origin': '*', + }, + body: JSON.stringify(request), + retries: 3, + retryDelay: 500, + }).then(async (response) => { + if (outStream) { + if (!response.body) { + throw new Error("Error running chat completion"); + } + const outPipe = response + .body + .pipeThrough(new TextDecoderStream()) + .pipeTo(outStream); + // Wait for all the streams to complete before returning from async function + await outPipe; + } + log(`[NITRO]::Debug: Chat completion success`); + return response; + }).catch((err) => { + log(`[NITRO]::Error: Chat completion failed with error ${err}`) + throw err + }) +} + /** * Validates the status of a model. * @returns {Promise} A promise that resolves to an object. @@ -235,7 +284,7 @@ async function validateModelStatus(): Promise { return Promise.resolve({}); } } - return Promise.reject("Validate model status failed"); + return Promise.resolve({ error: "Validate model status failed" }); }); } @@ -243,7 +292,7 @@ async function validateModelStatus(): Promise { * Terminates the Nitro subprocess. * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. */ -async function killSubprocess(): Promise { +async function killSubprocess(): Promise { const controller = new AbortController(); setTimeout(() => controller.abort(), 5000); log(`[NITRO]::Debug: Request to kill Nitro`); @@ -256,19 +305,20 @@ async function killSubprocess(): Promise { subprocess?.kill(); subprocess = undefined; }) - .catch(() => { }) + .catch((err) => ({ error: err })) .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)); + .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) + .then(() => Promise.resolve({})); } /** * Spawns a Nitro subprocess. * @returns A promise that resolves when the Nitro subprocess is started. */ -function spawnNitroProcess(): Promise { +function spawnNitroProcess(): Promise { log(`[NITRO]::Debug: Spawning Nitro subprocess...`); - return new Promise(async (resolve, reject) => { + return new Promise(async (resolve, reject) => { const binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default const executableOptions = executableNitroFile(); @@ -306,7 +356,7 @@ function spawnNitroProcess(): Promise { tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { log(`[NITRO]::Debug: Nitro is ready`); - resolve(); + resolve({}); }); }); } @@ -341,6 +391,9 @@ export default { setLogger, runModel, stopModel, + loadLLMModel, + validateModelStatus, + chatCompletion, killSubprocess, dispose, updateNvidiaInfo, diff --git a/nitro-node/test/nitro-process.test.ts b/nitro-node/test/nitro-process.test.ts index 27e407f5e..e2d02ebb9 100644 --- a/nitro-node/test/nitro-process.test.ts +++ b/nitro-node/test/nitro-process.test.ts @@ -1,4 +1,4 @@ -import { jest, describe, test } from '@jest/globals' +import { describe, test } from '@jest/globals' import fs from 'node:fs' import os from 'node:os' @@ -6,18 +6,20 @@ import path from 'node:path' import download from 'download' -import { default as nitro } from '../src/index' import { Duplex } from 'node:stream' -const { runModel, stopModel } = nitro +import { default as nitro } from '../src/index' +const { stopModel, runModel, loadLLMModel, validateModelStatus, chatCompletion } = nitro // FIXME: Shorthand only possible for es6 targets and up //import * as model from './model.json' assert {type: 'json'} +// Test assets dir +const TEST_ASSETS_PATH = path.join(__dirname, 'test_assets') +const MODEL_CONFIG_PATH = path.join(TEST_ASSETS_PATH, 'model.json') + // Get model config const getModelConfigHook = (callback: (modelCfg: any) => void) => () => { - const modelJson = fs.readFileSync(path.join(__dirname, 'model.json'), { - encoding: 'utf8', - }) + const modelJson = fs.readFileSync(MODEL_CONFIG_PATH, { encoding: 'utf8' }) const modelCfg = JSON.parse(modelJson) callback(modelCfg) } @@ -27,15 +29,24 @@ const createProgressReporter = (name: string) => (stream: Promise & Dupl 'downloadProgress', (progress: { transferred: any; total: any; percent: number }) => { // Print and update progress on a single line of terminal - process.stdout.write(`\r[${name}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`); + process.stdout.write(`\r\x1b[K[${name}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`); }).on('end', () => { // Jump to new line to log next message - process.stdout.write(`\n[${name}] Finished downloading!`); + process.stdout.write(`${os.EOL}[${name}] Finished downloading!`); }) // Download model file const downloadModelHook = (modelCfg: any, targetDir: string) => async () => { const fileName = modelCfg.source_url.split('/')?.pop() ?? 'model.gguf' + // Check if there is a downloaded model at TEST_ASSETS_PATH + const downloadedModelFile = fs.readdirSync(TEST_ASSETS_PATH).find((fname) => fname.match(/.*\.gguf/ig)) + if (downloadedModelFile) { + const downloadedModelPath = path.join(TEST_ASSETS_PATH, downloadedModelFile) + // Copy model files to targetDir and return + fs.cpSync(downloadedModelPath, path.join(targetDir, fileName)) + console.log(`Reuse cached model ${modelCfg.name} from path ${downloadedModelPath} => ${targetDir}`) + return + } const progressReporter = createProgressReporter(modelCfg.name) await progressReporter( download( @@ -84,6 +95,8 @@ describe('Manage nitro process', () => { ] // Teardown steps after running the suite const teardownHooks = [ + // Stop nitro after running, regardless of error or not + () => stopModel(), // On teardown, cleanup tmp directory that was created earlier cleanupTargetDirHook(modelFullPath), ] @@ -117,7 +130,77 @@ describe('Manage nitro process', () => { // Stop nitro await stopModel() }, - // Set default timeout to 1 minutes + // Set timeout to 30 seconds + 30 * 1000, + ) + test('chat completion', + async () => { + // Start nitro + await runModel({ + modelFullPath, + promptTemplate: modelCfg.settings.prompt_template, + }) + // Wait 5s for nitro to start + await sleep(5 * 1000) + // Load LLM model + await loadLLMModel({ + "llama_model_path": modelFullPath, + "ctx_len": 50, + "ngl": 32, + "embedding": false + }) + // Validate model status + await validateModelStatus() + // Arrays of all the chunked response + let streamedContent: string[] = [] + // Run chat completion with stream + const response = await chatCompletion( + { + "messages": [ + { "content": "Hello there", "role": "assistant" }, + { "content": "Write a long and sad story for me", "role": "user" } + ], + "model": "gpt-3.5-turbo", + "max_tokens": 50, + "stop": ["hello"], + "frequency_penalty": 0, + "presence_penalty": 0, + "temperature": 0.1 + }, + new WritableStream({ + write(chunk: string) { + if (chunk.trim() == 'data: [DONE]') { + return + } + return new Promise((resolve) => { + streamedContent.push( + chunk.slice('data:'.length).trim() + ) + resolve() + }) + }, + //close() {}, + //abort(_err) {} + }) + ) + // Remove the [DONE] message + streamedContent.pop() + // Parse json + streamedContent = streamedContent.map((str) => JSON.parse(str)) + // Show the streamed content + console.log(`[Streamed response] ${JSON.stringify(streamedContent, null, 2)}`) + + // The response body is unusable if consumed by out stream + await expect(response.text).rejects.toThrow() + await expect(response.json).rejects.toThrow() + // Response body should be used already + expect(response.bodyUsed).toBeTruthy() + // There should be multiple chunks of json data + expect(streamedContent.length).toBeGreaterThan(0) + // Stop nitro + await stopModel() + }, + // Set timeout to 1 minutes 1 * 60 * 1000, ) /// END TESTS diff --git a/nitro-node/test/model.json b/nitro-node/test/test_assets/model.json similarity index 100% rename from nitro-node/test/model.json rename to nitro-node/test/test_assets/model.json From d703f74f2cacb75e536c0a202b5cda613aa2de38 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Thu, 25 Jan 2024 17:07:16 +0700 Subject: [PATCH 13/49] fix(nitro-node): remove dependency on Jan's config file - Use runtime config instead - The caller of this library should provide their custom config instead --- .github/workflows/build-nitro-node.yml | 76 +++++++------- nitro-node/src/@types/global.d.ts | 18 ++++ nitro-node/src/execute.ts | 24 ++--- nitro-node/src/index.ts | 132 ++++++++++++++++--------- nitro-node/src/nvidia.ts | 88 +++-------------- 5 files changed, 165 insertions(+), 173 deletions(-) diff --git a/.github/workflows/build-nitro-node.yml b/.github/workflows/build-nitro-node.yml index ebb5a169f..e2196e429 100644 --- a/.github/workflows/build-nitro-node.yml +++ b/.github/workflows/build-nitro-node.yml @@ -208,49 +208,49 @@ jobs: # cd nitro-node # make clean test - #macOS-Intel-build: - # runs-on: macos-latest - # steps: - # - name: Clone - # id: checkout - # uses: actions/checkout@v4 - # with: - # submodules: recursive + macOS-Intel-build: + runs-on: macos-latest + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive - # - uses: actions/setup-node@v4 - # with: - # node-version: 18 + - uses: actions/setup-node@v4 + with: + node-version: 18 - # - name: Restore cached model file - # id: cache-model-restore - # uses: actions/cache/restore@v4 - # with: - # path: | - # test/test_assets/*.gguf - # key: ${{ runner.os }}-model-gguf + - name: Restore cached model file + id: cache-model-restore + uses: actions/cache/restore@v4 + with: + path: | + test/test_assets/*.gguf + key: ${{ runner.os }}-model-gguf - # - uses: suisei-cn/actions-download-file@v1.4.0 - # id: download-model-file - # name: Download the file - # with: - # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: test/test_assets/ - # auto-match: true - # retry-times: 3 + - uses: suisei-cn/actions-download-file@v1.4.0 + id: download-model-file + name: Download the file + with: + url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + target: test/test_assets/ + auto-match: true + retry-times: 3 - # - name: Save downloaded model file to cache - # id: cache-model-save - # uses: actions/cache/save@v4 - # with: - # path: | - # test/test_assets/*.gguf - # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + - name: Save downloaded model file to cache + id: cache-model-save + uses: actions/cache/save@v4 + with: + path: | + test/test_assets/*.gguf + key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - # - name: Run tests - # id: test_nitro_node - # run: | - # cd nitro-node - # make clean test + - name: Run tests + id: test_nitro_node + run: | + cd nitro-node + make clean test #windows-amd64-build: # runs-on: windows-latest diff --git a/nitro-node/src/@types/global.d.ts b/nitro-node/src/@types/global.d.ts index 3be794d55..bbf8a7f6c 100644 --- a/nitro-node/src/@types/global.d.ts +++ b/nitro-node/src/@types/global.d.ts @@ -42,4 +42,22 @@ interface NitroModelInitOptions { */ interface NitroLogger { (message: string, fileName?: string): void; +} + +/** + * Nvidia settings + */ +interface NitroNvidiaConfig { + notify: boolean, + run_mode: "cpu" | "gpu", + nvidia_driver: { + exist: boolean, + version: string, + }, + cuda: { + exist: boolean, + version: string, + }, + gpus: { id: string, vram: string }[], + gpu_highest_vram: string, } \ No newline at end of file diff --git a/nitro-node/src/execute.ts b/nitro-node/src/execute.ts index c190c678f..6f09d538c 100644 --- a/nitro-node/src/execute.ts +++ b/nitro-node/src/execute.ts @@ -1,7 +1,4 @@ -import { readFileSync } from "node:fs"; import path from "node:path"; -import { NVIDIA_INFO_FILE } from "./nvidia"; - export interface NitroExecutableOptions { executablePath: string; @@ -12,7 +9,9 @@ export interface NitroExecutableOptions { * Find which executable file to run based on the current platform. * @returns The name of the executable file to run. */ -export const executableNitroFile = (): NitroExecutableOptions => { +export const executableNitroFile = ( + nvidiaSettings: NitroNvidiaConfig, +): NitroExecutableOptions => { let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default let cudaVisibleDevices = ""; let binaryName = "nitro"; @@ -23,16 +22,15 @@ export const executableNitroFile = (): NitroExecutableOptions => { /** * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 */ - let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvidiaInfo["run_mode"] === "cpu") { + if (nvidiaSettings["run_mode"] === "cpu") { binaryFolder = path.join(binaryFolder, "win-cpu"); } else { - if (nvidiaInfo["cuda"].version === "12") { + if (nvidiaSettings["cuda"].version === "12") { binaryFolder = path.join(binaryFolder, "win-cuda-12-0"); } else { binaryFolder = path.join(binaryFolder, "win-cuda-11-7"); } - cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; + cudaVisibleDevices = nvidiaSettings["gpu_highest_vram"]; } binaryName = "nitro.exe"; } else if (process.platform === "darwin") { @@ -45,19 +43,15 @@ export const executableNitroFile = (): NitroExecutableOptions => { binaryFolder = path.join(binaryFolder, "mac-x64"); } } else { - /** - * For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0 - */ - let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvidiaInfo["run_mode"] === "cpu") { + if (nvidiaSettings["run_mode"] === "cpu") { binaryFolder = path.join(binaryFolder, "linux-cpu"); } else { - if (nvidiaInfo["cuda"].version === "12") { + if (nvidiaSettings["cuda"].version === "12") { binaryFolder = path.join(binaryFolder, "linux-cuda-12-0"); } else { binaryFolder = path.join(binaryFolder, "linux-cuda-11-7"); } - cudaVisibleDevices = nvidiaInfo["gpu_highest_vram"]; + cudaVisibleDevices = nvidiaSettings["gpu_highest_vram"]; } } return { diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index 5c068a3e4..bb97abe8c 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -1,4 +1,4 @@ -import os from 'node:os'; +import os from "node:os"; import fs from "node:fs"; import path from "node:path"; import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; @@ -23,7 +23,23 @@ const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llama // The URL for the Nitro subprocess to kill itself const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; // The URL for the Nitro subprocess to run chat completion -const NITRO_HTTP_CHAT_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/chat_completion` +const NITRO_HTTP_CHAT_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/chat_completion`; + +// The default config for using Nvidia GPU +const NVIDIA_DEFAULT_CONFIG: NitroNvidiaConfig = { + notify: true, + run_mode: "cpu", + nvidia_driver: { + exist: false, + version: "", + }, + cuda: { + exist: false, + version: "", + }, + gpus: [], + gpu_highest_vram: "", +}; // The supported model format // TODO: Should be an array to support more models @@ -35,8 +51,18 @@ let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; let currentModelFile: string = ""; // The current model settings let currentSettings: NitroModelSetting | undefined = undefined; +// The Nvidia info file for checking for CUDA support on the system +let nvidiaConfig: NitroNvidiaConfig = NVIDIA_DEFAULT_CONFIG; // The logger to use, default to console.log -let log: NitroLogger = (message, ..._) => process.stdout.write(message + os.EOL); +let log: NitroLogger = (message, ..._) => + process.stdout.write(message + os.EOL); + +/** + * Set custom Nvidia config for running inference over GPU + */ +function setNvidiaConfig(settings: NitroNvidiaConfig) { + nvidiaConfig = settings; +} /** * Set logger before running nitro @@ -61,19 +87,17 @@ function stopModel(): Promise { * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package */ -async function runModel( - { - modelFullPath, - promptTemplate, - }: NitroModelInitOptions -): Promise { +async function runModel({ + modelFullPath, + promptTemplate, +}: NitroModelInitOptions): Promise { const files: string[] = fs.readdirSync(modelFullPath); // Look for GGUF model file const ggufBinFile = files.find( (file) => file === path.basename(modelFullPath) || - file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT) + file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT), ); if (!ggufBinFile) return Promise.reject("No GGUF model file found"); @@ -95,7 +119,10 @@ async function runModel( ...prompt, llama_model_path: currentModelFile, // This is critical and requires real system information - cpu_threads: Math.max(1, Math.round(nitroResourceProbe.numCpuPhysicalCore / 2)), + cpu_threads: Math.max( + 1, + Math.round(nitroResourceProbe.numCpuPhysicalCore / 2), + ), }; return runNitroAndLoadModel(); } @@ -106,7 +133,9 @@ async function runModel( * 3. Validate model status * @returns */ -async function runNitroAndLoadModel(): Promise { +async function runNitroAndLoadModel(): Promise< + NitroModelOperationResponse | { error: any } +> { // Gather system information for CPU physical cores and memory return killSubprocess() .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) @@ -138,7 +167,9 @@ async function runNitroAndLoadModel(): Promise { }); // FIXME: Actually check response, as the model directory might not exist log( - `[NITRO]::Debug: Load model success with response ${JSON.stringify( - res - )}` + `[NITRO]::Debug: Load model success with response ${JSON.stringify(res)}`, ); return await Promise.resolve(res); } catch (err) { @@ -215,42 +244,48 @@ async function loadLLMModel(settings: any): Promise { * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data */ -async function chatCompletion(request: any, outStream?: WritableStream): Promise { +async function chatCompletion( + request: any, + outStream?: WritableStream, +): Promise { if (outStream) { // Add stream option if there is an outStream specified when calling this function Object.assign(request, { stream: true, - }) + }); } - log(`[NITRO]::Debug: Running chat completion with request ${JSON.stringify(request)}`); + log( + `[NITRO]::Debug: Running chat completion with request ${JSON.stringify(request)}`, + ); return fetchRetry(NITRO_HTTP_CHAT_URL, { method: "POST", headers: { "Content-Type": "application/json", - 'Accept': 'text/event-stream', - 'Access-Control-Allow-Origin': '*', + Accept: "text/event-stream", + "Access-Control-Allow-Origin": "*", }, body: JSON.stringify(request), retries: 3, retryDelay: 500, - }).then(async (response) => { - if (outStream) { - if (!response.body) { - throw new Error("Error running chat completion"); - } - const outPipe = response - .body - .pipeThrough(new TextDecoderStream()) - .pipeTo(outStream); - // Wait for all the streams to complete before returning from async function - await outPipe; - } - log(`[NITRO]::Debug: Chat completion success`); - return response; - }).catch((err) => { - log(`[NITRO]::Error: Chat completion failed with error ${err}`) - throw err }) + .then(async (response) => { + if (outStream) { + if (!response.body) { + throw new Error("Error running chat completion"); + } + const outPipe = response.body + .pipeThrough(new TextDecoderStream()) + .pipeTo(outStream); + // Wait for all the streams to complete before returning from async function + await outPipe; + } + log(`[NITRO]::Debug: Chat completion success`); + return response; + }) + .catch((err) => { + log(`[NITRO]::Error: Chat completion failed with error ${err}`); + throw err; + }); } /** @@ -272,8 +307,8 @@ async function validateModelStatus(): Promise { }).then(async (res: Response) => { log( `[NITRO]::Debug: Validate model state success with response ${JSON.stringify( - res - )}` + res, + )}`, ); // If the response is OK, check model_loaded status. if (res.ok) { @@ -320,12 +355,12 @@ function spawnNitroProcess(): Promise { return new Promise(async (resolve, reject) => { const binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default - const executableOptions = executableNitroFile(); + const executableOptions = executableNitroFile(nvidiaConfig); const args: string[] = ["1", LOCAL_HOST, PORT.toString()]; // Execute the binary log( - `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}` + `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}`, ); subprocess = spawn( executableOptions.executablePath, @@ -336,7 +371,7 @@ function spawnNitroProcess(): Promise { ...process.env, CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices, }, - } + }, ); // Handle subprocess output @@ -363,7 +398,7 @@ function spawnNitroProcess(): Promise { /** * Get the system resources information - * TODO: Move to Core so that it can be reused + * TODO: Move to @janhq/core so that it can be reused */ function getResourcesInfo(): Promise { return new Promise(async (resolve) => { @@ -388,6 +423,7 @@ function dispose() { } export default { + setNvidiaConfig, setLogger, runModel, stopModel, @@ -396,6 +432,6 @@ export default { chatCompletion, killSubprocess, dispose, - updateNvidiaInfo, + updateNvidiaInfo: async () => await updateNvidiaInfo(nvidiaConfig), getCurrentNitroProcessInfo: () => getNitroProcessInfo(subprocess), }; diff --git a/nitro-node/src/nvidia.ts b/nitro-node/src/nvidia.ts index a700d0bb5..53cf64664 100644 --- a/nitro-node/src/nvidia.ts +++ b/nitro-node/src/nvidia.ts @@ -1,35 +1,7 @@ import { writeFileSync, existsSync, readFileSync } from "node:fs"; import { exec } from "node:child_process"; import path from "node:path"; -import { homedir } from "node:os"; -/** - * Default GPU settings - **/ -const DEFALT_SETTINGS = { - notify: true, - run_mode: "cpu", - nvidia_driver: { - exist: false, - version: "", - }, - cuda: { - exist: false, - version: "", - }, - gpus: [], - gpu_highest_vram: "", -}; - -/** - * Path to the settings file - **/ -export const NVIDIA_INFO_FILE = path.join( - homedir(), - "jan", - "settings", - "settings.json" -); /** * Current nitro process @@ -47,12 +19,12 @@ export interface NitroProcessInfo { * This will retrive GPU informations and persist settings.json * Will be called when the extension is loaded to turn on GPU acceleration if supported */ -export async function updateNvidiaInfo() { +export async function updateNvidiaInfo(nvidiaSettings: NitroNvidiaConfig) { if (process.platform !== "darwin") { await Promise.all([ - updateNvidiaDriverInfo(), - updateCudaExistence(), - updateGpuInfo(), + updateNvidiaDriverInfo(nvidiaSettings), + updateCudaExistence(nvidiaSettings), + updateGpuInfo(nvidiaSettings), ]); } } @@ -70,27 +42,17 @@ export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { /** * Validate nvidia and cuda for linux and windows */ -export async function updateNvidiaDriverInfo(): Promise { +export async function updateNvidiaDriverInfo(nvidiaSettings: NitroNvidiaConfig): Promise { exec( "nvidia-smi --query-gpu=driver_version --format=csv,noheader", (error, stdout) => { - let data; - try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - } catch (error) { - data = DEFALT_SETTINGS; - } - if (!error) { const firstLine = stdout.split("\n")[0].trim(); - data["nvidia_driver"].exist = true; - data["nvidia_driver"].version = firstLine; + nvidiaSettings["nvidia_driver"].exist = true; + nvidiaSettings["nvidia_driver"].version = firstLine; } else { - data["nvidia_driver"].exist = false; + nvidiaSettings["nvidia_driver"].exist = false; } - - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - Promise.resolve(); } ); } @@ -108,7 +70,7 @@ export function checkFileExistenceInPaths( /** * Validate cuda for linux and windows */ -export function updateCudaExistence() { +export function updateCudaExistence(nvidiaSettings: NitroNvidiaConfig) { let filesCuda12: string[]; let filesCuda11: string[]; let paths: string[]; @@ -142,35 +104,20 @@ export function updateCudaExistence() { cudaVersion = "12"; } - let data; - try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - } catch (error) { - data = DEFALT_SETTINGS; - } - - data["cuda"].exist = cudaExists; - data["cuda"].version = cudaVersion; + nvidiaSettings["cuda"].exist = cudaExists; + nvidiaSettings["cuda"].version = cudaVersion; if (cudaExists) { - data.run_mode = "gpu"; + nvidiaSettings.run_mode = "gpu"; } - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); } /** * Get GPU information */ -export async function updateGpuInfo(): Promise { +export async function updateGpuInfo(nvidiaSettings: NitroNvidiaConfig): Promise { exec( "nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", (error, stdout) => { - let data; - try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - } catch (error) { - data = DEFALT_SETTINGS; - } - if (!error) { // Get GPU info and gpu has higher memory first let highestVram = 0; @@ -188,14 +135,11 @@ export async function updateGpuInfo(): Promise { return { id, vram }; }); - data["gpus"] = gpus; - data["gpu_highest_vram"] = highestVramId; + nvidiaSettings["gpus"] = gpus; + nvidiaSettings["gpu_highest_vram"] = highestVramId; } else { - data["gpus"] = []; + nvidiaSettings["gpus"] = []; } - - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - Promise.resolve(); } ); } From 60063bd2cd0082a670b3751ef245af5246f16215 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Thu, 25 Jan 2024 21:47:08 +0700 Subject: [PATCH 14/49] chore(nitro-node/ci): cleanup CI workflow - Cleanup unused configs - Fix cache for LLM model file - Remove unnecessary dependencies - Remove unused imports and remaining business logic from Jan --- .github/workflows/build-nitro-node.yml | 91 +++++++++++--------------- nitro-node/Makefile | 2 +- nitro-node/download-nitro.js | 73 +++++++++++---------- nitro-node/package.json | 3 - nitro-node/src/index.ts | 37 ++++++----- nitro-node/src/nvidia.ts | 23 ++++--- 6 files changed, 110 insertions(+), 119 deletions(-) diff --git a/.github/workflows/build-nitro-node.yml b/.github/workflows/build-nitro-node.yml index e2196e429..015b7a138 100644 --- a/.github/workflows/build-nitro-node.yml +++ b/.github/workflows/build-nitro-node.yml @@ -1,32 +1,19 @@ name: Build nitro-node on: - schedule: - - cron: "0 20 * * *" # At 8 PM UTC, which is 3 AM UTC+7 + #schedule: + # - cron: "0 20 * * *" # At 8 PM UTC, which is 3 AM UTC+7 push: branches: - main - feat/1635/decouple-nitro-inference-engine-into-a-library #tags: ["v[0-9]+.[0-9]+.[0-9]+"] - paths: - [ - ".github/scripts/**", - ".github/workflows/build-nitro-node.yml", - "nitro-node", - ] + paths: [".github/workflows/build-nitro-node.yml", "nitro-node"] pull_request: types: [opened, synchronize, reopened] - paths: - [ - ".github/scripts/**", - ".github/workflows/build-nitro-node.yml", - "nitro-node", - ] + paths: [".github/workflows/build-nitro-node.yml", "nitro-node"] workflow_dispatch: -env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - jobs: ubuntu-amd64-non-cuda-build: runs-on: ubuntu-latest @@ -46,15 +33,15 @@ jobs: uses: actions/cache/restore@v4 with: path: | - test/test_assets/*.gguf + nitro-node/test/test_assets/*.gguf key: ${{ runner.os }}-model-gguf - uses: suisei-cn/actions-download-file@v1.4.0 id: download-model-file - name: Download the file + name: Download model file with: url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - target: test/test_assets/ + target: nitro-node/test/test_assets/ auto-match: true retry-times: 3 @@ -63,14 +50,14 @@ jobs: uses: actions/cache/save@v4 with: path: | - test/test_assets/*.gguf + nitro-node/test/test_assets/*.gguf key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - name: Run tests id: test_nitro_node run: | cd nitro-node - make clean test + make clean test-ci #ubuntu-amd64-build: # runs-on: ubuntu-18-04-cuda-11-7 @@ -90,15 +77,15 @@ jobs: # uses: actions/cache/restore@v4 # with: # path: | - # test/test_assets/*.gguf + # nitro-node/test/test_assets/*.gguf # key: ${{ runner.os }}-model-gguf # - uses: suisei-cn/actions-download-file@v1.4.0 # id: download-model-file - # name: Download the file + # name: Download model file # with: # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: test/test_assets/ + # target: nitro-node/test/test_assets/ # auto-match: true # retry-times: 3 @@ -107,14 +94,14 @@ jobs: # uses: actions/cache/save@v4 # with: # path: | - # test/test_assets/*.gguf + # nitro-node/test/test_assets/*.gguf # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} # - name: Run tests # id: test_nitro_node # run: | # cd nitro-node - # make clean test + # make clean test-ci #ubuntu-amd64-cuda-build: # runs-on: ubuntu-18-04-cuda-${{ matrix.cuda }} @@ -138,15 +125,15 @@ jobs: # uses: actions/cache/restore@v4 # with: # path: | - # test/test_assets/*.gguf + # nitro-node/test/test_assets/*.gguf # key: ${{ runner.os }}-model-gguf # - uses: suisei-cn/actions-download-file@v1.4.0 # id: download-model-file - # name: Download the file + # name: Download model file # with: # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: test/test_assets/ + # target: nitro-node/test/test_assets/ # auto-match: true # retry-times: 3 @@ -155,14 +142,14 @@ jobs: # uses: actions/cache/save@v4 # with: # path: | - # test/test_assets/*.gguf + # nitro-node/test/test_assets/*.gguf # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} # - name: Run tests # id: test_nitro_node # run: | # cd nitro-node - # make clean test + # make clean test-ci #macOS-M-build: # runs-on: mac-silicon @@ -182,15 +169,15 @@ jobs: # uses: actions/cache/restore@v4 # with: # path: | - # test/test_assets/*.gguf + # nitro-node/test/test_assets/*.gguf # key: ${{ runner.os }}-model-gguf # - uses: suisei-cn/actions-download-file@v1.4.0 # id: download-model-file - # name: Download the file + # name: Download model file # with: # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: test/test_assets/ + # target: nitro-node/test/test_assets/ # auto-match: true # retry-times: 3 @@ -199,14 +186,14 @@ jobs: # uses: actions/cache/save@v4 # with: # path: | - # test/test_assets/*.gguf + # nitro-node/test/test_assets/*.gguf # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} # - name: Run tests # id: test_nitro_node # run: | # cd nitro-node - # make clean test + # make clean test-ci macOS-Intel-build: runs-on: macos-latest @@ -226,15 +213,15 @@ jobs: uses: actions/cache/restore@v4 with: path: | - test/test_assets/*.gguf + nitro-node/test/test_assets/*.gguf key: ${{ runner.os }}-model-gguf - uses: suisei-cn/actions-download-file@v1.4.0 id: download-model-file - name: Download the file + name: Download model file with: url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - target: test/test_assets/ + target: nitro-node/test/test_assets/ auto-match: true retry-times: 3 @@ -243,14 +230,14 @@ jobs: uses: actions/cache/save@v4 with: path: | - test/test_assets/*.gguf + nitro-node/test/test_assets/*.gguf key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - name: Run tests id: test_nitro_node run: | cd nitro-node - make clean test + make clean test-ci #windows-amd64-build: # runs-on: windows-latest @@ -279,15 +266,15 @@ jobs: # uses: actions/cache/restore@v4 # with: # path: | - # test/test_assets/*.gguf + # nitro-node/test/test_assets/*.gguf # key: ${{ runner.os }}-model-gguf # - uses: suisei-cn/actions-download-file@v1.4.0 # id: download-model-file - # name: Download the file + # name: Download model file # with: # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: test/test_assets/ + # target: nitro-node/test/test_assets/ # auto-match: true # retry-times: 3 @@ -296,14 +283,14 @@ jobs: # uses: actions/cache/save@v4 # with: # path: | - # test/test_assets/*.gguf + # nitro-node/test/test_assets/*.gguf # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} # - name: Run tests # id: test_nitro_node # run: | # cd nitro-node - # make clean test + # make clean test-ci #windows-amd64-cuda-build: # runs-on: windows-cuda-${{ matrix.cuda }} @@ -342,15 +329,15 @@ jobs: # uses: actions/cache/restore@v4 # with: # path: | - # test/test_assets/*.gguf + # nitro-node/test/test_assets/*.gguf # key: ${{ runner.os }}-model-gguf # - uses: suisei-cn/actions-download-file@v1.4.0 # id: download-model-file - # name: Download the file + # name: Download model file # with: # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: test/test_assets/ + # target: nitro-node/test/test_assets/ # auto-match: true # retry-times: 3 @@ -359,11 +346,11 @@ jobs: # uses: actions/cache/save@v4 # with: # path: | - # test/test_assets/*.gguf + # nitro-node/test/test_assets/*.gguf # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} # - name: Run tests # id: test_nitro_node # run: | # cd nitro-node - # make clean test + # make clean test-ci diff --git a/nitro-node/Makefile b/nitro-node/Makefile index 6827c2536..a673a2ac9 100644 --- a/nitro-node/Makefile +++ b/nitro-node/Makefile @@ -26,7 +26,7 @@ test-ci: install # Note, this make target is just for testing on *NIX systems test: install - @test -e test/test_assets/model.gguf && echo "test/test_assets/model.gguf is already downloaded" || (mkdir -p test/test_assets && curl -JLo test/test_assets/model.gguf "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf") + @test -e test/test_assets/*.gguf && echo "test/test_assets/*.gguf is already downloaded" || (mkdir -p test/test_assets && cd test/test_assets/ && curl -JLO "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf") yarn test # Builds and publishes the extension diff --git a/nitro-node/download-nitro.js b/nitro-node/download-nitro.js index 34de656b3..71eec4f42 100644 --- a/nitro-node/download-nitro.js +++ b/nitro-node/download-nitro.js @@ -1,37 +1,36 @@ -const os = require('os'); -const path = require('path'); -const download = require('download'); +const path = require("path"); +const download = require("download"); // Define nitro version to download in env variable -const NITRO_VERSION = process.env.NITRO_VERSION || '0.2.11'; +const NITRO_VERSION = process.env.NITRO_VERSION || "0.2.11"; // The platform OS to download nitro for const PLATFORM = process.env.npm_config_platform || process.platform; // The platform architecture //const ARCH = process.env.npm_config_arch || process.arch; const linuxVariants = { - 'linux-amd64': path.join(__dirname, 'bin', 'linux-cpu'), - 'linux-amd64-cuda-12-0': path.join(__dirname, 'bin', 'linux-cuda-12-0'), - 'linux-amd64-cuda-11-7': path.join(__dirname, 'bin', 'linux-cuda-11-7'), -} + "linux-amd64": path.join(__dirname, "bin", "linux-cpu"), + "linux-amd64-cuda-12-0": path.join(__dirname, "bin", "linux-cuda-12-0"), + "linux-amd64-cuda-11-7": path.join(__dirname, "bin", "linux-cuda-11-7"), +}; const darwinVariants = { - 'mac-arm64': path.join(__dirname, 'bin', 'mac-arm64'), - 'mac-amd64': path.join(__dirname, 'bin', 'mac-x64'), -} + "mac-arm64": path.join(__dirname, "bin", "mac-arm64"), + "mac-amd64": path.join(__dirname, "bin", "mac-x64"), +}; const win32Variants = { - 'win-amd64-cuda-12-0': path.join(__dirname, 'bin', 'win-cuda-12-0'), - 'win-amd64-cuda-11-7': path.join(__dirname, 'bin', 'win-cuda-11-7'), - 'win-amd64': path.join(__dirname, 'bin', 'win-cpu'), -} + "win-amd64-cuda-12-0": path.join(__dirname, "bin", "win-cuda-12-0"), + "win-amd64-cuda-11-7": path.join(__dirname, "bin", "win-cuda-11-7"), + "win-amd64": path.join(__dirname, "bin", "win-cpu"), +}; // Mapping to installation variants const variantMapping = { - 'darwin': darwinVariants, - 'linux': linuxVariants, - 'win32': win32Variants, -} + darwin: darwinVariants, + linux: linuxVariants, + win32: win32Variants, +}; if (!(PLATFORM in variantMapping)) { throw Error(`Invalid platform: ${PLATFORM}`); @@ -40,19 +39,23 @@ if (!(PLATFORM in variantMapping)) { const variantConfig = variantMapping[PLATFORM]; // Generate download link for each tarball -const getTarUrl = (version, suffix) => `https://github.com/janhq/nitro/releases/download/v${version}/nitro-${version}-${suffix}.tar.gz` +const getTarUrl = (version, suffix) => + `https://github.com/janhq/nitro/releases/download/v${version}/nitro-${version}-${suffix}.tar.gz`; // Report download progress -const createProgressReporter = (variant) => (stream) => stream.on( - 'downloadProgress', - (progress) => { - // Print and update progress on a single line of terminal - process.stdout.write(`\r\x1b[K[${variant}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`); - }).on('end', () => { - // Jump to new line to log next message - console.log(); - console.log(`[${variant}] Finished downloading!`); - }) +const createProgressReporter = (variant) => (stream) => + stream + .on("downloadProgress", (progress) => { + // Print and update progress on a single line of terminal + process.stdout.write( + `\r\x1b[K[${variant}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`, + ); + }) + .on("end", () => { + // Jump to new line to log next message + console.log(); + console.log(`[${variant}] Finished downloading!`); + }); // Download single binary const downloadBinary = (version, suffix, filePath) => { @@ -63,9 +66,9 @@ const downloadBinary = (version, suffix, filePath) => { download(tarUrl, filePath, { strip: 1, extract: true, - }) + }), ); -} +}; // Download the binaries const downloadBinaries = (version, config) => { @@ -73,15 +76,15 @@ const downloadBinaries = (version, config) => { (p, [k, v]) => p.then(() => downloadBinary(version, k, v)), Promise.resolve(), ); -} +}; // Call the download function with version and config const run = () => { downloadBinaries(NITRO_VERSION, variantConfig); -} +}; module.exports = run; -module.exports.createProgressReporter = createProgressReporter +module.exports.createProgressReporter = createProgressReporter; // Run script if called directly instead of import as module if (require.main === module) { diff --git a/nitro-node/package.json b/nitro-node/package.json index 137db29f6..caf9ee7e0 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -22,16 +22,13 @@ "@rollup/plugin-commonjs": "^25.0.7", "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.2.3", - "@rollup/plugin-replace": "^5.0.5", "@types/download": "^8.0.5", "@types/jest": "^29.5.11", "@types/node": "^20.11.4", "@types/os-utils": "^0.0.4", "@types/tcp-port-used": "^1.0.4", "jest": "^29.7.0", - "rimraf": "^3.0.2", "rollup": "^2.38.5", - "rollup-plugin-define": "^1.0.1", "rollup-plugin-sourcemaps": "^0.6.3", "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^29.1.2", diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index bb97abe8c..afe16f550 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -43,7 +43,7 @@ const NVIDIA_DEFAULT_CONFIG: NitroNvidiaConfig = { // The supported model format // TODO: Should be an array to support more models -const SUPPORTED_MODEL_FORMAT = ".gguf"; +const SUPPORTED_MODEL_FORMATS = [".gguf"]; // The subprocess instance for Nitro let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; @@ -53,19 +53,31 @@ let currentModelFile: string = ""; let currentSettings: NitroModelSetting | undefined = undefined; // The Nvidia info file for checking for CUDA support on the system let nvidiaConfig: NitroNvidiaConfig = NVIDIA_DEFAULT_CONFIG; -// The logger to use, default to console.log +// The logger to use, default to stdout let log: NitroLogger = (message, ..._) => process.stdout.write(message + os.EOL); +/** + * Get current Nvidia config + * @returns {NitroNvidiaConfig} A copy of the config object + * The returned object should be used for reading only + * Writing to config should be via the function {@setNvidiaConfig} + */ +function getNvidiaConfig(): NitroNvidiaConfig { + return Object.assign({}, nvidiaConfig); +} + /** * Set custom Nvidia config for running inference over GPU + * @param {NitroNvidiaConfig} config The new config to apply */ -function setNvidiaConfig(settings: NitroNvidiaConfig) { - nvidiaConfig = settings; +function setNvidiaConfig(config: NitroNvidiaConfig) { + nvidiaConfig = config; } /** * Set logger before running nitro + * @param {NitroLogger} logger The logger to use */ function setLogger(logger: NitroLogger) { log = logger; @@ -93,11 +105,11 @@ async function runModel({ }: NitroModelInitOptions): Promise { const files: string[] = fs.readdirSync(modelFullPath); - // Look for GGUF model file + // Look for model file with supported format const ggufBinFile = files.find( (file) => file === path.basename(modelFullPath) || - file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT), + SUPPORTED_MODEL_FORMATS.some((ext) => file.toLowerCase().endsWith(ext)), ); if (!ggufBinFile) return Promise.reject("No GGUF model file found"); @@ -398,7 +410,6 @@ function spawnNitroProcess(): Promise { /** * Get the system resources information - * TODO: Move to @janhq/core so that it can be reused */ function getResourcesInfo(): Promise { return new Promise(async (resolve) => { @@ -412,17 +423,8 @@ function getResourcesInfo(): Promise { }); } -/** - * Every module should have a dispose function - * This will be called when the extension is unloaded and should clean up any resources - * Also called when app is closed - */ -function dispose() { - // clean other registered resources here - killSubprocess(); -} - export default { + getNvidiaConfig, setNvidiaConfig, setLogger, runModel, @@ -431,7 +433,6 @@ export default { validateModelStatus, chatCompletion, killSubprocess, - dispose, updateNvidiaInfo: async () => await updateNvidiaInfo(nvidiaConfig), getCurrentNitroProcessInfo: () => getNitroProcessInfo(subprocess), }; diff --git a/nitro-node/src/nvidia.ts b/nitro-node/src/nvidia.ts index 53cf64664..c73173e73 100644 --- a/nitro-node/src/nvidia.ts +++ b/nitro-node/src/nvidia.ts @@ -1,8 +1,7 @@ -import { writeFileSync, existsSync, readFileSync } from "node:fs"; +import { existsSync } from "node:fs"; import { exec } from "node:child_process"; import path from "node:path"; - /** * Current nitro process */ @@ -12,7 +11,7 @@ let nitroProcessInfo: NitroProcessInfo | undefined = undefined; * Nitro process info */ export interface NitroProcessInfo { - isRunning: boolean + isRunning: boolean; } /** @@ -42,7 +41,9 @@ export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { /** * Validate nvidia and cuda for linux and windows */ -export async function updateNvidiaDriverInfo(nvidiaSettings: NitroNvidiaConfig): Promise { +export async function updateNvidiaDriverInfo( + nvidiaSettings: NitroNvidiaConfig, +): Promise { exec( "nvidia-smi --query-gpu=driver_version --format=csv,noheader", (error, stdout) => { @@ -53,7 +54,7 @@ export async function updateNvidiaDriverInfo(nvidiaSettings: NitroNvidiaConfig): } else { nvidiaSettings["nvidia_driver"].exist = false; } - } + }, ); } @@ -62,7 +63,7 @@ export async function updateNvidiaDriverInfo(nvidiaSettings: NitroNvidiaConfig): */ export function checkFileExistenceInPaths( file: string, - paths: string[] + paths: string[], ): boolean { return paths.some((p) => existsSync(path.join(p, file))); } @@ -90,12 +91,12 @@ export function updateCudaExistence(nvidiaSettings: NitroNvidiaConfig) { } let cudaExists = filesCuda12.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths), ); if (!cudaExists) { cudaExists = filesCuda11.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths), ); if (cudaExists) { cudaVersion = "11"; @@ -114,7 +115,9 @@ export function updateCudaExistence(nvidiaSettings: NitroNvidiaConfig) { /** * Get GPU information */ -export async function updateGpuInfo(nvidiaSettings: NitroNvidiaConfig): Promise { +export async function updateGpuInfo( + nvidiaSettings: NitroNvidiaConfig, +): Promise { exec( "nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", (error, stdout) => { @@ -140,6 +143,6 @@ export async function updateGpuInfo(nvidiaSettings: NitroNvidiaConfig): Promise< } else { nvidiaSettings["gpus"] = []; } - } + }, ); } From e8e7752218d98f478e86e7b058bb5f68d40d9695 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Thu, 25 Jan 2024 23:20:05 +0700 Subject: [PATCH 15/49] chore(nitro-node/ci): test install on different platforms --- .../scripts/e2e-test-install-nitro-node.js | 143 ++++++++++++++++++ .github/workflows/test-install-nitro-node.yml | 83 ++++++++++ 2 files changed, 226 insertions(+) create mode 100644 .github/scripts/e2e-test-install-nitro-node.js create mode 100644 .github/workflows/test-install-nitro-node.yml diff --git a/.github/scripts/e2e-test-install-nitro-node.js b/.github/scripts/e2e-test-install-nitro-node.js new file mode 100644 index 000000000..4352bdfca --- /dev/null +++ b/.github/scripts/e2e-test-install-nitro-node.js @@ -0,0 +1,143 @@ +const os = require("node:os"); +const path = require("node:path"); +const fs = require("node:fs"); +const { exec } = require("node:child_process"); + +// Test on both npm and yarn +const PACKAGE_MANAGERS = ["npm", "yarn"]; +// TODO: Path to the package to install. Change to '@janhq/nitro-node' after published +const NITRO_NODE_PKG = path.normalize( + path.join(__dirname, "..", "..", "nitro-node"), +); +const BIN_DIR_PREFIXES = { + darwin: "mac", + win32: "win", + linux: "linux", +}; + +// Utility to check for a file with nitro in name in the corresponding directory +const checkBinaries = (repoDir) => { + // FIXME: Check for unsupported platforms + const binDirPrefix = BIN_DIR_PREFIXES[process.platform]; + const searchRoot = path.join(repoDir, "node_modules", "@janhq", "nitro-node"); + // Get the dir and files that indicate successful download of binaries + const matched = fs + .readdirSync(searchRoot, { recursive: true }) + .filter( + (fname) => fname.startsWith(binDirPrefix) || fname.startsWith("nitro"), + ); + + // Must have both the directory for the platform and the binary + return ( + matched.some((fname) => fname.startsWith(binDirPrefix)) && + matched.some((fname) => fname.startsWith(binDirPrefix)) + ); +}; + +// Wrapper to wait for child process to finish +const childProcessPromise = (cmd) => + new Promise((resolve, reject) => { + cmd.on("exit", (exitCode) => { + const exitNum = Number(exitCode); + if (0 == exitNum) { + resolve(); + } else { + reject(exitNum); + } + }); + }); + +// Create a temporary directory for testing +const createTestDir = () => + fs.mkdtempSync(path.join(os.tmpdir(), "dummy-project")); + +// First test, create empty project dir and add nitro-node as dependency +const firstTest = async (packageManager, repoDir) => { + console.log(`[First test @ ${repoDir}] install with ${packageManager}`); + // Init project with default package.json + const cmd1 = exec(`npm init -y`, { cwd: repoDir }, (err) => { + if (err) { + console.error(err); + // Error at first step + process.exit(1); + } + }); + await childProcessPromise(cmd1); + + // Add nitro-node as dependency + const cmd2 = exec( + `${packageManager} add ${NITRO_NODE_PKG}`, + { cwd: repoDir }, + (err) => { + if (err) { + console.error(err); + // Error at second step + process.exit(2); + } + }, + ); + await childProcessPromise(cmd2); + + // Check that the binaries exists + if (!checkBinaries(repoDir)) process.exit(3); + + // Cleanup node_modules after success + fs.rmSync(path.join(repoDir, "node_modules"), { recursive: true }); +}; + +// Second test, install the wrapper from another project dir +const secondTest = async (packageManager, repoDir, wrapperDir) => { + console.log( + `[Second test @ ${repoDir}] install ${wrapperDir} with ${packageManager}`, + ); + // Init project with default package.json + const cmd1 = exec(`npm init -y`, { cwd: repoDir }, (err) => { + if (err) { + console.error(err); + // Error at first step + process.exit(1); + } + }); + await childProcessPromise(cmd1); + + // Add wrapper as dependency + const cmd2 = exec( + `${packageManager} add ${wrapperDir}`, + { cwd: repoDir }, + (err) => { + if (err) { + console.error(err); + // Error at second step + process.exit(2); + } + }, + ); + await childProcessPromise(cmd2); + + // Check that the binaries exists + if (!checkBinaries(repoDir)) process.exit(3); +}; + +// Run all the tests for the chosen package manger +const run = async (packageManager) => { + const firstRepoDir = createTestDir(); + const secondRepoDir = createTestDir(); + + // Run first test + await firstTest(packageManager, firstRepoDir); + // Run second test + await secondTest(packageManager, secondRepoDir, firstRepoDir); +}; + +// Main, run tests for npm and yarn +const main = async () => { + await PACKAGE_MANAGERS.reduce( + (p, pkgMng) => p.then(() => run(pkgMng)), + Promise.resolve(), + ); +}; + +// Run script if called directly instead of import as module +if (require.main === module) { + main(); +} diff --git a/.github/workflows/test-install-nitro-node.yml b/.github/workflows/test-install-nitro-node.yml new file mode 100644 index 000000000..1dadc3d5d --- /dev/null +++ b/.github/workflows/test-install-nitro-node.yml @@ -0,0 +1,83 @@ +name: Test install nitro-node + +on: + #schedule: + # - cron: "0 20 * * *" # At 8 PM UTC, which is 3 AM UTC+7 + push: + branches: + - main + - feat/1635/decouple-nitro-inference-engine-into-a-library + #tags: ["v[0-9]+.[0-9]+.[0-9]+"] + paths: + - ".github/scripts/e2e-test-install-nitro-node.js" + - ".github/workflows/test-install-nitro-node.yml" + - "nitro-node" + pull_request: + types: [opened, synchronize, reopened] + paths: + - ".github/scripts/e2e-test-install-nitro-node.js" + - ".github/workflows/test-install-nitro-node.yml" + - "nitro-node" + workflow_dispatch: + +jobs: + ubuntu-install: + runs-on: ubuntu-latest + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Run tests + id: test_install_nitro_node + run: | + cd .github + cd scripts + node e2e-test-install-nitro-node.js + + macOS-install: + runs-on: macos-latest + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Run tests + id: test_install_nitro_node + run: | + cd .github + cd scripts + node e2e-test-install-nitro-node.js + + windows-build: + runs-on: windows-latest + steps: + - name: Clone + + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Run tests + id: test_install_nitro_node + run: | + cd .github + cd scripts + node e2e-test-install-nitro-node.js From f7ca9c879eece3056c76fe800c5fdedf74163a11 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Fri, 26 Jan 2024 00:19:47 +0700 Subject: [PATCH 16/49] chore(nitro-node/ci): allow install from git repo - Add prebuilt dist directory to git - Bundle dependencies of download-nitro script - Postinstall now only possible if install from the built package - For downloading nitro when project is not yet built, run with ts-node --- .../scripts/e2e-test-install-nitro-node.js | 96 +- .github/workflows/test-install-nitro-node.yml | 4 +- nitro-node/.gitignore | 2 +- nitro-node/Makefile | 7 +- nitro-node/dist/download-nitro.cjs.js | 86 + nitro-node/dist/execute.js | 66 + nitro-node/dist/index.cjs.js | 4209 +++++++++++++++++ nitro-node/dist/index.js | 448 ++ nitro-node/dist/nvidia.js | 147 + nitro-node/dist/types/execute.d.ts | 9 + nitro-node/dist/types/index.d.ts | 70 + nitro-node/dist/types/nvidia.d.ts | 31 + .../dist/types/scripts/download-nitro.d.ts | 2 + nitro-node/dist/types/src/execute.d.ts | 9 + nitro-node/dist/types/src/index.d.ts | 70 + nitro-node/dist/types/src/nvidia.d.ts | 31 + nitro-node/download-nitro.js | 92 - nitro-node/package.json | 15 +- nitro-node/postinstall.js | 5 +- nitro-node/rollup.config.ts | 33 +- nitro-node/scripts/download-nitro.ts | 102 + nitro-node/tsconfig.json | 11 + 22 files changed, 5381 insertions(+), 164 deletions(-) create mode 100644 nitro-node/dist/download-nitro.cjs.js create mode 100644 nitro-node/dist/execute.js create mode 100644 nitro-node/dist/index.cjs.js create mode 100644 nitro-node/dist/index.js create mode 100644 nitro-node/dist/nvidia.js create mode 100644 nitro-node/dist/types/execute.d.ts create mode 100644 nitro-node/dist/types/index.d.ts create mode 100644 nitro-node/dist/types/nvidia.d.ts create mode 100644 nitro-node/dist/types/scripts/download-nitro.d.ts create mode 100644 nitro-node/dist/types/src/execute.d.ts create mode 100644 nitro-node/dist/types/src/index.d.ts create mode 100644 nitro-node/dist/types/src/nvidia.d.ts delete mode 100644 nitro-node/download-nitro.js create mode 100644 nitro-node/scripts/download-nitro.ts diff --git a/.github/scripts/e2e-test-install-nitro-node.js b/.github/scripts/e2e-test-install-nitro-node.js index 4352bdfca..4b53e5df6 100644 --- a/.github/scripts/e2e-test-install-nitro-node.js +++ b/.github/scripts/e2e-test-install-nitro-node.js @@ -1,14 +1,20 @@ const os = require("node:os"); const path = require("node:path"); const fs = require("node:fs"); -const { exec } = require("node:child_process"); +const { spawn } = require("node:child_process"); // Test on both npm and yarn const PACKAGE_MANAGERS = ["npm", "yarn"]; -// TODO: Path to the package to install. Change to '@janhq/nitro-node' after published -const NITRO_NODE_PKG = path.normalize( - path.join(__dirname, "..", "..", "nitro-node"), +const ADD_DEP_CMDS = { + // Need to copy dependency instead of linking so test logic can check the bin + npm: "install --install-links", + yarn: "add", +}; +// Path to the package to install +const NITRO_NODE_PKG = path.resolve( + path.normalize(path.join(__dirname, "..", "..", "nitro-node")), ); +// Prefixes of downloaded nitro bin subdirectories const BIN_DIR_PREFIXES = { darwin: "mac", win32: "win", @@ -19,13 +25,20 @@ const BIN_DIR_PREFIXES = { const checkBinaries = (repoDir) => { // FIXME: Check for unsupported platforms const binDirPrefix = BIN_DIR_PREFIXES[process.platform]; - const searchRoot = path.join(repoDir, "node_modules", "@janhq", "nitro-node"); + const searchRoot = path.join( + repoDir, + "node_modules", + "@janhq", + "nitro-node", + "bin", + ); // Get the dir and files that indicate successful download of binaries const matched = fs .readdirSync(searchRoot, { recursive: true }) .filter( (fname) => fname.startsWith(binDirPrefix) || fname.startsWith("nitro"), ); + console.log(`Downloaded bin paths:`, matched); // Must have both the directory for the platform and the binary return ( @@ -35,9 +48,9 @@ const checkBinaries = (repoDir) => { }; // Wrapper to wait for child process to finish -const childProcessPromise = (cmd) => +const childProcessPromise = (childProcess) => new Promise((resolve, reject) => { - cmd.on("exit", (exitCode) => { + childProcess.on("exit", (exitCode) => { const exitNum = Number(exitCode); if (0 == exitNum) { resolve(); @@ -49,40 +62,30 @@ const childProcessPromise = (cmd) => // Create a temporary directory for testing const createTestDir = () => - fs.mkdtempSync(path.join(os.tmpdir(), "dummy-project")); + fs.mkdtempSync(path.join(os.tmpdir(), "dummy-project-")); // First test, create empty project dir and add nitro-node as dependency const firstTest = async (packageManager, repoDir) => { console.log(`[First test @ ${repoDir}] install with ${packageManager}`); // Init project with default package.json - const cmd1 = exec(`npm init -y`, { cwd: repoDir }, (err) => { - if (err) { - console.error(err); - // Error at first step - process.exit(1); - } - }); - await childProcessPromise(cmd1); + const cmd1 = `npm init -y`; + console.log("🖥️ " + cmd1); + await childProcessPromise( + spawn(cmd1, [], { cwd: repoDir, shell: true, stdio: "inherit" }), + ); // Add nitro-node as dependency - const cmd2 = exec( - `${packageManager} add ${NITRO_NODE_PKG}`, - { cwd: repoDir }, - (err) => { - if (err) { - console.error(err); - // Error at second step - process.exit(2); - } - }, + const cmd2 = `${packageManager} ${ADD_DEP_CMDS[packageManager]} ${NITRO_NODE_PKG}`; + console.log("🖥️ " + cmd2); + await childProcessPromise( + spawn(cmd2, [], { cwd: repoDir, shell: true, stdio: "inherit" }), ); - await childProcessPromise(cmd2); // Check that the binaries exists if (!checkBinaries(repoDir)) process.exit(3); // Cleanup node_modules after success - fs.rmSync(path.join(repoDir, "node_modules"), { recursive: true }); + //fs.rmSync(path.join(repoDir, "node_modules"), { recursive: true }); }; // Second test, install the wrapper from another project dir @@ -91,28 +94,18 @@ const secondTest = async (packageManager, repoDir, wrapperDir) => { `[Second test @ ${repoDir}] install ${wrapperDir} with ${packageManager}`, ); // Init project with default package.json - const cmd1 = exec(`npm init -y`, { cwd: repoDir }, (err) => { - if (err) { - console.error(err); - // Error at first step - process.exit(1); - } - }); - await childProcessPromise(cmd1); + const cmd1 = `npm init -y`; + console.log("🖥️ " + cmd1); + await childProcessPromise( + spawn(cmd1, [], { cwd: repoDir, shell: true, stdio: "inherit" }), + ); // Add wrapper as dependency - const cmd2 = exec( - `${packageManager} add ${wrapperDir}`, - { cwd: repoDir }, - (err) => { - if (err) { - console.error(err); - // Error at second step - process.exit(2); - } - }, + const cmd2 = `${packageManager} ${ADD_DEP_CMDS[packageManager]} ${wrapperDir}`; + console.log("🖥️ " + cmd2); + await childProcessPromise( + spawn(cmd2, [], { cwd: repoDir, shell: true, stdio: "inherit" }), ); - await childProcessPromise(cmd2); // Check that the binaries exists if (!checkBinaries(repoDir)) process.exit(3); @@ -121,12 +114,17 @@ const secondTest = async (packageManager, repoDir, wrapperDir) => { // Run all the tests for the chosen package manger const run = async (packageManager) => { const firstRepoDir = createTestDir(); - const secondRepoDir = createTestDir(); // Run first test await firstTest(packageManager, firstRepoDir); + console.log("First test ran success"); + + // FIXME: Currently failed with npm due to wrong path being resolved. + //const secondRepoDir = createTestDir(); + // Run second test - await secondTest(packageManager, secondRepoDir, firstRepoDir); + //await secondTest(packageManager, secondRepoDir, firstRepoDir); + //console.log("Second test ran success"); }; // Main, run tests for npm and yarn diff --git a/.github/workflows/test-install-nitro-node.yml b/.github/workflows/test-install-nitro-node.yml index 1dadc3d5d..6ec9db7fd 100644 --- a/.github/workflows/test-install-nitro-node.yml +++ b/.github/workflows/test-install-nitro-node.yml @@ -11,13 +11,13 @@ on: paths: - ".github/scripts/e2e-test-install-nitro-node.js" - ".github/workflows/test-install-nitro-node.yml" - - "nitro-node" + - "nitro-node/**" pull_request: types: [opened, synchronize, reopened] paths: - ".github/scripts/e2e-test-install-nitro-node.js" - ".github/workflows/test-install-nitro-node.yml" - - "nitro-node" + - "nitro-node/**" workflow_dispatch: jobs: diff --git a/nitro-node/.gitignore b/nitro-node/.gitignore index 446285591..b671335b4 100644 --- a/nitro-node/.gitignore +++ b/nitro-node/.gitignore @@ -6,7 +6,7 @@ node_modules *.tgz yarn.lock -dist +dist/bin/ build .DS_Store package-lock.json diff --git a/nitro-node/Makefile b/nitro-node/Makefile index a673a2ac9..e58f9afa0 100644 --- a/nitro-node/Makefile +++ b/nitro-node/Makefile @@ -35,10 +35,9 @@ publish: build clean: ifeq ($(OS),Windows_NT) - powershell -Command "Remove-Item -Recurse -Force -Path *.tgz, .yarn, yarn.lock, package-lock.json, bin" - powershell -Command "Get-ChildItem -Path . -Include node_modules, dist -Recurse -Directory | Remove-Item -Recurse -Force" + powershell -Command "Remove-Item -Recurse -Force -Path *.tgz, .yarn, yarn.lock, package-lock.json, bin, dist" + powershell -Command "Get-ChildItem -Path . -Include node_modules -Recurse -Directory | Remove-Item -Recurse -Force" else - rm -rf *.tgz .yarn yarn.lock package-lock.json bin + rm -rf *.tgz .yarn yarn.lock package-lock.json bin dist find . -name "node_modules" -type d -prune -exec rm -rf '{}' + - find . -name "dist" -type d -exec rm -rf '{}' + endif diff --git a/nitro-node/dist/download-nitro.cjs.js b/nitro-node/dist/download-nitro.cjs.js new file mode 100644 index 000000000..cad272c3d --- /dev/null +++ b/nitro-node/dist/download-nitro.cjs.js @@ -0,0 +1,86 @@ +'use strict'; + +var path = require('node:path'); +var download = require('download'); + +function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; } + +var path__default = /*#__PURE__*/_interopDefaultLegacy(path); +var download__default = /*#__PURE__*/_interopDefaultLegacy(download); + +// Define nitro version to download in env variable +var NITRO_VERSION = process.env.NITRO_VERSION || "0.2.11"; +// The platform OS to download nitro for +var PLATFORM = process.env.npm_config_platform || process.platform; +// The platform architecture +//const ARCH = process.env.npm_config_arch || process.arch; +var linuxVariants = { + "linux-amd64": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "linux-cpu")), + "linux-amd64-cuda-12-0": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "linux-cuda-12-0")), + "linux-amd64-cuda-11-7": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "linux-cuda-11-7")), +}; +var darwinVariants = { + "mac-arm64": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "mac-arm64")), + "mac-amd64": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "mac-x64")), +}; +var win32Variants = { + "win-amd64-cuda-12-0": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "win-cuda-12-0")), + "win-amd64-cuda-11-7": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "win-cuda-11-7")), + "win-amd64": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "win-cpu")), +}; +// Mapping to installation variants +var variantMapping = { + darwin: darwinVariants, + linux: linuxVariants, + win32: win32Variants, +}; +if (!(PLATFORM in variantMapping)) { + throw Error("Invalid platform: ".concat(PLATFORM)); +} +// Get download config for this platform +var variantConfig = variantMapping[PLATFORM]; +// Generate download link for each tarball +var getTarUrl = function (version, suffix) { + return "https://github.com/janhq/nitro/releases/download/v".concat(version, "/nitro-").concat(version, "-").concat(suffix, ".tar.gz"); +}; +// Report download progress +var createProgressReporter = function (variant) { return function (stream) { + return stream + .on("downloadProgress", function (progress) { + // Print and update progress on a single line of terminal + process.stdout.write("\r\u001B[K[".concat(variant, "] ").concat(progress.transferred, "/").concat(progress.total, " ").concat(Math.floor(progress.percent * 100), "%...")); + }) + .on("end", function () { + // Jump to new line to log next message + console.log(); + console.log("[".concat(variant, "] Finished downloading!")); + }); +}; }; +// Download single binary +var downloadBinary = function (version, suffix, filePath) { + var tarUrl = getTarUrl(version, suffix); + console.log("Downloading ".concat(tarUrl, " to ").concat(filePath)); + var progressReporter = createProgressReporter(suffix); + return progressReporter(download__default["default"](tarUrl, filePath, { + strip: 1, + extract: true, + })); +}; +// Download the binaries +var downloadBinaries = function (version, config) { + return Object.entries(config).reduce(function (p, _a) { + var k = _a[0], v = _a[1]; + return p.then(function () { return downloadBinary(version, k, v); }); + }, Promise.resolve()); +}; +// Call the download function with version and config +var downloadNitro = function () { + downloadBinaries(NITRO_VERSION, variantConfig); +}; +// Run script if called directly instead of import as module +if (require.main === module) { + downloadNitro(); +} + +module.exports = downloadNitro; +//# sourceMappingURL=download-nitro.cjs.js.map diff --git a/nitro-node/dist/execute.js b/nitro-node/dist/execute.js new file mode 100644 index 000000000..098649f31 --- /dev/null +++ b/nitro-node/dist/execute.js @@ -0,0 +1,66 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.executableNitroFile = void 0; +var tslib_1 = require("tslib"); +var node_path_1 = tslib_1.__importDefault(require("node:path")); +/** + * Find which executable file to run based on the current platform. + * @returns The name of the executable file to run. + */ +var executableNitroFile = function (nvidiaSettings) { + var binaryFolder = node_path_1.default.join(__dirname, "..", "bin"); // Current directory by default + var cudaVisibleDevices = ""; + var binaryName = "nitro"; + /** + * The binary folder is different for each platform. + */ + if (process.platform === "win32") { + /** + * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 + */ + if (nvidiaSettings["run_mode"] === "cpu") { + binaryFolder = node_path_1.default.join(binaryFolder, "win-cpu"); + } + else { + if (nvidiaSettings["cuda"].version === "12") { + binaryFolder = node_path_1.default.join(binaryFolder, "win-cuda-12-0"); + } + else { + binaryFolder = node_path_1.default.join(binaryFolder, "win-cuda-11-7"); + } + cudaVisibleDevices = nvidiaSettings["gpu_highest_vram"]; + } + binaryName = "nitro.exe"; + } + else if (process.platform === "darwin") { + /** + * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) + */ + if (process.arch === "arm64") { + binaryFolder = node_path_1.default.join(binaryFolder, "mac-arm64"); + } + else { + binaryFolder = node_path_1.default.join(binaryFolder, "mac-x64"); + } + } + else { + if (nvidiaSettings["run_mode"] === "cpu") { + binaryFolder = node_path_1.default.join(binaryFolder, "linux-cpu"); + } + else { + if (nvidiaSettings["cuda"].version === "12") { + binaryFolder = node_path_1.default.join(binaryFolder, "linux-cuda-12-0"); + } + else { + binaryFolder = node_path_1.default.join(binaryFolder, "linux-cuda-11-7"); + } + cudaVisibleDevices = nvidiaSettings["gpu_highest_vram"]; + } + } + return { + executablePath: node_path_1.default.join(binaryFolder, binaryName), + cudaVisibleDevices: cudaVisibleDevices, + }; +}; +exports.executableNitroFile = executableNitroFile; +//# sourceMappingURL=execute.js.map \ No newline at end of file diff --git a/nitro-node/dist/index.cjs.js b/nitro-node/dist/index.cjs.js new file mode 100644 index 000000000..17273aa89 --- /dev/null +++ b/nitro-node/dist/index.cjs.js @@ -0,0 +1,4209 @@ +'use strict'; + +var os = require('node:os'); +var fs = require('node:fs'); +var path = require('node:path'); +var node_child_process = require('node:child_process'); +var require$$1$2 = require('net'); +var require$$1$1 = require('util'); +var require$$1 = require('tty'); +var require$$0 = require('os'); +var require$$1$3 = require('child_process'); + +function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; } + +var os__default = /*#__PURE__*/_interopDefaultLegacy(os); +var fs__default = /*#__PURE__*/_interopDefaultLegacy(fs); +var path__default = /*#__PURE__*/_interopDefaultLegacy(path); +var require$$1__default$2 = /*#__PURE__*/_interopDefaultLegacy(require$$1$2); +var require$$1__default$1 = /*#__PURE__*/_interopDefaultLegacy(require$$1$1); +var require$$1__default = /*#__PURE__*/_interopDefaultLegacy(require$$1); +var require$$0__default = /*#__PURE__*/_interopDefaultLegacy(require$$0); +var require$$1__default$3 = /*#__PURE__*/_interopDefaultLegacy(require$$1$3); + +/****************************************************************************** +Copyright (c) Microsoft Corporation. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +***************************************************************************** */ + +var __assign = function() { + __assign = Object.assign || function __assign(t) { + for (var s, i = 1, n = arguments.length; i < n; i++) { + s = arguments[i]; + for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p]; + } + return t; + }; + return __assign.apply(this, arguments); +}; + +function __awaiter(thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +} + +function __generator(thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +} + +typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) { + var e = new Error(message); + return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e; +}; + +function getDefaultExportFromCjs (x) { + return x && x.__esModule && Object.prototype.hasOwnProperty.call(x, 'default') ? x['default'] : x; +} + +var tcpPortUsed = {}; + +var is2 = {}; + +var deepIs = {exports: {}}; + +var pSlice = Array.prototype.slice; +var Object_keys = typeof Object.keys === 'function' + ? Object.keys + : function (obj) { + var keys = []; + for (var key in obj) keys.push(key); + return keys; + } +; + +var deepEqual = deepIs.exports = function (actual, expected) { + // enforce Object.is +0 !== -0 + if (actual === 0 && expected === 0) { + return areZerosEqual(actual, expected); + + // 7.1. All identical values are equivalent, as determined by ===. + } else if (actual === expected) { + return true; + + } else if (actual instanceof Date && expected instanceof Date) { + return actual.getTime() === expected.getTime(); + + } else if (isNumberNaN(actual)) { + return isNumberNaN(expected); + + // 7.3. Other pairs that do not both pass typeof value == 'object', + // equivalence is determined by ==. + } else if (typeof actual != 'object' && typeof expected != 'object') { + return actual == expected; + + // 7.4. For all other Object pairs, including Array objects, equivalence is + // determined by having the same number of owned properties (as verified + // with Object.prototype.hasOwnProperty.call), the same set of keys + // (although not necessarily the same order), equivalent values for every + // corresponding key, and an identical 'prototype' property. Note: this + // accounts for both named and indexed properties on Arrays. + } else { + return objEquiv(actual, expected); + } +}; + +function isUndefinedOrNull(value) { + return value === null || value === undefined; +} + +function isArguments(object) { + return Object.prototype.toString.call(object) == '[object Arguments]'; +} + +function isNumberNaN(value) { + // NaN === NaN -> false + return typeof value == 'number' && value !== value; +} + +function areZerosEqual(zeroA, zeroB) { + // (1 / +0|0) -> Infinity, but (1 / -0) -> -Infinity and (Infinity !== -Infinity) + return (1 / zeroA) === (1 / zeroB); +} + +function objEquiv(a, b) { + if (isUndefinedOrNull(a) || isUndefinedOrNull(b)) + return false; + + // an identical 'prototype' property. + if (a.prototype !== b.prototype) return false; + //~~~I've managed to break Object.keys through screwy arguments passing. + // Converting to array solves the problem. + if (isArguments(a)) { + if (!isArguments(b)) { + return false; + } + a = pSlice.call(a); + b = pSlice.call(b); + return deepEqual(a, b); + } + try { + var ka = Object_keys(a), + kb = Object_keys(b), + key, i; + } catch (e) {//happens when one is a string literal and the other isn't + return false; + } + // having the same number of owned properties (keys incorporates + // hasOwnProperty) + if (ka.length != kb.length) + return false; + //the same set of keys (although not necessarily the same order), + ka.sort(); + kb.sort(); + //~~~cheap key test + for (i = ka.length - 1; i >= 0; i--) { + if (ka[i] != kb[i]) + return false; + } + //equivalent values for every corresponding key, and + //~~~possibly expensive deep test + for (i = ka.length - 1; i >= 0; i--) { + key = ka[i]; + if (!deepEqual(a[key], b[key])) return false; + } + return true; +} + +var deepIsExports = deepIs.exports; + +const word = '[a-fA-F\\d:]'; +const b = options => options && options.includeBoundaries ? + `(?:(?<=\\s|^)(?=${word})|(?<=${word})(?=\\s|$))` : + ''; + +const v4 = '(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)(?:\\.(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)){3}'; + +const v6seg = '[a-fA-F\\d]{1,4}'; +const v6 = ` +(?: +(?:${v6seg}:){7}(?:${v6seg}|:)| // 1:2:3:4:5:6:7:: 1:2:3:4:5:6:7:8 +(?:${v6seg}:){6}(?:${v4}|:${v6seg}|:)| // 1:2:3:4:5:6:: 1:2:3:4:5:6::8 1:2:3:4:5:6::8 1:2:3:4:5:6::1.2.3.4 +(?:${v6seg}:){5}(?::${v4}|(?::${v6seg}){1,2}|:)| // 1:2:3:4:5:: 1:2:3:4:5::7:8 1:2:3:4:5::8 1:2:3:4:5::7:1.2.3.4 +(?:${v6seg}:){4}(?:(?::${v6seg}){0,1}:${v4}|(?::${v6seg}){1,3}|:)| // 1:2:3:4:: 1:2:3:4::6:7:8 1:2:3:4::8 1:2:3:4::6:7:1.2.3.4 +(?:${v6seg}:){3}(?:(?::${v6seg}){0,2}:${v4}|(?::${v6seg}){1,4}|:)| // 1:2:3:: 1:2:3::5:6:7:8 1:2:3::8 1:2:3::5:6:7:1.2.3.4 +(?:${v6seg}:){2}(?:(?::${v6seg}){0,3}:${v4}|(?::${v6seg}){1,5}|:)| // 1:2:: 1:2::4:5:6:7:8 1:2::8 1:2::4:5:6:7:1.2.3.4 +(?:${v6seg}:){1}(?:(?::${v6seg}){0,4}:${v4}|(?::${v6seg}){1,6}|:)| // 1:: 1::3:4:5:6:7:8 1::8 1::3:4:5:6:7:1.2.3.4 +(?::(?:(?::${v6seg}){0,5}:${v4}|(?::${v6seg}){1,7}|:)) // ::2:3:4:5:6:7:8 ::2:3:4:5:6:7:8 ::8 ::1.2.3.4 +)(?:%[0-9a-zA-Z]{1,})? // %eth0 %1 +`.replace(/\s*\/\/.*$/gm, '').replace(/\n/g, '').trim(); + +// Pre-compile only the exact regexes because adding a global flag make regexes stateful +const v46Exact = new RegExp(`(?:^${v4}$)|(?:^${v6}$)`); +const v4exact = new RegExp(`^${v4}$`); +const v6exact = new RegExp(`^${v6}$`); + +const ip = options => options && options.exact ? + v46Exact : + new RegExp(`(?:${b(options)}${v4}${b(options)})|(?:${b(options)}${v6}${b(options)})`, 'g'); + +ip.v4 = options => options && options.exact ? v4exact : new RegExp(`${b(options)}${v4}${b(options)}`, 'g'); +ip.v6 = options => options && options.exact ? v6exact : new RegExp(`${b(options)}${v6}${b(options)}`, 'g'); + +var ipRegex = ip; + +var name = "is2"; +var version = "2.0.9"; +var description = "A type checking library where each exported function returns either true or false and does not throw. Also added tests."; +var license = "MIT"; +var tags = [ + "type", + "check", + "checker", + "checking", + "utilities", + "network", + "networking", + "credit", + "card", + "validation" +]; +var keywords = [ + "type", + "check", + "checker", + "checking", + "utilities", + "network", + "networking", + "credit", + "card", + "validation" +]; +var author = "Enrico Marino "; +var maintainers = "Edmond Meinfelder , Chris Oyler "; +var homepage = "http://github.com/stdarg/is2"; +var repository = { + type: "git", + url: "git@github.com:stdarg/is2.git" +}; +var bugs = { + url: "http://github.com/stdarg/is/issues" +}; +var main = "./index.js"; +var scripts = { + test: "./node_modules/.bin/mocha -C --reporter list tests.js" +}; +var engines = { + node: ">=v0.10.0" +}; +var dependencies = { + "deep-is": "^0.1.3", + "ip-regex": "^4.1.0", + "is-url": "^1.2.4" +}; +var devDependencies = { + mocha: "6.2.3", + mongodb: "3.2.4" +}; +var require$$2 = { + name: name, + version: version, + description: description, + license: license, + tags: tags, + keywords: keywords, + author: author, + maintainers: maintainers, + homepage: homepage, + repository: repository, + bugs: bugs, + main: main, + scripts: scripts, + engines: engines, + dependencies: dependencies, + devDependencies: devDependencies +}; + +var isUrl_1; +var hasRequiredIsUrl; + +function requireIsUrl () { + if (hasRequiredIsUrl) return isUrl_1; + hasRequiredIsUrl = 1; + /** + * Expose `isUrl`. + */ + + isUrl_1 = isUrl; + + /** + * RegExps. + * A URL must match #1 and then at least one of #2/#3. + * Use two levels of REs to avoid REDOS. + */ + + var protocolAndDomainRE = /^(?:\w+:)?\/\/(\S+)$/; + + var localhostDomainRE = /^localhost[\:?\d]*(?:[^\:?\d]\S*)?$/; + var nonLocalhostDomainRE = /^[^\s\.]+\.\S{2,}$/; + + /** + * Loosely validate a URL `string`. + * + * @param {String} string + * @return {Boolean} + */ + + function isUrl(string){ + if (typeof string !== 'string') { + return false; + } + + var match = string.match(protocolAndDomainRE); + if (!match) { + return false; + } + + var everythingAfterProtocol = match[1]; + if (!everythingAfterProtocol) { + return false; + } + + if (localhostDomainRE.test(everythingAfterProtocol) || + nonLocalhostDomainRE.test(everythingAfterProtocol)) { + return true; + } + + return false; + } + return isUrl_1; +} + +/** + * @fileOverview + * is2 derived from is by Enrico Marino, adapted for Node.js. + * Slightly modified by Edmond Meinfelder + * + * is + * the definitive JavaScript type testing library + * Copyright(c) 2013,2014 Edmond Meinfelder + * Copyright(c) 2011 Enrico Marino + * MIT license + */ + +(function (exports) { + const owns = {}.hasOwnProperty; + const toString = {}.toString; + const is = exports; + const deepIs = deepIsExports; + const ipRegEx = ipRegex; + is.version = require$$2.version; + + //////////////////////////////////////////////////////////////////////////////// + // Environment + + /** + * Tests if is is running under a browser. + * @return {Boolean} true if the environment has process, process.version and process.versions. + */ + is.browser = function() { + return (!is.node() && typeof window !== 'undefined' && toString.call(window) === '[object global]'); + }; + + /** + * Test if 'value' is defined. + * Alias: def + * @param {Any} value The value to test. + * @return {Boolean} true if 'value' is defined, false otherwise. + */ + is.defined = function(value) { + return typeof value !== 'undefined'; + }; + is.def = is.defined; + + /** + * Tests if is is running under node.js + * @return {Boolean} true if the environment has process, process.version and process.versions. + */ + is.nodejs = function() { + return (process && process.hasOwnProperty('version') && + process.hasOwnProperty('versions')); + }; + is.node = is.nodejs; + + /** + * Test if 'value' is undefined. + * Aliases: undef, udef + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is undefined, false otherwise. + */ + is.undefined = function(value) { + return value === undefined; + }; + is.udef = is.undef = is.undefined; + + + //////////////////////////////////////////////////////////////////////////////// + // Types + + /** + * Test if 'value' is an array. + * Alias: ary, arry + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is an array, false otherwise. + */ + is.array = function(value) { + return '[object Array]' === toString.call(value); + }; + is.arr = is.ary = is.arry = is.array; + + /** + * Test if 'value' is an arraylike object (i.e. it has a length property with a valid value) + * Aliases: arraylike, arryLike, aryLike + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is an arguments object, false otherwise. + */ + is.arrayLike = function(value) { + if (is.nullOrUndef(value)) + return false; + return value !== undefined && + owns.call(value, 'length') && + isFinite(value.length); + }; + is.arrLike = is.arryLike = is.aryLike = is.arraylike = is.arrayLike; + + /** + * Test if 'value' is an arguments object. + * Alias: args + * @param {Any} value value to test + * @return {Boolean} true if 'value' is an arguments object, false otherwise + */ + is.arguments = function(value) { + return '[object Arguments]' === toString.call(value); + }; + is.args = is.arguments; + + /** + * Test if 'value' is a boolean. + * Alias: bool + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is a boolean, false otherwise. + */ + is.boolean = function(value) { + return '[object Boolean]' === toString.call(value); + }; + is.bool = is.boolean; + + /** + * Test if 'value' is an instance of Buffer. + * Aliases: instOf, instanceof + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is an instance of 'constructor'. + */ + is.buffer = function(value) { + return is.nodejs() && Buffer && Buffer.hasOwnProperty('isBuffer') && Buffer.isBuffer(value); + }; + is.buff = is.buf = is.buffer; + + /** + * Test if 'value' is a date. + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is a date, false otherwise. + */ + is.date = function(value) { + return '[object Date]' === toString.call(value); + }; + + /** + * Test if 'value' is an error object. + * Alias: err + * @param value value to test. + * @return {Boolean} true if 'value' is an error object, false otherwise. + */ + is.error = function(value) { + return '[object Error]' === toString.call(value); + }; + is.err = is.error; + + /** + * Test if 'value' is false. + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is false, false otherwise + */ + is.false = function(value) { + return value === false; + }; + + /** + * Test if 'value' is a function or async function. + * Alias: func + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is a function, false otherwise. + */ + is.function = function(value) { + return is.syncFunction(value) || is.asyncFunction(value) + }; + is.fun = is.func = is.function; + + /** + * Test if 'value' is an async function using `async () => {}` or `async function () {}`. + * Alias: func + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is a function, false otherwise. + */ + is.asyncFunction = function(value) { + return '[object AsyncFunction]' === toString.call(value); + }; + is.asyncFun = is.asyncFunc = is.asyncFunction; + + /** + * Test if 'value' is a synchronous function. + * Alias: syncFunc + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is a function, false otherwise. + */ + is.syncFunction = function (value) { + return '[object Function]' === toString.call(value); + }; + is.syncFun = is.syncFunc = is.syncFunction; + /** + * Test if 'value' is null. + * @param {Any} value to test. + * @return {Boolean} true if 'value' is null, false otherwise. + */ + is.null = function(value) { + return value === null; + }; + + /** + * Test is 'value' is either null or undefined. + * Alias: nullOrUndef + * @param {Any} value value to test. + * @return {Boolean} True if value is null or undefined, false otherwise. + */ + is.nullOrUndefined = function(value) { + return value === null || typeof value === 'undefined'; + }; + is.nullOrUndef = is.nullOrUndefined; + + /** + * Test if 'value' is a number. + * Alias: num + * @param {Any} value to test. + * @return {Boolean} true if 'value' is a number, false otherwise. + */ + is.number = function(value) { + return '[object Number]' === toString.call(value); + }; + is.num = is.number; + + /** + * Test if 'value' is an object. Note: Arrays, RegExps, Date, Error, etc all return false. + * Alias: obj + * @param {Any} value to test. + * @return {Boolean} true if 'value' is an object, false otherwise. + */ + is.object = function(value) { + return '[object Object]' === toString.call(value); + }; + is.obj = is.object; + + /** + * Test if 'value' is a regular expression. + * Alias: regexp + * @param {Any} value to test. + * @return {Boolean} true if 'value' is a regexp, false otherwise. + */ + is.regExp = function(value) { + return '[object RegExp]' === toString.call(value); + }; + is.re = is.regexp = is.regExp; + + /** + * Test if 'value' is a string. + * Alias: str + * @param {Any} value to test. + * @return {Boolean} true if 'value' is a string, false otherwise. + */ + is.string = function(value) { + return '[object String]' === toString.call(value); + }; + is.str = is.string; + + /** + * Test if 'value' is true. + * @param {Any} value to test. + * @return {Boolean} true if 'value' is true, false otherwise. + */ + is.true = function(value) { + return value === true; + }; + + /** + * Test if 'value' is a uuid (v1-v5) + * @param {Any} value to test. + * @return {Boolean} true if 'value is a valid RFC4122 UUID. Case non-specific. + */ + var uuidRegExp = new RegExp('[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab]'+ + '[0-9a-f]{3}-[0-9a-f]{12}', 'i'); + is.uuid = function(value) { + return uuidRegExp.test(value); + }; + + //////////////////////////////////////////////////////////////////////////////// + // Object Relationships + + /** + * Test if 'value' is equal to 'other'. Works for objects and arrays and will do deep comparisions, + * using recursion. + * Alias: eq + * @param {Any} value value. + * @param {Any} other value to compare with. + * @return {Boolean} true if 'value' is equal to 'other', false otherwise + */ + is.equal = function(value, other) { + var type = toString.call(value); + + if (typeof value !== typeof other) { + return false; + } + + if (type !== toString.call(other)) { + return false; + } + + if ('[object Object]' === type || '[object Array]' === type) { + return deepIs(value, other); + } else if ('[object Function]' === type) { + return value.prototype === other.prototype; + } else if ('[object Date]' === type) { + return value.getTime() === other.getTime(); + } + + return value === other; + }; + is.objEquals = is.eq = is.equal; + + /** + * JS Type definitions which cannot host values. + * @api private + */ + var NON_HOST_TYPES = { + 'boolean': 1, + 'number': 1, + 'string': 1, + 'undefined': 1 + }; + + /** + * Test if 'key' in host is an object. To be hosted means host[value] is an object. + * @param {Any} value The value to test. + * @param {Any} host Host that may contain value. + * @return {Boolean} true if 'value' is hosted by 'host', false otherwise. + */ + is.hosted = function(value, host) { + if (is.nullOrUndef(value)) + return false; + var type = typeof host[value]; + return type === 'object' ? !!host[value] : !NON_HOST_TYPES[type]; + }; + + /** + * Test if 'value' is an instance of 'constructor'. + * Aliases: instOf, instanceof + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is an instance of 'constructor'. + */ + is.instanceOf = function(value, constructor) { + if (is.nullOrUndef(value) || is.nullOrUndef(constructor)) + return false; + return (value instanceof constructor); + }; + is.instOf = is.instanceof = is.instanceOf; + + /** + * Test if 'value' is an instance type objType. + * Aliases: objInstOf, objectinstanceof, instOf, instanceOf + * @param {object} objInst an object to testfor type. + * @param {object} objType an object type to compare. + * @return {Boolean} true if 'value' is an object, false otherwise. + */ + is.objectInstanceOf = function(objInst, objType) { + try { + return '[object Object]' === toString.call(objInst) && (objInst instanceof objType); + } catch(err) { + return false; + } + }; + is.instOf = is.instanceOf = is.objInstOf = is.objectInstanceOf; + + /** + * Test if 'value' is a type of 'type'. + * Alias: a + * @param value value to test. + * @param {String} type The name of the type. + * @return {Boolean} true if 'value' is an arguments object, false otherwise. + */ + is.type = function(value, type) { + return typeof value === type; + }; + is.a = is.type; + + //////////////////////////////////////////////////////////////////////////////// + // Object State + + /** + * Test if 'value' is empty. To be empty means to be an array, object or string with nothing contained. + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is empty, false otherwise. + */ + is.empty = function(value) { + var type = toString.call(value); + + if ('[object Array]' === type || '[object Arguments]' === type) { + return value.length === 0; + } + + if ('[object Object]' === type) { + for (var key in value) if (owns.call(value, key)) return false; + return true; + } + + if ('[object String]' === type) { + return value === ''; + } + + return false; + }; + + /** + * Test if 'value' is an arguments object that is empty. + * Alias: args + * @param {Any} value value to test + * @return {Boolean} true if 'value' is an arguments object with no args, false otherwise + */ + is.emptyArguments = function(value) { + return '[object Arguments]' === toString.call(value) && value.length === 0; + }; + is.noArgs = is.emptyArgs = is.emptyArguments; + + /** + * Test if 'value' is an array containing no entries. + * Aliases: emptyArry, emptyAry + * @param {Any} value The value to test. + * @return {Boolean} true if 'value' is an array with no elemnets. + */ + is.emptyArray = function(value) { + return '[object Array]' === toString.call(value) && value.length === 0; + }; + is.emptyArry = is.emptyAry = is.emptyArray; + + /** + * Test if 'value' is an empty array(like) object. + * Aliases: arguents.empty, args.empty, ary.empty, arry.empty + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is an empty array(like), false otherwise. + */ + is.emptyArrayLike = function(value) { + return value.length === 0; + }; + is.emptyArrLike = is.emptyArrayLike; + + /** + * Test if 'value' is an empty string. + * Alias: emptyStr + * @param {Any} value to test. + * @return {Boolean} true if 'value' is am empty string, false otherwise. + */ + is.emptyString = function(value) { + return is.string(value) && value.length === 0; + }; + is.emptyStr = is.emptyString; + + /** + * Test if 'value' is an array containing at least 1 entry. + * Aliases: nonEmptyArry, nonEmptyAry + * @param {Any} value The value to test. + * @return {Boolean} true if 'value' is an array with at least 1 value, false otherwise. + */ + is.nonEmptyArray = function(value) { + return '[object Array]' === toString.call(value) && value.length > 0; + }; + is.nonEmptyArr = is.nonEmptyArry = is.nonEmptyAry = is.nonEmptyArray; + + /** + * Test if 'value' is an object with properties. Note: Arrays are objects. + * Alias: nonEmptyObj + * @param {Any} value to test. + * @return {Boolean} true if 'value' is an object, false otherwise. + */ + is.nonEmptyObject = function(value) { + return '[object Object]' === toString.call(value) && Object.keys(value).length > 0; + }; + is.nonEmptyObj = is.nonEmptyObject; + + /** + * Test if 'value' is an object with no properties. Note: Arrays are objects. + * Alias: nonEmptyObj + * @param {Any} value to test. + * @return {Boolean} true if 'value' is an object, false otherwise. + */ + is.emptyObject = function(value) { + return '[object Object]' === toString.call(value) && Object.keys(value).length === 0; + }; + is.emptyObj = is.emptyObject; + + /** + * Test if 'value' is a non-empty string. + * Alias: nonEmptyStr + * @param {Any} value to test. + * @return {Boolean} true if 'value' is a non-empty string, false otherwise. + */ + is.nonEmptyString = function(value) { + return is.string(value) && value.length > 0; + }; + is.nonEmptyStr = is.nonEmptyString; + + //////////////////////////////////////////////////////////////////////////////// + // Numeric Types within Number + + /** + * Test if 'value' is an even number. + * @param {Number} value to test. + * @return {Boolean} true if 'value' is an even number, false otherwise. + */ + is.even = function(value) { + return '[object Number]' === toString.call(value) && value % 2 === 0; + }; + + /** + * Test if 'value' is a decimal number. + * Aliases: decimalNumber, decNum + * @param {Any} value value to test. + * @return {Boolean} true if 'value' is a decimal number, false otherwise. + */ + is.decimal = function(value) { + return '[object Number]' === toString.call(value) && value % 1 !== 0; + }; + is.dec = is.decNum = is.decimal; + + /** + * Test if 'value' is an integer. + * Alias: integer + * @param {Any} value to test. + * @return {Boolean} true if 'value' is an integer, false otherwise. + */ + is.integer = function(value) { + return '[object Number]' === toString.call(value) && value % 1 === 0; + }; + is.int = is.integer; + + /** + * is.nan + * Test if `value` is not a number. + * + * @param {Mixed} value value to test + * @return {Boolean} true if `value` is not a number, false otherwise + * @api public + */ + is.notANumber = function(value) { + return !is.num(value) || value !== value; + }; + is.nan = is.notANum = is.notANumber; + + /** + * Test if 'value' is an odd number. + * @param {Number} value to test. + * @return {Boolean} true if 'value' is an odd number, false otherwise. + */ + is.odd = function(value) { + return !is.decimal(value) && '[object Number]' === toString.call(value) && value % 2 !== 0; + }; + is.oddNumber = is.oddNum = is.odd; + + //////////////////////////////////////////////////////////////////////////////// + // Numeric Type & State + + /** + * Test if 'value' is a positive number. + * Alias: positiveNum, posNum + * @param {Any} value to test. + * @return {Boolean} true if 'value' is a number, false otherwise. + */ + is.positiveNumber = function(value) { + return '[object Number]' === toString.call(value) && value > 0; + }; + is.pos = is.positive = is.posNum = is.positiveNum = is.positiveNumber; + + /** + * Test if 'value' is a negative number. + * Aliases: negNum, negativeNum + * @param {Any} value to test. + * @return {Boolean} true if 'value' is a number, false otherwise. + */ + is.negativeNumber = function(value) { + return '[object Number]' === toString.call(value) && value < 0; + }; + is.neg = is.negNum = is.negativeNum = is.negativeNumber; + + /** + * Test if 'value' is a negative integer. + * Aliases: negInt, negativeInteger + * @param {Any} value to test. + * @return {Boolean} true if 'value' is a negative integer, false otherwise. + */ + is.negativeInteger = function(value) { + return '[object Number]' === toString.call(value) && value % 1 === 0 && value < 0; + }; + is.negativeInt = is.negInt = is.negativeInteger; + + /** + * Test if 'value' is a positive integer. + * Alias: posInt + * @param {Any} value to test. + * @return {Boolean} true if 'value' is a positive integer, false otherwise. + */ + is.positiveInteger = function(value) { + return '[object Number]' === toString.call(value) && value % 1 === 0 && value > 0; + }; + is.posInt = is.positiveInt = is.positiveInteger; + + //////////////////////////////////////////////////////////////////////////////// + // Numeric Relationships + + /** + * Test if 'value' is divisible by 'n'. + * Alias: divisBy + * @param {Number} value value to test. + * @param {Number} n dividend. + * @return {Boolean} true if 'value' is divisible by 'n', false otherwise. + */ + is.divisibleBy = function(value, n) { + if (value === 0) + return false; + return '[object Number]' === toString.call(value) && + n !== 0 && + value % n === 0; + }; + is.divBy = is.divisBy = is.divisibleBy; + + /** + * Test if 'value' is greater than or equal to 'other'. + * Aliases: greaterOrEq, greaterOrEqual + * @param {Number} value value to test. + * @param {Number} other value to compare with. + * @return {Boolean} true, if value is greater than or equal to other, false otherwise. + */ + is.greaterOrEqualTo = function(value, other) { + return value >= other; + }; + is.greaterOrEqual = is.ge = is.greaterOrEqualTo; + + /** + * Test if 'value' is greater than 'other'. + * Aliases: greaterThan + * @param {Number} value value to test. + * @param {Number} other value to compare with. + * @return {Boolean} true, if value is greater than other, false otherwise. + */ + is.greaterThan = function(value, other) { + return value > other; + }; + is.gt = is.greaterThan; + + /** + * Test if 'value' is less than or equal to 'other'. + * Alias: lessThanOrEq, lessThanOrEqual + * @param {Number} value value to test + * @param {Number} other value to compare with + * @return {Boolean} true, if 'value' is less than or equal to 'other', false otherwise. + */ + is.lessThanOrEqualTo = function(value, other) { + return value <= other; + }; + is.lessThanOrEq = is.lessThanOrEqual = is.le = is.lessThanOrEqualTo; + + /** + * Test if 'value' is less than 'other'. + * Alias: lessThan + * @param {Number} value value to test + * @param {Number} other value to compare with + * @return {Boolean} true, if 'value' is less than 'other', false otherwise. + */ + is.lessThan = function(value, other) { + return value < other; + }; + is.lt = is.lessThan; + + /** + * Test if 'value' is greater than 'others' values. + * Alias: max + * @param {Number} value value to test. + * @param {Array} others values to compare with. + * @return {Boolean} true if 'value' is greater than 'others' values. + */ + is.maximum = function(value, others) { + if (!is.arrayLike(others) || !is.number(value)) + return false; + + var len = others.length; + while (--len > -1) { + if (value < others[len]) { + return false; + } + } + + return true; + }; + is.max = is.maximum; + + /** + * Test if 'value' is less than 'others' values. + * Alias: min + * @param {Number} value value to test. + * @param {Array} others values to compare with. + * @return {Boolean} true if 'value' is less than 'others' values. + */ + is.minimum = function(value, others) { + if (!is.arrayLike(others) || !is.number(value)) + return false; + + var len = others.length; + while (--len > -1) { + if (value > others[len]) { + return false; + } + } + + return true; + }; + is.min = is.minimum; + + /** + * Test if 'value' is within 'start' and 'finish'. + * Alias: withIn + * @param {Number} value value to test. + * @param {Number} start lower bound. + * @param {Number} finish upper bound. + * @return {Boolean} true if 'value' is is within 'start' and 'finish', false otherwise. + */ + is.within = function(value, start, finish) { + return value >= start && value <= finish; + }; + is.withIn = is.within; + + /** + * Test if 'value' is within 'precision' decimal places from 'comparitor'. + * Alias: closish, near. + * @param {Number} value value to test + * @param {Number} comparitor value to test 'value' against + * @param {Number} precision number of decimals to compare floating points, defaults to 2 + * @return {Boolean} true if 'value' is within 'precision' decimal places from 'comparitor', false otherwise. + */ + is.prettyClose = function(value, comparitor, precision) { + if (!is.number(value) || !is.number(comparitor)) return false; + if (is.defined(precision) && !is.posInt(precision)) return false; + if (is.undefined(precision)) precision = 2; + + return value.toFixed(precision) === comparitor.toFixed(precision); + }; + is.closish = is.near = is.prettyClose; + //////////////////////////////////////////////////////////////////////////////// + // Networking + + /** + * Test if a value is a valid DNS address. eg www.stdarg.com is true while + * 127.0.0.1 is false. + * @param {Any} value to test if a DNS address. + * @return {Boolean} true if a DNS address, false otherwise. + * DNS Address is made up of labels separated by '.' + * Each label must be between 1 and 63 characters long + * The entire hostname (including the delimiting dots) has a maximum of 255 characters. + * Hostname may not contain other characters, such as the underscore character (_) + * other DNS names may contain the underscore. + */ + is.dnsAddress = function(value) { + if (!is.nonEmptyStr(value)) return false; + if (value.length > 255) return false; + if (numbersLabel.test(value)) return false; + if (!dnsLabel.test(value)) return false; + return true; + //var names = value.split('.'); + //if (!is.array(names) || !names.length) return false; + //if (names[0].indexOf('_') > -1) return false; + //for (var i=0; i 15) return false; + var octets = value.split('.'); + if (!is.array(octets) || octets.length !== 4) return false; + for (var i=0; i 255) return false; + } + return true; + }; + is.ipv4 = is.ipv4Addr = is.ipv4Address; + + /** + * Test if a value is either an IPv6 numeric IP address. + * @param {Any} value to test if an ip address. + * @return {Boolean} true if an ip address, false otherwise. + */ + is.ipv6Address = function(value) { + if (!is.nonEmptyStr(value)) return false; + return ipRegEx.v6({extract: true}).test(value); + }; + is.ipv6 = is.ipv6Addr = is.ipv6Address; + + /** + * Test if a value is either an IPv4 or IPv6 numeric IP address. + * @param {Any} value to test if an ip address. + * @return {Boolean} true if an ip address, false otherwise. + */ + is.ipAddress = function(value) { + if (!is.nonEmptyStr(value)) return false; + return is.ipv4Address(value) || is.ipv6Address(value) + }; + is.ip = is.ipAddr = is.ipAddress; + + /** + * Test is a value is a valid ipv4, ipv6 or DNS name. + * Aliases: host, hostAddr, hostAddress. + * @param {Any} value to test if a host address. + * @return {Boolean} true if a host address, false otherwise. + */ + is.hostAddress = function(value) { + if (!is.nonEmptyStr(value)) return false; + return is.dns(value) || is.ipv4(value) || is.ipv6(value); + }; + is.host = is.hostIp = is.hostAddr = is.hostAddress; + + /** + * Test if a number is a valid TCP port + * @param {Any} value to test if its a valid TCP port + */ + is.port = function(value) { + if (!is.num(value) || is.negativeInt(value) || value > 65535) + return false; + return true; + }; + + /** + * Test if a number is a valid TCP port in the range 0-1023. + * Alias: is.sysPort. + * @param {Any} value to test if its a valid TCP port + */ + is.systemPort = function(value) { + if (is.port(value) && value < 1024) + return true; + return false; + }; + is.sysPort = is.systemPort; + + /** + * Test if a number is a valid TCP port in the range 1024-65535. + * @param {Any} value to test if its a valid TCP port + */ + is.userPort = function(value) { + if (is.port(value) && value > 1023) + return true; + return false; + }; + + /* + function sumDigits(num) { + var str = num.toString(); + var sum = 0; + for (var i = 0; i < str.length; i++) + sum += (str[i]-0); + return sum; + } + */ + + /** + * Test if a string is a credit card. + * From http://en.wikipedia.org/wiki/Luhn_algorithm + * @param {String} value to test if a credit card. + * @return true if the string is the correct format, false otherwise + */ + is.creditCardNumber = function(str) { + if (!is.str(str)) + return false; + + var ary = str.split(''); + var i, cnt; + // From the rightmost digit, which is the check digit, moving left, double + // the value of every second digit; + for (i=ary.length-1, cnt=1; i>-1; i--, cnt++) { + if (cnt%2 === 0) + ary[i] *= 2; + } + + str = ary.join(''); + var sum = 0; + // if the product of the previous doubling operation is greater than 9 + // (e.g., 7 * 2 = 14), then sum the digits of the products (e.g., 10: 1 + 0 + // = 1, 14: 1 + 4 = 5). We do the this by joining the array of numbers and + // add adding the int value of all the characters in the string. + for (i=0; i 19)) + return false; + + var prefix = Math.floor(str.slice(0,2)); + if (prefix !== 62 && prefix !== 88) + return false; + + // no validation for this card + return true; + }; + is.chinaUnion = is.chinaUnionPayCard = is.chinaUnionPayCardNumber; + + /** + * Test if card number is a Diner's Club Carte Blance card. + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.dinersClubCarteBlancheCardNumber = function(str) { + if (!is.str(str) || str.length !== 14) + return false; + + var prefix = Math.floor(str.slice(0,3)); + if (prefix < 300 || prefix > 305) + return false; + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + is.dinersClubCB = is.dinersClubCarteBlancheCard = + is.dinersClubCarteBlancheCardNumber; + + /** + * Test if card number is a Diner's Club International card. + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.dinersClubInternationalCardNumber = function(str) { + if (!is.str(str) || str.length !== 14) + return false; + var prefix = Math.floor(str.slice(0,3)); + var prefix2 = Math.floor(str.slice(0,2)); + + // 300-305, 309, 36, 38-39 + if ((prefix < 300 || prefix > 305) && prefix !== 309 && prefix2 !== 36 && + (prefix2 < 38 || prefix2 > 39)) { + return false; + } + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + is.dinersClubInt = is.dinersClubInternationalCard = + is.dinersClubInternationalCardNumber; + + /** + * Test if card number is a Diner's Club USA & CA card. + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.dinersClubUSACanadaCardNumber = function(str) { + if (!is.str(str) || str.length !== 16) + return false; + var prefix = Math.floor(str.slice(0,2)); + + if (prefix !== 54 && prefix !== 55) + return false; + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + is.dinersClub = is.dinersClubUSACanCard = is.dinersClubUSACanadaCardNumber; + + /** + * Test if card number is a Diner's Club USA/CA card. + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.discoverCardNumber = function(str) { + if (!is.str(str) || str.length !== 16) + return false; + + var prefix = Math.floor(str.slice(0,6)); + var prefix2 = Math.floor(str.slice(0,3)); + + if (str.slice(0,4) !== '6011' && (prefix < 622126 || prefix > 622925) && + (prefix2 < 644 || prefix2 > 649) && str.slice(0,2) !== '65') { + return false; + } + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + is.discover = is.discoverCard = is.discoverCardNumber; + + /** + * Test if card number is an InstaPayment card number + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.instaPaymentCardNumber = function(str) { + if (!is.str(str) || str.length !== 16) + return false; + + var prefix = Math.floor(str.slice(0,3)); + if (prefix < 637 || prefix > 639) + return false; + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + is.instaPayment = is.instaPaymentCardNumber; + + /** + * Test if card number is a JCB card number + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.jcbCardNumber = function(str) { + if (!is.str(str) || str.length !== 16) + return false; + + var prefix = Math.floor(str.slice(0,4)); + if (prefix < 3528 || prefix > 3589) + return false; + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + is.jcb = is.jcbCard = is.jcbCardNumber; + + /** + * Test if card number is a Laser card number + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.laserCardNumber = function(str) { + if (!is.str(str) || (str.length < 16 && str.length > 19)) + return false; + + var prefix = Math.floor(str.slice(0,4)); + var valid = [ 6304, 6706, 6771, 6709 ]; + if (valid.indexOf(prefix) === -1) + return false; + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + is.laser = is.laserCard = is.laserCardNumber; + + /** + * Test if card number is a Maestro card number + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.maestroCardNumber = function(str) { + if (!is.str(str) || str.length < 12 || str.length > 19) + return false; + + var prefix = str.slice(0,4); + var valid = [ '5018', '5020', '5038', '5612', '5893', '6304', '6759', + '6761', '6762', '6763', '0604', '6390' ]; + + if (valid.indexOf(prefix) === -1) + return false; + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + is.maestro = is.maestroCard = is.maestroCardNumber; + + /** + * Test if card number is a Dankort card number + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.dankortCardNumber = function(str) { + if (!is.str(str) || str.length !== 16) + return false; + + if (str.slice(0,4) !== '5019') + return false; + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + is.dankort = is.dankortCard = is.dankortCardNumber; + + /** + * Test if card number is a MasterCard card number + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.masterCardCardNumber = function(str) { + if (!is.str(str) || str.length !== 16) + return false; + + var prefix = Math.floor(str.slice(0,2)); + if (prefix < 50 || prefix > 55) + return false; + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + is.masterCard = is.masterCardCard = is.masterCardCardNumber; + + /** + * Test if card number is a Visa card number + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.visaCardNumber = function(str) { + if (!is.str(str) || (str.length !== 13 && str.length !== 16)) + return false; + + if ('4' !== str.slice(0,1)) + return false; + + if (!is.creditCardNumber(str)) + return false; + + return true; + }; + + is.visa = is.visaCard = is.visaCardNumber; + + /** + * Test if card number is a Visa card number + * @param {String} the credit card number string to test. + * @return true if the string is the correct format, false otherwise + */ + is.visaElectronCardNumber = function(str) { + if (!is.str(str) || str.length !== 16) + return false; + + var prefix = Math.floor(str.slice(0,4)); + var valid = [ 4026, 4405, 4508, 4844, 4913, 4917 ]; + if ('417500' !== str.slice(0,6) && valid.indexOf(prefix) === -1) + return false; + + if (!is.creditCardNumber(str)) + return false; + + return false; + }; + + is.visaElectron = is.visaElectronCard = is.visaElectronCardNumber; + + /** + * Test if the input is a valid MongoDB id. + * @param {String|Object} Either a mongodb object id or a string representation. + * @return true if the string is the correct format, false otherwise + * Thanks to Jason Denizac (https://github.com/jden) for pointing this out. + * https://github.com/jden/objectid/blob/master/index.js#L7-L10 + */ + var objIdPattern = /^[0-9a-fA-F]{24}$/; + is.mongoId = is.objectId = is.objId = function(id) { + return (Boolean(id) && !Array.isArray(id) && objIdPattern.test(String(id))); + }; + + /** + * Test is the first argument is structly equal to any of the subsequent args. + * @param Value to test against subsequent arguments. + * @return true if the first value matches any of subsequent values. + */ + is.matching = is.match = is.inArgs = function(val) { + if (arguments.length < 2) + return false; + var result = false; + for (var i=1; i 0) { + return parse(val); + } else if (type === 'number' && isFinite(val)) { + return options.long ? fmtLong(val) : fmtShort(val); + } + throw new Error( + 'val is not a non-empty string or a valid number. val=' + + JSON.stringify(val) + ); + }; + + /** + * Parse the given `str` and return milliseconds. + * + * @param {String} str + * @return {Number} + * @api private + */ + + function parse(str) { + str = String(str); + if (str.length > 100) { + return; + } + var match = /^(-?(?:\d+)?\.?\d+) *(milliseconds?|msecs?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d|weeks?|w|years?|yrs?|y)?$/i.exec( + str + ); + if (!match) { + return; + } + var n = parseFloat(match[1]); + var type = (match[2] || 'ms').toLowerCase(); + switch (type) { + case 'years': + case 'year': + case 'yrs': + case 'yr': + case 'y': + return n * y; + case 'weeks': + case 'week': + case 'w': + return n * w; + case 'days': + case 'day': + case 'd': + return n * d; + case 'hours': + case 'hour': + case 'hrs': + case 'hr': + case 'h': + return n * h; + case 'minutes': + case 'minute': + case 'mins': + case 'min': + case 'm': + return n * m; + case 'seconds': + case 'second': + case 'secs': + case 'sec': + case 's': + return n * s; + case 'milliseconds': + case 'millisecond': + case 'msecs': + case 'msec': + case 'ms': + return n; + default: + return undefined; + } + } + + /** + * Short format for `ms`. + * + * @param {Number} ms + * @return {String} + * @api private + */ + + function fmtShort(ms) { + var msAbs = Math.abs(ms); + if (msAbs >= d) { + return Math.round(ms / d) + 'd'; + } + if (msAbs >= h) { + return Math.round(ms / h) + 'h'; + } + if (msAbs >= m) { + return Math.round(ms / m) + 'm'; + } + if (msAbs >= s) { + return Math.round(ms / s) + 's'; + } + return ms + 'ms'; + } + + /** + * Long format for `ms`. + * + * @param {Number} ms + * @return {String} + * @api private + */ + + function fmtLong(ms) { + var msAbs = Math.abs(ms); + if (msAbs >= d) { + return plural(ms, msAbs, d, 'day'); + } + if (msAbs >= h) { + return plural(ms, msAbs, h, 'hour'); + } + if (msAbs >= m) { + return plural(ms, msAbs, m, 'minute'); + } + if (msAbs >= s) { + return plural(ms, msAbs, s, 'second'); + } + return ms + ' ms'; + } + + /** + * Pluralization helper. + */ + + function plural(ms, msAbs, n, name) { + var isPlural = msAbs >= n * 1.5; + return Math.round(ms / n) + ' ' + name + (isPlural ? 's' : ''); + } + return ms; +} + +var common; +var hasRequiredCommon; + +function requireCommon () { + if (hasRequiredCommon) return common; + hasRequiredCommon = 1; + /** + * This is the common logic for both the Node.js and web browser + * implementations of `debug()`. + */ + + function setup(env) { + createDebug.debug = createDebug; + createDebug.default = createDebug; + createDebug.coerce = coerce; + createDebug.disable = disable; + createDebug.enable = enable; + createDebug.enabled = enabled; + createDebug.humanize = requireMs(); + createDebug.destroy = destroy; + + Object.keys(env).forEach(key => { + createDebug[key] = env[key]; + }); + + /** + * The currently active debug mode names, and names to skip. + */ + + createDebug.names = []; + createDebug.skips = []; + + /** + * Map of special "%n" handling functions, for the debug "format" argument. + * + * Valid key names are a single, lower or upper-case letter, i.e. "n" and "N". + */ + createDebug.formatters = {}; + + /** + * Selects a color for a debug namespace + * @param {String} namespace The namespace string for the for the debug instance to be colored + * @return {Number|String} An ANSI color code for the given namespace + * @api private + */ + function selectColor(namespace) { + let hash = 0; + + for (let i = 0; i < namespace.length; i++) { + hash = ((hash << 5) - hash) + namespace.charCodeAt(i); + hash |= 0; // Convert to 32bit integer + } + + return createDebug.colors[Math.abs(hash) % createDebug.colors.length]; + } + createDebug.selectColor = selectColor; + + /** + * Create a debugger with the given `namespace`. + * + * @param {String} namespace + * @return {Function} + * @api public + */ + function createDebug(namespace) { + let prevTime; + let enableOverride = null; + + function debug(...args) { + // Disabled? + if (!debug.enabled) { + return; + } + + const self = debug; + + // Set `diff` timestamp + const curr = Number(new Date()); + const ms = curr - (prevTime || curr); + self.diff = ms; + self.prev = prevTime; + self.curr = curr; + prevTime = curr; + + args[0] = createDebug.coerce(args[0]); + + if (typeof args[0] !== 'string') { + // Anything else let's inspect with %O + args.unshift('%O'); + } + + // Apply any `formatters` transformations + let index = 0; + args[0] = args[0].replace(/%([a-zA-Z%])/g, (match, format) => { + // If we encounter an escaped % then don't increase the array index + if (match === '%%') { + return '%'; + } + index++; + const formatter = createDebug.formatters[format]; + if (typeof formatter === 'function') { + const val = args[index]; + match = formatter.call(self, val); + + // Now we need to remove `args[index]` since it's inlined in the `format` + args.splice(index, 1); + index--; + } + return match; + }); + + // Apply env-specific formatting (colors, etc.) + createDebug.formatArgs.call(self, args); + + const logFn = self.log || createDebug.log; + logFn.apply(self, args); + } + + debug.namespace = namespace; + debug.useColors = createDebug.useColors(); + debug.color = createDebug.selectColor(namespace); + debug.extend = extend; + debug.destroy = createDebug.destroy; // XXX Temporary. Will be removed in the next major release. + + Object.defineProperty(debug, 'enabled', { + enumerable: true, + configurable: false, + get: () => enableOverride === null ? createDebug.enabled(namespace) : enableOverride, + set: v => { + enableOverride = v; + } + }); + + // Env-specific initialization logic for debug instances + if (typeof createDebug.init === 'function') { + createDebug.init(debug); + } + + return debug; + } + + function extend(namespace, delimiter) { + const newDebug = createDebug(this.namespace + (typeof delimiter === 'undefined' ? ':' : delimiter) + namespace); + newDebug.log = this.log; + return newDebug; + } + + /** + * Enables a debug mode by namespaces. This can include modes + * separated by a colon and wildcards. + * + * @param {String} namespaces + * @api public + */ + function enable(namespaces) { + createDebug.save(namespaces); + + createDebug.names = []; + createDebug.skips = []; + + let i; + const split = (typeof namespaces === 'string' ? namespaces : '').split(/[\s,]+/); + const len = split.length; + + for (i = 0; i < len; i++) { + if (!split[i]) { + // ignore empty strings + continue; + } + + namespaces = split[i].replace(/\*/g, '.*?'); + + if (namespaces[0] === '-') { + createDebug.skips.push(new RegExp('^' + namespaces.substr(1) + '$')); + } else { + createDebug.names.push(new RegExp('^' + namespaces + '$')); + } + } + } + + /** + * Disable debug output. + * + * @return {String} namespaces + * @api public + */ + function disable() { + const namespaces = [ + ...createDebug.names.map(toNamespace), + ...createDebug.skips.map(toNamespace).map(namespace => '-' + namespace) + ].join(','); + createDebug.enable(''); + return namespaces; + } + + /** + * Returns true if the given mode name is enabled, false otherwise. + * + * @param {String} name + * @return {Boolean} + * @api public + */ + function enabled(name) { + if (name[name.length - 1] === '*') { + return true; + } + + let i; + let len; + + for (i = 0, len = createDebug.skips.length; i < len; i++) { + if (createDebug.skips[i].test(name)) { + return false; + } + } + + for (i = 0, len = createDebug.names.length; i < len; i++) { + if (createDebug.names[i].test(name)) { + return true; + } + } + + return false; + } + + /** + * Convert regexp to namespace + * + * @param {RegExp} regxep + * @return {String} namespace + * @api private + */ + function toNamespace(regexp) { + return regexp.toString() + .substring(2, regexp.toString().length - 2) + .replace(/\.\*\?$/, '*'); + } + + /** + * Coerce `val`. + * + * @param {Mixed} val + * @return {Mixed} + * @api private + */ + function coerce(val) { + if (val instanceof Error) { + return val.stack || val.message; + } + return val; + } + + /** + * XXX DO NOT USE. This is a temporary stub function. + * XXX It WILL be removed in the next major release. + */ + function destroy() { + console.warn('Instance method `debug.destroy()` is deprecated and no longer does anything. It will be removed in the next major version of `debug`.'); + } + + createDebug.enable(createDebug.load()); + + return createDebug; + } + + common = setup; + return common; +} + +/* eslint-env browser */ +browser.exports; + +var hasRequiredBrowser; + +function requireBrowser () { + if (hasRequiredBrowser) return browser.exports; + hasRequiredBrowser = 1; + (function (module, exports) { + /** + * This is the web browser implementation of `debug()`. + */ + + exports.formatArgs = formatArgs; + exports.save = save; + exports.load = load; + exports.useColors = useColors; + exports.storage = localstorage(); + exports.destroy = (() => { + let warned = false; + + return () => { + if (!warned) { + warned = true; + console.warn('Instance method `debug.destroy()` is deprecated and no longer does anything. It will be removed in the next major version of `debug`.'); + } + }; + })(); + + /** + * Colors. + */ + + exports.colors = [ + '#0000CC', + '#0000FF', + '#0033CC', + '#0033FF', + '#0066CC', + '#0066FF', + '#0099CC', + '#0099FF', + '#00CC00', + '#00CC33', + '#00CC66', + '#00CC99', + '#00CCCC', + '#00CCFF', + '#3300CC', + '#3300FF', + '#3333CC', + '#3333FF', + '#3366CC', + '#3366FF', + '#3399CC', + '#3399FF', + '#33CC00', + '#33CC33', + '#33CC66', + '#33CC99', + '#33CCCC', + '#33CCFF', + '#6600CC', + '#6600FF', + '#6633CC', + '#6633FF', + '#66CC00', + '#66CC33', + '#9900CC', + '#9900FF', + '#9933CC', + '#9933FF', + '#99CC00', + '#99CC33', + '#CC0000', + '#CC0033', + '#CC0066', + '#CC0099', + '#CC00CC', + '#CC00FF', + '#CC3300', + '#CC3333', + '#CC3366', + '#CC3399', + '#CC33CC', + '#CC33FF', + '#CC6600', + '#CC6633', + '#CC9900', + '#CC9933', + '#CCCC00', + '#CCCC33', + '#FF0000', + '#FF0033', + '#FF0066', + '#FF0099', + '#FF00CC', + '#FF00FF', + '#FF3300', + '#FF3333', + '#FF3366', + '#FF3399', + '#FF33CC', + '#FF33FF', + '#FF6600', + '#FF6633', + '#FF9900', + '#FF9933', + '#FFCC00', + '#FFCC33' + ]; + + /** + * Currently only WebKit-based Web Inspectors, Firefox >= v31, + * and the Firebug extension (any Firefox version) are known + * to support "%c" CSS customizations. + * + * TODO: add a `localStorage` variable to explicitly enable/disable colors + */ + + // eslint-disable-next-line complexity + function useColors() { + // NB: In an Electron preload script, document will be defined but not fully + // initialized. Since we know we're in Chrome, we'll just detect this case + // explicitly + if (typeof window !== 'undefined' && window.process && (window.process.type === 'renderer' || window.process.__nwjs)) { + return true; + } + + // Internet Explorer and Edge do not support colors. + if (typeof navigator !== 'undefined' && navigator.userAgent && navigator.userAgent.toLowerCase().match(/(edge|trident)\/(\d+)/)) { + return false; + } + + // Is webkit? http://stackoverflow.com/a/16459606/376773 + // document is undefined in react-native: https://github.com/facebook/react-native/pull/1632 + return (typeof document !== 'undefined' && document.documentElement && document.documentElement.style && document.documentElement.style.WebkitAppearance) || + // Is firebug? http://stackoverflow.com/a/398120/376773 + (typeof window !== 'undefined' && window.console && (window.console.firebug || (window.console.exception && window.console.table))) || + // Is firefox >= v31? + // https://developer.mozilla.org/en-US/docs/Tools/Web_Console#Styling_messages + (typeof navigator !== 'undefined' && navigator.userAgent && navigator.userAgent.toLowerCase().match(/firefox\/(\d+)/) && parseInt(RegExp.$1, 10) >= 31) || + // Double check webkit in userAgent just in case we are in a worker + (typeof navigator !== 'undefined' && navigator.userAgent && navigator.userAgent.toLowerCase().match(/applewebkit\/(\d+)/)); + } + + /** + * Colorize log arguments if enabled. + * + * @api public + */ + + function formatArgs(args) { + args[0] = (this.useColors ? '%c' : '') + + this.namespace + + (this.useColors ? ' %c' : ' ') + + args[0] + + (this.useColors ? '%c ' : ' ') + + '+' + module.exports.humanize(this.diff); + + if (!this.useColors) { + return; + } + + const c = 'color: ' + this.color; + args.splice(1, 0, c, 'color: inherit'); + + // The final "%c" is somewhat tricky, because there could be other + // arguments passed either before or after the %c, so we need to + // figure out the correct index to insert the CSS into + let index = 0; + let lastC = 0; + args[0].replace(/%[a-zA-Z%]/g, match => { + if (match === '%%') { + return; + } + index++; + if (match === '%c') { + // We only are interested in the *last* %c + // (the user may have provided their own) + lastC = index; + } + }); + + args.splice(lastC, 0, c); + } + + /** + * Invokes `console.debug()` when available. + * No-op when `console.debug` is not a "function". + * If `console.debug` is not available, falls back + * to `console.log`. + * + * @api public + */ + exports.log = console.debug || console.log || (() => {}); + + /** + * Save `namespaces`. + * + * @param {String} namespaces + * @api private + */ + function save(namespaces) { + try { + if (namespaces) { + exports.storage.setItem('debug', namespaces); + } else { + exports.storage.removeItem('debug'); + } + } catch (error) { + // Swallow + // XXX (@Qix-) should we be logging these? + } + } + + /** + * Load `namespaces`. + * + * @return {String} returns the previously persisted debug modes + * @api private + */ + function load() { + let r; + try { + r = exports.storage.getItem('debug'); + } catch (error) { + // Swallow + // XXX (@Qix-) should we be logging these? + } + + // If debug isn't set in LS, and we're in Electron, try to load $DEBUG + if (!r && typeof process !== 'undefined' && 'env' in process) { + r = process.env.DEBUG; + } + + return r; + } + + /** + * Localstorage attempts to return the localstorage. + * + * This is necessary because safari throws + * when a user disables cookies/localstorage + * and you attempt to access it. + * + * @return {LocalStorage} + * @api private + */ + + function localstorage() { + try { + // TVMLKit (Apple TV JS Runtime) does not have a window object, just localStorage in the global context + // The Browser also has localStorage in the global context. + return localStorage; + } catch (error) { + // Swallow + // XXX (@Qix-) should we be logging these? + } + } + + module.exports = requireCommon()(exports); + + const {formatters} = module.exports; + + /** + * Map %j to `JSON.stringify()`, since no Web Inspectors do that by default. + */ + + formatters.j = function (v) { + try { + return JSON.stringify(v); + } catch (error) { + return '[UnexpectedJSONParseError]: ' + error.message; + } + }; + } (browser, browser.exports)); + return browser.exports; +} + +var node = {exports: {}}; + +var hasFlag; +var hasRequiredHasFlag; + +function requireHasFlag () { + if (hasRequiredHasFlag) return hasFlag; + hasRequiredHasFlag = 1; + + hasFlag = (flag, argv = process.argv) => { + const prefix = flag.startsWith('-') ? '' : (flag.length === 1 ? '-' : '--'); + const position = argv.indexOf(prefix + flag); + const terminatorPosition = argv.indexOf('--'); + return position !== -1 && (terminatorPosition === -1 || position < terminatorPosition); + }; + return hasFlag; +} + +var supportsColor_1; +var hasRequiredSupportsColor; + +function requireSupportsColor () { + if (hasRequiredSupportsColor) return supportsColor_1; + hasRequiredSupportsColor = 1; + const os = require$$0__default["default"]; + const tty = require$$1__default["default"]; + const hasFlag = requireHasFlag(); + + const {env} = process; + + let forceColor; + if (hasFlag('no-color') || + hasFlag('no-colors') || + hasFlag('color=false') || + hasFlag('color=never')) { + forceColor = 0; + } else if (hasFlag('color') || + hasFlag('colors') || + hasFlag('color=true') || + hasFlag('color=always')) { + forceColor = 1; + } + + if ('FORCE_COLOR' in env) { + if (env.FORCE_COLOR === 'true') { + forceColor = 1; + } else if (env.FORCE_COLOR === 'false') { + forceColor = 0; + } else { + forceColor = env.FORCE_COLOR.length === 0 ? 1 : Math.min(parseInt(env.FORCE_COLOR, 10), 3); + } + } + + function translateLevel(level) { + if (level === 0) { + return false; + } + + return { + level, + hasBasic: true, + has256: level >= 2, + has16m: level >= 3 + }; + } + + function supportsColor(haveStream, streamIsTTY) { + if (forceColor === 0) { + return 0; + } + + if (hasFlag('color=16m') || + hasFlag('color=full') || + hasFlag('color=truecolor')) { + return 3; + } + + if (hasFlag('color=256')) { + return 2; + } + + if (haveStream && !streamIsTTY && forceColor === undefined) { + return 0; + } + + const min = forceColor || 0; + + if (env.TERM === 'dumb') { + return min; + } + + if (process.platform === 'win32') { + // Windows 10 build 10586 is the first Windows release that supports 256 colors. + // Windows 10 build 14931 is the first release that supports 16m/TrueColor. + const osRelease = os.release().split('.'); + if ( + Number(osRelease[0]) >= 10 && + Number(osRelease[2]) >= 10586 + ) { + return Number(osRelease[2]) >= 14931 ? 3 : 2; + } + + return 1; + } + + if ('CI' in env) { + if (['TRAVIS', 'CIRCLECI', 'APPVEYOR', 'GITLAB_CI', 'GITHUB_ACTIONS', 'BUILDKITE'].some(sign => sign in env) || env.CI_NAME === 'codeship') { + return 1; + } + + return min; + } + + if ('TEAMCITY_VERSION' in env) { + return /^(9\.(0*[1-9]\d*)\.|\d{2,}\.)/.test(env.TEAMCITY_VERSION) ? 1 : 0; + } + + if (env.COLORTERM === 'truecolor') { + return 3; + } + + if ('TERM_PROGRAM' in env) { + const version = parseInt((env.TERM_PROGRAM_VERSION || '').split('.')[0], 10); + + switch (env.TERM_PROGRAM) { + case 'iTerm.app': + return version >= 3 ? 3 : 2; + case 'Apple_Terminal': + return 2; + // No default + } + } + + if (/-256(color)?$/i.test(env.TERM)) { + return 2; + } + + if (/^screen|^xterm|^vt100|^vt220|^rxvt|color|ansi|cygwin|linux/i.test(env.TERM)) { + return 1; + } + + if ('COLORTERM' in env) { + return 1; + } + + return min; + } + + function getSupportLevel(stream) { + const level = supportsColor(stream, stream && stream.isTTY); + return translateLevel(level); + } + + supportsColor_1 = { + supportsColor: getSupportLevel, + stdout: translateLevel(supportsColor(true, tty.isatty(1))), + stderr: translateLevel(supportsColor(true, tty.isatty(2))) + }; + return supportsColor_1; +} + +/** + * Module dependencies. + */ +node.exports; + +var hasRequiredNode; + +function requireNode () { + if (hasRequiredNode) return node.exports; + hasRequiredNode = 1; + (function (module, exports) { + const tty = require$$1__default["default"]; + const util = require$$1__default$1["default"]; + + /** + * This is the Node.js implementation of `debug()`. + */ + + exports.init = init; + exports.log = log; + exports.formatArgs = formatArgs; + exports.save = save; + exports.load = load; + exports.useColors = useColors; + exports.destroy = util.deprecate( + () => {}, + 'Instance method `debug.destroy()` is deprecated and no longer does anything. It will be removed in the next major version of `debug`.' + ); + + /** + * Colors. + */ + + exports.colors = [6, 2, 3, 4, 5, 1]; + + try { + // Optional dependency (as in, doesn't need to be installed, NOT like optionalDependencies in package.json) + // eslint-disable-next-line import/no-extraneous-dependencies + const supportsColor = requireSupportsColor(); + + if (supportsColor && (supportsColor.stderr || supportsColor).level >= 2) { + exports.colors = [ + 20, + 21, + 26, + 27, + 32, + 33, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 56, + 57, + 62, + 63, + 68, + 69, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 92, + 93, + 98, + 99, + 112, + 113, + 128, + 129, + 134, + 135, + 148, + 149, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 178, + 179, + 184, + 185, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 214, + 215, + 220, + 221 + ]; + } + } catch (error) { + // Swallow - we only care if `supports-color` is available; it doesn't have to be. + } + + /** + * Build up the default `inspectOpts` object from the environment variables. + * + * $ DEBUG_COLORS=no DEBUG_DEPTH=10 DEBUG_SHOW_HIDDEN=enabled node script.js + */ + + exports.inspectOpts = Object.keys(process.env).filter(key => { + return /^debug_/i.test(key); + }).reduce((obj, key) => { + // Camel-case + const prop = key + .substring(6) + .toLowerCase() + .replace(/_([a-z])/g, (_, k) => { + return k.toUpperCase(); + }); + + // Coerce string value into JS value + let val = process.env[key]; + if (/^(yes|on|true|enabled)$/i.test(val)) { + val = true; + } else if (/^(no|off|false|disabled)$/i.test(val)) { + val = false; + } else if (val === 'null') { + val = null; + } else { + val = Number(val); + } + + obj[prop] = val; + return obj; + }, {}); + + /** + * Is stdout a TTY? Colored output is enabled when `true`. + */ + + function useColors() { + return 'colors' in exports.inspectOpts ? + Boolean(exports.inspectOpts.colors) : + tty.isatty(process.stderr.fd); + } + + /** + * Adds ANSI color escape codes if enabled. + * + * @api public + */ + + function formatArgs(args) { + const {namespace: name, useColors} = this; + + if (useColors) { + const c = this.color; + const colorCode = '\u001B[3' + (c < 8 ? c : '8;5;' + c); + const prefix = ` ${colorCode};1m${name} \u001B[0m`; + + args[0] = prefix + args[0].split('\n').join('\n' + prefix); + args.push(colorCode + 'm+' + module.exports.humanize(this.diff) + '\u001B[0m'); + } else { + args[0] = getDate() + name + ' ' + args[0]; + } + } + + function getDate() { + if (exports.inspectOpts.hideDate) { + return ''; + } + return new Date().toISOString() + ' '; + } + + /** + * Invokes `util.format()` with the specified arguments and writes to stderr. + */ + + function log(...args) { + return process.stderr.write(util.format(...args) + '\n'); + } + + /** + * Save `namespaces`. + * + * @param {String} namespaces + * @api private + */ + function save(namespaces) { + if (namespaces) { + process.env.DEBUG = namespaces; + } else { + // If you set a process.env field to null or undefined, it gets cast to the + // string 'null' or 'undefined'. Just delete instead. + delete process.env.DEBUG; + } + } + + /** + * Load `namespaces`. + * + * @return {String} returns the previously persisted debug modes + * @api private + */ + + function load() { + return process.env.DEBUG; + } + + /** + * Init logic for `debug` instances. + * + * Create a new `inspectOpts` object in case `useColors` is set + * differently for a particular `debug` instance. + */ + + function init(debug) { + debug.inspectOpts = {}; + + const keys = Object.keys(exports.inspectOpts); + for (let i = 0; i < keys.length; i++) { + debug.inspectOpts[keys[i]] = exports.inspectOpts[keys[i]]; + } + } + + module.exports = requireCommon()(exports); + + const {formatters} = module.exports; + + /** + * Map %o to `util.inspect()`, all on a single line. + */ + + formatters.o = function (v) { + this.inspectOpts.colors = this.useColors; + return util.inspect(v, this.inspectOpts) + .split('\n') + .map(str => str.trim()) + .join(' '); + }; + + /** + * Map %O to `util.inspect()`, allowing multiple lines if needed. + */ + + formatters.O = function (v) { + this.inspectOpts.colors = this.useColors; + return util.inspect(v, this.inspectOpts); + }; + } (node, node.exports)); + return node.exports; +} + +/** + * Detect Electron renderer / nwjs process, which is node, but we should + * treat as a browser. + */ + +if (typeof process === 'undefined' || process.type === 'renderer' || process.browser === true || process.__nwjs) { + src.exports = requireBrowser(); +} else { + src.exports = requireNode(); +} + +var srcExports = src.exports; + +/** + * @fileOverview + * A simple promises-based check to see if a TCP port is already in use. + */ + +// define the exports first to avoid cyclic dependencies. +tcpPortUsed.check = check; +tcpPortUsed.waitUntilFreeOnHost = waitUntilFreeOnHost; +tcpPortUsed.waitUntilFree = waitUntilFree; +tcpPortUsed.waitUntilUsedOnHost = waitUntilUsedOnHost; +tcpPortUsed.waitUntilUsed = waitUntilUsed; +tcpPortUsed.waitForStatus = waitForStatus; + +var is = is2; +var net = require$$1__default$2["default"]; +var util = require$$1__default$1["default"]; +var debug = srcExports('tcp-port-used'); + +// Global Values +var TIMEOUT = 2000; +var RETRYTIME = 250; + +function getDeferred() { + var resolve, reject, promise = new Promise(function(res, rej) { + resolve = res; + reject = rej; + }); + + return { + resolve: resolve, + reject: reject, + promise: promise + }; +} + +/** + * Creates an options object from all the possible arguments + * @private + * @param {Number} port a valid TCP port number + * @param {String} host The DNS name or IP address. + * @param {Boolean} status The desired in use status to wait for: false === not in use, true === in use + * @param {Number} retryTimeMs the retry interval in milliseconds - defaultis is 200ms + * @param {Number} timeOutMs the amount of time to wait until port is free default is 1000ms + * @return {Object} An options object with all the above parameters as properties. + */ +function makeOptionsObj(port, host, inUse, retryTimeMs, timeOutMs) { + var opts = {}; + opts.port = port; + opts.host = host; + opts.inUse = inUse; + opts.retryTimeMs = retryTimeMs; + opts.timeOutMs = timeOutMs; + return opts; +} + +/** + * Checks if a TCP port is in use by creating the socket and binding it to the + * target port. Once bound, successfully, it's assume the port is availble. + * After the socket is closed or in error, the promise is resolved. + * Note: you have to be super user to correctly test system ports (0-1023). + * @param {Number|Object} port The port you are curious to see if available. If an object, must have the parameters as properties. + * @param {String} [host] May be a DNS name or IP address. Default '127.0.0.1' + * @return {Object} A deferred Q promise. + * + * Example usage: + * + * var tcpPortUsed = require('tcp-port-used'); + * tcpPortUsed.check(22, '127.0.0.1') + * .then(function(inUse) { + * debug('Port 22 usage: '+inUse); + * }, function(err) { + * console.error('Error on check: '+util.inspect(err)); + * }); + */ +function check(port, host) { + + var deferred = getDeferred(); + var inUse = true; + var client; + + var opts; + if (!is.obj(port)) { + opts = makeOptionsObj(port, host); + } else { + opts = port; + } + + if (!is.port(opts.port)) { + debug('Error invalid port: '+util.inspect(opts.port)); + deferred.reject(new Error('invalid port: '+util.inspect(opts.port))); + return deferred.promise; + } + + if (is.nullOrUndefined(opts.host)) { + debug('set host address to default 127.0.0.1'); + opts.host = '127.0.0.1'; + } + + function cleanUp() { + if (client) { + client.removeAllListeners('connect'); + client.removeAllListeners('error'); + client.end(); + client.destroy(); + client.unref(); + } + //debug('listeners removed from client socket'); + } + + function onConnectCb() { + //debug('check - promise resolved - in use'); + deferred.resolve(inUse); + cleanUp(); + } + + function onErrorCb(err) { + if (err.code !== 'ECONNREFUSED') { + //debug('check - promise rejected, error: '+err.message); + deferred.reject(err); + } else { + //debug('ECONNREFUSED'); + inUse = false; + //debug('check - promise resolved - not in use'); + deferred.resolve(inUse); + } + cleanUp(); + } + + client = new net.Socket(); + client.once('connect', onConnectCb); + client.once('error', onErrorCb); + client.connect({port: opts.port, host: opts.host}, function() {}); + + return deferred.promise; +} + +/** + * Creates a deferred promise and fulfills it only when the socket's usage + * equals status in terms of 'in use' (false === not in use, true === in use). + * Will retry on an interval specified in retryTimeMs. Note: you have to be + * super user to correctly test system ports (0-1023). + * @param {Number|Object} port a valid TCP port number, if an object, has all the parameters described as properties. + * @param {String} host The DNS name or IP address. + * @param {Boolean} status The desired in use status to wait for false === not in use, true === in use + * @param {Number} [retryTimeMs] the retry interval in milliseconds - defaultis is 200ms + * @param {Number} [timeOutMs] the amount of time to wait until port is free default is 1000ms + * @return {Object} A deferred promise from the Q library. + * + * Example usage: + * + * var tcpPortUsed = require('tcp-port-used'); + * tcpPortUsed.waitForStatus(44204, 'some.host.com', true, 500, 4000) + * .then(function() { + * console.log('Port 44204 is now in use.'); + * }, function(err) { + * console.log('Error: ', error.message); + * }); + */ +function waitForStatus(port, host, inUse, retryTimeMs, timeOutMs) { + + var deferred = getDeferred(); + var timeoutId; + var timedout = false; + var retryId; + + // the first arument may be an object, if it is not, make an object + var opts; + if (is.obj(port)) { + opts = port; + } else { + opts = makeOptionsObj(port, host, inUse, retryTimeMs, timeOutMs); + } + + //debug('opts:'+util.inspect(opts); + + if (!is.bool(opts.inUse)) { + deferred.reject(new Error('inUse must be a boolean')); + return deferred.promise; + } + + if (!is.positiveInt(opts.retryTimeMs)) { + opts.retryTimeMs = RETRYTIME; + debug('set retryTime to default '+RETRYTIME+'ms'); + } + + if (!is.positiveInt(opts.timeOutMs)) { + opts.timeOutMs = TIMEOUT; + debug('set timeOutMs to default '+TIMEOUT+'ms'); + } + + function cleanUp() { + if (timeoutId) { + clearTimeout(timeoutId); + } + if (retryId) { + clearTimeout(retryId); + } + } + + function timeoutFunc() { + timedout = true; + cleanUp(); + deferred.reject(new Error('timeout')); + } + timeoutId = setTimeout(timeoutFunc, opts.timeOutMs); + + function doCheck() { + check(opts.port, opts.host) + .then(function(inUse) { + if (timedout) { + return; + } + //debug('doCheck inUse: '+inUse); + //debug('doCheck opts.inUse: '+opts.inUse); + if (inUse === opts.inUse) { + deferred.resolve(); + cleanUp(); + return; + } else { + retryId = setTimeout(function() { doCheck(); }, opts.retryTimeMs); + return; + } + }, function(err) { + if (timedout) { + return; + } + deferred.reject(err); + cleanUp(); + }); + } + + doCheck(); + return deferred.promise; +} + +/** + * Creates a deferred promise and fulfills it only when the socket is free. + * Will retry on an interval specified in retryTimeMs. + * Note: you have to be super user to correctly test system ports (0-1023). + * @param {Number} port a valid TCP port number + * @param {String} [host] The hostname or IP address of where the socket is. + * @param {Number} [retryTimeMs] the retry interval in milliseconds - defaultis is 100ms. + * @param {Number} [timeOutMs] the amount of time to wait until port is free. Default 300ms. + * @return {Object} A deferred promise from the q library. + * + * Example usage: + * + * var tcpPortUsed = require('tcp-port-used'); + * tcpPortUsed.waitUntilFreeOnHost(44203, 'some.host.com', 500, 4000) + * .then(function() { + * console.log('Port 44203 is now free.'); + * }, function(err) { + * console.loh('Error: ', error.message); + * }); + */ +function waitUntilFreeOnHost(port, host, retryTimeMs, timeOutMs) { + + // the first arument may be an object, if it is not, make an object + var opts; + if (is.obj(port)) { + opts = port; + opts.inUse = false; + } else { + opts = makeOptionsObj(port, host, false, retryTimeMs, timeOutMs); + } + + return waitForStatus(opts); +} + +/** + * For compatibility with previous version of the module, that did not provide + * arguements for hostnames. The host is set to the localhost '127.0.0.1'. + * @param {Number|Object} port a valid TCP port number. If an object, must contain all the parameters as properties. + * @param {Number} [retryTimeMs] the retry interval in milliseconds - defaultis is 100ms. + * @param {Number} [timeOutMs] the amount of time to wait until port is free. Default 300ms. + * @return {Object} A deferred promise from the q library. + * + * Example usage: + * + * var tcpPortUsed = require('tcp-port-used'); + * tcpPortUsed.waitUntilFree(44203, 500, 4000) + * .then(function() { + * console.log('Port 44203 is now free.'); + * }, function(err) { + * console.loh('Error: ', error.message); + * }); + */ +function waitUntilFree(port, retryTimeMs, timeOutMs) { + + // the first arument may be an object, if it is not, make an object + var opts; + if (is.obj(port)) { + opts = port; + opts.host = '127.0.0.1'; + opts.inUse = false; + } else { + opts = makeOptionsObj(port, '127.0.0.1', false, retryTimeMs, timeOutMs); + } + + return waitForStatus(opts); +} + +/** + * Creates a deferred promise and fulfills it only when the socket is used. + * Will retry on an interval specified in retryTimeMs. + * Note: you have to be super user to correctly test system ports (0-1023). + * @param {Number|Object} port a valid TCP port number. If an object, must contain all the parameters as properties. + * @param {Number} [retryTimeMs] the retry interval in milliseconds - defaultis is 500ms + * @param {Number} [timeOutMs] the amount of time to wait until port is free + * @return {Object} A deferred promise from the q library. + * + * Example usage: + * + * var tcpPortUsed = require('tcp-port-used'); + * tcpPortUsed.waitUntilUsedOnHost(44204, 'some.host.com', 500, 4000) + * .then(function() { + * console.log('Port 44204 is now in use.'); + * }, function(err) { + * console.log('Error: ', error.message); + * }); + */ +function waitUntilUsedOnHost(port, host, retryTimeMs, timeOutMs) { + + // the first arument may be an object, if it is not, make an object + var opts; + if (is.obj(port)) { + opts = port; + opts.inUse = true; + } else { + opts = makeOptionsObj(port, host, true, retryTimeMs, timeOutMs); + } + + return waitForStatus(opts); +} + +/** + * For compatibility to previous version of module which did not have support + * for host addresses. This function works only for localhost. + * @param {Number} port a valid TCP port number. If an Object, must contain all the parameters as properties. + * @param {Number} [retryTimeMs] the retry interval in milliseconds - defaultis is 500ms + * @param {Number} [timeOutMs] the amount of time to wait until port is free + * @return {Object} A deferred promise from the q library. + * + * Example usage: + * + * var tcpPortUsed = require('tcp-port-used'); + * tcpPortUsed.waitUntilUsed(44204, 500, 4000) + * .then(function() { + * console.log('Port 44204 is now in use.'); + * }, function(err) { + * console.log('Error: ', error.message); + * }); + */ +function waitUntilUsed(port, retryTimeMs, timeOutMs) { + + // the first arument may be an object, if it is not, make an object + var opts; + if (is.obj(port)) { + opts = port; + opts.host = '127.0.0.1'; + opts.inUse = true; + } else { + opts = makeOptionsObj(port, '127.0.0.1', true, retryTimeMs, timeOutMs); + } + + return waitUntilUsedOnHost(opts); +} + +var fetchRetry$1 = function (fetch, defaults) { + defaults = defaults || {}; + if (typeof fetch !== 'function') { + throw new ArgumentError('fetch must be a function'); + } + + if (typeof defaults !== 'object') { + throw new ArgumentError('defaults must be an object'); + } + + if (defaults.retries !== undefined && !isPositiveInteger(defaults.retries)) { + throw new ArgumentError('retries must be a positive integer'); + } + + if (defaults.retryDelay !== undefined && !isPositiveInteger(defaults.retryDelay) && typeof defaults.retryDelay !== 'function') { + throw new ArgumentError('retryDelay must be a positive integer or a function returning a positive integer'); + } + + if (defaults.retryOn !== undefined && !Array.isArray(defaults.retryOn) && typeof defaults.retryOn !== 'function') { + throw new ArgumentError('retryOn property expects an array or function'); + } + + var baseDefaults = { + retries: 3, + retryDelay: 1000, + retryOn: [], + }; + + defaults = Object.assign(baseDefaults, defaults); + + return function fetchRetry(input, init) { + var retries = defaults.retries; + var retryDelay = defaults.retryDelay; + var retryOn = defaults.retryOn; + + if (init && init.retries !== undefined) { + if (isPositiveInteger(init.retries)) { + retries = init.retries; + } else { + throw new ArgumentError('retries must be a positive integer'); + } + } + + if (init && init.retryDelay !== undefined) { + if (isPositiveInteger(init.retryDelay) || (typeof init.retryDelay === 'function')) { + retryDelay = init.retryDelay; + } else { + throw new ArgumentError('retryDelay must be a positive integer or a function returning a positive integer'); + } + } + + if (init && init.retryOn) { + if (Array.isArray(init.retryOn) || (typeof init.retryOn === 'function')) { + retryOn = init.retryOn; + } else { + throw new ArgumentError('retryOn property expects an array or function'); + } + } + + // eslint-disable-next-line no-undef + return new Promise(function (resolve, reject) { + var wrappedFetch = function (attempt) { + // As of node 18, this is no longer needed since node comes with native support for fetch: + /* istanbul ignore next */ + var _input = + typeof Request !== 'undefined' && input instanceof Request + ? input.clone() + : input; + fetch(_input, init) + .then(function (response) { + if (Array.isArray(retryOn) && retryOn.indexOf(response.status) === -1) { + resolve(response); + } else if (typeof retryOn === 'function') { + try { + // eslint-disable-next-line no-undef + return Promise.resolve(retryOn(attempt, null, response)) + .then(function (retryOnResponse) { + if(retryOnResponse) { + retry(attempt, null, response); + } else { + resolve(response); + } + }).catch(reject); + } catch (error) { + reject(error); + } + } else { + if (attempt < retries) { + retry(attempt, null, response); + } else { + resolve(response); + } + } + }) + .catch(function (error) { + if (typeof retryOn === 'function') { + try { + // eslint-disable-next-line no-undef + Promise.resolve(retryOn(attempt, error, null)) + .then(function (retryOnResponse) { + if(retryOnResponse) { + retry(attempt, error, null); + } else { + reject(error); + } + }) + .catch(function(error) { + reject(error); + }); + } catch(error) { + reject(error); + } + } else if (attempt < retries) { + retry(attempt, error, null); + } else { + reject(error); + } + }); + }; + + function retry(attempt, error, response) { + var delay = (typeof retryDelay === 'function') ? + retryDelay(attempt, error, response) : retryDelay; + setTimeout(function () { + wrappedFetch(++attempt); + }, delay); + } + + wrappedFetch(0); + }); + }; +}; + +function isPositiveInteger(value) { + return Number.isInteger(value) && value >= 0; +} + +function ArgumentError(message) { + this.name = 'ArgumentError'; + this.message = message; +} + +var fetchRT = /*@__PURE__*/getDefaultExportFromCjs(fetchRetry$1); + +var osutils = {}; + +var _os = require$$0__default["default"]; + +osutils.platform = function(){ + return process.platform; +}; + +osutils.cpuCount = function(){ + return _os.cpus().length; +}; + +osutils.sysUptime = function(){ + //seconds + return _os.uptime(); +}; + +osutils.processUptime = function(){ + //seconds + return process.uptime(); +}; + + + +// Memory +osutils.freemem = function(){ + return _os.freemem() / ( 1024 * 1024 ); +}; + +osutils.totalmem = function(){ + + return _os.totalmem() / ( 1024 * 1024 ); +}; + +osutils.freememPercentage = function(){ + return _os.freemem() / _os.totalmem(); +}; + +osutils.freeCommand = function(callback){ + + // Only Linux + require$$1__default$3["default"].exec('free -m', function(error, stdout, stderr) { + + var lines = stdout.split("\n"); + + + var str_mem_info = lines[1].replace( /[\s\n\r]+/g,' '); + + var mem_info = str_mem_info.split(' '); + + total_mem = parseFloat(mem_info[1]); + free_mem = parseFloat(mem_info[3]); + buffers_mem = parseFloat(mem_info[5]); + cached_mem = parseFloat(mem_info[6]); + + used_mem = total_mem - (free_mem + buffers_mem + cached_mem); + + callback(used_mem -2); + }); +}; + + +// Hard Disk Drive +osutils.harddrive = function(callback){ + + require$$1__default$3["default"].exec('df -k', function(error, stdout, stderr) { + + var total = 0; + var used = 0; + var free = 0; + + var lines = stdout.split("\n"); + + var str_disk_info = lines[1].replace( /[\s\n\r]+/g,' '); + + var disk_info = str_disk_info.split(' '); + + total = Math.ceil((disk_info[1] * 1024)/ Math.pow(1024,2)); + used = Math.ceil(disk_info[2] * 1024 / Math.pow(1024,2)) ; + free = Math.ceil(disk_info[3] * 1024 / Math.pow(1024,2)) ; + + callback(total, free, used); + }); +}; + + + +// Return process running current +osutils.getProcesses = function(nProcess, callback){ + + // if nprocess is undefined then is function + if(typeof nProcess === 'function'){ + + callback =nProcess; + nProcess = 0; + } + + command = 'ps -eo pcpu,pmem,time,args | sort -k 1 -r | head -n'+10; + //command = 'ps aux | head -n '+ 11 + //command = 'ps aux | head -n '+ (nProcess + 1) + if (nProcess > 0) + command = 'ps -eo pcpu,pmem,time,args | sort -k 1 -r | head -n'+(nProcess + 1); + + require$$1__default$3["default"].exec(command, function(error, stdout, stderr) { + + var lines = stdout.split("\n"); + lines.shift(); + lines.pop(); + + var result = ''; + + + lines.forEach(function(_item,_i){ + + var _str = _item.replace( /[\s\n\r]+/g,' '); + + _str = _str.split(' '); + + // result += _str[10]+" "+_str[9]+" "+_str[2]+" "+_str[3]+"\n"; // process + result += _str[1]+" "+_str[2]+" "+_str[3]+" "+_str[4].substring((_str[4].length - 25))+"\n"; // process + + }); + + callback(result); + }); +}; + + + +/* +* Returns All the load average usage for 1, 5 or 15 minutes. +*/ +osutils.allLoadavg = function(){ + + var loads = _os.loadavg(); + + return loads[0].toFixed(4)+','+loads[1].toFixed(4)+','+loads[2].toFixed(4); +}; + +/* +* Returns the load average usage for 1, 5 or 15 minutes. +*/ +osutils.loadavg = function(_time){ + + if(_time === undefined || (_time !== 5 && _time !== 15) ) _time = 1; + + var loads = _os.loadavg(); + var v = 0; + if(_time == 1) v = loads[0]; + if(_time == 5) v = loads[1]; + if(_time == 15) v = loads[2]; + + return v; +}; + + +osutils.cpuFree = function(callback){ + getCPUUsage(callback, true); +}; + +osutils.cpuUsage = function(callback){ + getCPUUsage(callback, false); +}; + +function getCPUUsage(callback, free){ + + var stats1 = getCPUInfo(); + var startIdle = stats1.idle; + var startTotal = stats1.total; + + setTimeout(function() { + var stats2 = getCPUInfo(); + var endIdle = stats2.idle; + var endTotal = stats2.total; + + var idle = endIdle - startIdle; + var total = endTotal - startTotal; + var perc = idle / total; + + if(free === true) + callback( perc ); + else + callback( (1 - perc) ); + + }, 1000 ); +} + +function getCPUInfo(callback){ + var cpus = _os.cpus(); + + var user = 0; + var nice = 0; + var sys = 0; + var idle = 0; + var irq = 0; + var total = 0; + + for(var cpu in cpus){ + + user += cpus[cpu].times.user; + nice += cpus[cpu].times.nice; + sys += cpus[cpu].times.sys; + irq += cpus[cpu].times.irq; + idle += cpus[cpu].times.idle; + } + + var total = user + nice + sys + idle + irq; + + return { + 'idle': idle, + 'total': total + }; +} + +/** + * Current nitro process + */ +var nitroProcessInfo = undefined; +/** + * This will retrive GPU informations and persist settings.json + * Will be called when the extension is loaded to turn on GPU acceleration if supported + */ +function updateNvidiaInfo(nvidiaSettings) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + if (!(process.platform !== "darwin")) return [3 /*break*/, 2]; + return [4 /*yield*/, Promise.all([ + updateNvidiaDriverInfo(nvidiaSettings), + updateCudaExistence(nvidiaSettings), + updateGpuInfo(nvidiaSettings), + ])]; + case 1: + _a.sent(); + _a.label = 2; + case 2: return [2 /*return*/]; + } + }); + }); +} +/** + * Retrieve current nitro process + */ +var getNitroProcessInfo = function (subprocess) { + nitroProcessInfo = { + isRunning: subprocess != null, + }; + return nitroProcessInfo; +}; +/** + * Validate nvidia and cuda for linux and windows + */ +function updateNvidiaDriverInfo(nvidiaSettings) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + node_child_process.exec("nvidia-smi --query-gpu=driver_version --format=csv,noheader", function (error, stdout) { + if (!error) { + var firstLine = stdout.split("\n")[0].trim(); + nvidiaSettings["nvidia_driver"].exist = true; + nvidiaSettings["nvidia_driver"].version = firstLine; + } + else { + nvidiaSettings["nvidia_driver"].exist = false; + } + }); + return [2 /*return*/]; + }); + }); +} +/** + * Check if file exists in paths + */ +function checkFileExistenceInPaths(file, paths) { + return paths.some(function (p) { return fs.existsSync(path__default["default"].join(p, file)); }); +} +/** + * Validate cuda for linux and windows + */ +function updateCudaExistence(nvidiaSettings) { + var filesCuda12; + var filesCuda11; + var paths; + var cudaVersion = ""; + if (process.platform === "win32") { + filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; + filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; + paths = process.env.PATH ? process.env.PATH.split(path__default["default"].delimiter) : []; + } + else { + filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; + filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; + paths = process.env.LD_LIBRARY_PATH + ? process.env.LD_LIBRARY_PATH.split(path__default["default"].delimiter) + : []; + paths.push("/usr/lib/x86_64-linux-gnu/"); + } + var cudaExists = filesCuda12.every(function (file) { return fs.existsSync(file) || checkFileExistenceInPaths(file, paths); }); + if (!cudaExists) { + cudaExists = filesCuda11.every(function (file) { return fs.existsSync(file) || checkFileExistenceInPaths(file, paths); }); + if (cudaExists) { + cudaVersion = "11"; + } + } + else { + cudaVersion = "12"; + } + nvidiaSettings["cuda"].exist = cudaExists; + nvidiaSettings["cuda"].version = cudaVersion; + if (cudaExists) { + nvidiaSettings.run_mode = "gpu"; + } +} +/** + * Get GPU information + */ +function updateGpuInfo(nvidiaSettings) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + node_child_process.exec("nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", function (error, stdout) { + if (!error) { + // Get GPU info and gpu has higher memory first + var highestVram_1 = 0; + var highestVramId_1 = "0"; + var gpus = stdout + .trim() + .split("\n") + .map(function (line) { + var _a = line.split(", "), id = _a[0], vram = _a[1]; + vram = vram.replace(/\r/g, ""); + if (parseFloat(vram) > highestVram_1) { + highestVram_1 = parseFloat(vram); + highestVramId_1 = id; + } + return { id: id, vram: vram }; + }); + nvidiaSettings["gpus"] = gpus; + nvidiaSettings["gpu_highest_vram"] = highestVramId_1; + } + else { + nvidiaSettings["gpus"] = []; + } + }); + return [2 /*return*/]; + }); + }); +} + +/** + * Find which executable file to run based on the current platform. + * @returns The name of the executable file to run. + */ +var executableNitroFile = function (nvidiaSettings) { + var binaryFolder = path__default["default"].join(__dirname, "..", "bin"); // Current directory by default + var cudaVisibleDevices = ""; + var binaryName = "nitro"; + /** + * The binary folder is different for each platform. + */ + if (process.platform === "win32") { + /** + * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 + */ + if (nvidiaSettings["run_mode"] === "cpu") { + binaryFolder = path__default["default"].join(binaryFolder, "win-cpu"); + } + else { + if (nvidiaSettings["cuda"].version === "12") { + binaryFolder = path__default["default"].join(binaryFolder, "win-cuda-12-0"); + } + else { + binaryFolder = path__default["default"].join(binaryFolder, "win-cuda-11-7"); + } + cudaVisibleDevices = nvidiaSettings["gpu_highest_vram"]; + } + binaryName = "nitro.exe"; + } + else if (process.platform === "darwin") { + /** + * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) + */ + if (process.arch === "arm64") { + binaryFolder = path__default["default"].join(binaryFolder, "mac-arm64"); + } + else { + binaryFolder = path__default["default"].join(binaryFolder, "mac-x64"); + } + } + else { + if (nvidiaSettings["run_mode"] === "cpu") { + binaryFolder = path__default["default"].join(binaryFolder, "linux-cpu"); + } + else { + if (nvidiaSettings["cuda"].version === "12") { + binaryFolder = path__default["default"].join(binaryFolder, "linux-cuda-12-0"); + } + else { + binaryFolder = path__default["default"].join(binaryFolder, "linux-cuda-11-7"); + } + cudaVisibleDevices = nvidiaSettings["gpu_highest_vram"]; + } + } + return { + executablePath: path__default["default"].join(binaryFolder, binaryName), + cudaVisibleDevices: cudaVisibleDevices, + }; +}; + +// Polyfill fetch with retry +var fetchRetry = fetchRT(fetch); +// The PORT to use for the Nitro subprocess +var PORT = 3928; +// The HOST address to use for the Nitro subprocess +var LOCAL_HOST = "127.0.0.1"; +// The URL for the Nitro subprocess +var NITRO_HTTP_SERVER_URL = "http://".concat(LOCAL_HOST, ":").concat(PORT); +// The URL for the Nitro subprocess to load a model +var NITRO_HTTP_LOAD_MODEL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/loadmodel"); +// The URL for the Nitro subprocess to validate a model +var NITRO_HTTP_VALIDATE_MODEL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/modelstatus"); +// The URL for the Nitro subprocess to kill itself +var NITRO_HTTP_KILL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/processmanager/destroy"); +// The URL for the Nitro subprocess to run chat completion +var NITRO_HTTP_CHAT_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/chat_completion"); +// The default config for using Nvidia GPU +var NVIDIA_DEFAULT_CONFIG = { + notify: true, + run_mode: "cpu", + nvidia_driver: { + exist: false, + version: "", + }, + cuda: { + exist: false, + version: "", + }, + gpus: [], + gpu_highest_vram: "", +}; +// The supported model format +// TODO: Should be an array to support more models +var SUPPORTED_MODEL_FORMATS = [".gguf"]; +// The subprocess instance for Nitro +var subprocess = undefined; +// The current model file url +var currentModelFile = ""; +// The current model settings +var currentSettings = undefined; +// The Nvidia info file for checking for CUDA support on the system +var nvidiaConfig = NVIDIA_DEFAULT_CONFIG; +// The logger to use, default to stdout +var log = function (message) { + return process.stdout.write(message + os__default["default"].EOL); +}; +/** + * Get current Nvidia config + * @returns {NitroNvidiaConfig} A copy of the config object + * The returned object should be used for reading only + * Writing to config should be via the function {@setNvidiaConfig} + */ +function getNvidiaConfig() { + return Object.assign({}, nvidiaConfig); +} +/** + * Set custom Nvidia config for running inference over GPU + * @param {NitroNvidiaConfig} config The new config to apply + */ +function setNvidiaConfig(config) { + nvidiaConfig = config; +} +/** + * Set logger before running nitro + * @param {NitroLogger} logger The logger to use + */ +function setLogger(logger) { + log = logger; +} +/** + * Stops a Nitro subprocess. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +function stopModel() { + return killSubprocess(); +} +/** + * Initializes a Nitro subprocess to load a machine learning model. + * @param modelFullPath - The absolute full path to model directory. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package + */ +function runModel(_a) { + var modelFullPath = _a.modelFullPath, promptTemplate = _a.promptTemplate; + return __awaiter(this, void 0, void 0, function () { + var files, ggufBinFile, nitroResourceProbe, prompt; + return __generator(this, function (_b) { + switch (_b.label) { + case 0: + files = fs__default["default"].readdirSync(modelFullPath); + ggufBinFile = files.find(function (file) { + return file === path__default["default"].basename(modelFullPath) || + SUPPORTED_MODEL_FORMATS.some(function (ext) { return file.toLowerCase().endsWith(ext); }); + }); + if (!ggufBinFile) + return [2 /*return*/, Promise.reject("No GGUF model file found")]; + currentModelFile = path__default["default"].join(modelFullPath, ggufBinFile); + return [4 /*yield*/, getResourcesInfo()]; + case 1: + nitroResourceProbe = _b.sent(); + prompt = {}; + if (promptTemplate) { + try { + Object.assign(prompt, promptTemplateConverter(promptTemplate)); + } + catch (e) { + return [2 /*return*/, Promise.reject(e)]; + } + } + currentSettings = __assign(__assign({}, prompt), { llama_model_path: currentModelFile, + // This is critical and requires real system information + cpu_threads: Math.max(1, Math.round(nitroResourceProbe.numCpuPhysicalCore / 2)) }); + return [2 /*return*/, runNitroAndLoadModel()]; + } + }); + }); +} +/** + * 1. Spawn Nitro process + * 2. Load model into Nitro subprocess + * 3. Validate model status + * @returns + */ +function runNitroAndLoadModel() { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + // Gather system information for CPU physical cores and memory + return [2 /*return*/, killSubprocess() + .then(function () { return tcpPortUsed.waitUntilFree(PORT, 300, 5000); }) + .then(function () { + /** + * There is a problem with Windows process manager + * Should wait for awhile to make sure the port is free and subprocess is killed + * The tested threshold is 500ms + **/ + if (process.platform === "win32") { + return new Promise(function (resolve) { return setTimeout(function () { return resolve({}); }, 500); }); + } + else { + return Promise.resolve({}); + } + }) + .then(spawnNitroProcess) + .then(function () { return loadLLMModel(currentSettings); }) + .then(validateModelStatus) + .catch(function (err) { + // TODO: Broadcast error so app could display proper error message + log("[NITRO]::Error: ".concat(err)); + return { error: err }; + })]; + }); + }); +} +/** + * Parse prompt template into agrs settings + * @param {string} promptTemplate Template as string + * @returns {(NitroPromptSetting | never)} parsed prompt setting + * @throws {Error} if cannot split promptTemplate + */ +function promptTemplateConverter(promptTemplate) { + // Split the string using the markers + var systemMarker = "{system_message}"; + var promptMarker = "{prompt}"; + if (promptTemplate.includes(systemMarker) && + promptTemplate.includes(promptMarker)) { + // Find the indices of the markers + var systemIndex = promptTemplate.indexOf(systemMarker); + var promptIndex = promptTemplate.indexOf(promptMarker); + // Extract the parts of the string + var system_prompt = promptTemplate.substring(0, systemIndex); + var user_prompt = promptTemplate.substring(systemIndex + systemMarker.length, promptIndex); + var ai_prompt = promptTemplate.substring(promptIndex + promptMarker.length); + // Return the split parts + return { system_prompt: system_prompt, user_prompt: user_prompt, ai_prompt: ai_prompt }; + } + else if (promptTemplate.includes(promptMarker)) { + // Extract the parts of the string for the case where only promptMarker is present + var promptIndex = promptTemplate.indexOf(promptMarker); + var user_prompt = promptTemplate.substring(0, promptIndex); + var ai_prompt = promptTemplate.substring(promptIndex + promptMarker.length); + // Return the split parts + return { user_prompt: user_prompt, ai_prompt: ai_prompt }; + } + // Throw error if none of the conditions are met + throw Error("Cannot split prompt template"); +} +/** + * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + */ +function loadLLMModel(settings) { + return __awaiter(this, void 0, void 0, function () { + var res, err_1; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + log("[NITRO]::Debug: Loading model with params ".concat(JSON.stringify(settings))); + _a.label = 1; + case 1: + _a.trys.push([1, 4, , 6]); + return [4 /*yield*/, fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(settings), + retries: 3, + retryDelay: 500, + })]; + case 2: + res = _a.sent(); + // FIXME: Actually check response, as the model directory might not exist + log("[NITRO]::Debug: Load model success with response ".concat(JSON.stringify(res))); + return [4 /*yield*/, Promise.resolve(res)]; + case 3: return [2 /*return*/, _a.sent()]; + case 4: + err_1 = _a.sent(); + log("[NITRO]::Error: Load model failed with error ".concat(err_1)); + return [4 /*yield*/, Promise.reject()]; + case 5: return [2 /*return*/, _a.sent()]; + case 6: return [2 /*return*/]; + } + }); + }); +} +/** + * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified + * @param {any} request The request that is then sent to nitro + * @param {WritableStream} outStream Optional stream that consume the response body + * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. + * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data + */ +function chatCompletion(request, outStream) { + return __awaiter(this, void 0, void 0, function () { + var _this = this; + return __generator(this, function (_a) { + if (outStream) { + // Add stream option if there is an outStream specified when calling this function + Object.assign(request, { + stream: true, + }); + } + log("[NITRO]::Debug: Running chat completion with request ".concat(JSON.stringify(request))); + return [2 /*return*/, fetchRetry(NITRO_HTTP_CHAT_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "text/event-stream", + "Access-Control-Allow-Origin": "*", + }, + body: JSON.stringify(request), + retries: 3, + retryDelay: 500, + }) + .then(function (response) { return __awaiter(_this, void 0, void 0, function () { + var outPipe; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + if (!outStream) return [3 /*break*/, 2]; + if (!response.body) { + throw new Error("Error running chat completion"); + } + outPipe = response.body + .pipeThrough(new TextDecoderStream()) + .pipeTo(outStream); + // Wait for all the streams to complete before returning from async function + return [4 /*yield*/, outPipe]; + case 1: + // Wait for all the streams to complete before returning from async function + _a.sent(); + _a.label = 2; + case 2: + log("[NITRO]::Debug: Chat completion success"); + return [2 /*return*/, response]; + } + }); + }); }) + .catch(function (err) { + log("[NITRO]::Error: Chat completion failed with error ".concat(err)); + throw err; + })]; + }); + }); +} +/** + * Validates the status of a model. + * @returns {Promise} A promise that resolves to an object. + * If the model is loaded successfully, the object is empty. + * If the model is not loaded successfully, the object contains an error message. + */ +function validateModelStatus() { + return __awaiter(this, void 0, void 0, function () { + var _this = this; + return __generator(this, function (_a) { + // Send a GET request to the validation URL. + // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. + return [2 /*return*/, fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + retries: 5, + retryDelay: 500, + }).then(function (res) { return __awaiter(_this, void 0, void 0, function () { + var body; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + log("[NITRO]::Debug: Validate model state success with response ".concat(JSON.stringify(res))); + if (!res.ok) return [3 /*break*/, 2]; + return [4 /*yield*/, res.json()]; + case 1: + body = _a.sent(); + // If the model is loaded, return an empty object. + // Otherwise, return an object with an error message. + if (body.model_loaded) { + return [2 /*return*/, Promise.resolve({})]; + } + _a.label = 2; + case 2: return [2 /*return*/, Promise.resolve({ error: "Validate model status failed" })]; + } + }); + }); })]; + }); + }); +} +/** + * Terminates the Nitro subprocess. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +function killSubprocess() { + return __awaiter(this, void 0, void 0, function () { + var controller; + return __generator(this, function (_a) { + controller = new AbortController(); + setTimeout(function () { return controller.abort(); }, 5000); + log("[NITRO]::Debug: Request to kill Nitro"); + return [2 /*return*/, fetch(NITRO_HTTP_KILL_URL, { + method: "DELETE", + signal: controller.signal, + }) + .then(function () { + subprocess === null || subprocess === void 0 ? void 0 : subprocess.kill(); + subprocess = undefined; + }) + .catch(function (err) { return ({ error: err }); }) + .then(function () { return tcpPortUsed.waitUntilFree(PORT, 300, 5000); }) + .then(function () { return log("[NITRO]::Debug: Nitro process is terminated"); }) + .then(function () { return Promise.resolve({}); })]; + }); + }); +} +/** + * Spawns a Nitro subprocess. + * @returns A promise that resolves when the Nitro subprocess is started. + */ +function spawnNitroProcess() { + var _this = this; + log("[NITRO]::Debug: Spawning Nitro subprocess..."); + return new Promise(function (resolve, reject) { return __awaiter(_this, void 0, void 0, function () { + var binaryFolder, executableOptions, args; + return __generator(this, function (_a) { + binaryFolder = path__default["default"].join(__dirname, "..", "bin"); + executableOptions = executableNitroFile(nvidiaConfig); + args = ["1", LOCAL_HOST, PORT.toString()]; + // Execute the binary + log("[NITRO]::Debug: Spawn nitro at path: ".concat(executableOptions.executablePath, ", and args: ").concat(args)); + subprocess = node_child_process.spawn(executableOptions.executablePath, ["1", LOCAL_HOST, PORT.toString()], { + cwd: binaryFolder, + env: __assign(__assign({}, process.env), { CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices }), + }); + // Handle subprocess output + subprocess.stdout.on("data", function (data) { + log("[NITRO]::Debug: ".concat(data)); + }); + subprocess.stderr.on("data", function (data) { + log("[NITRO]::Error: ".concat(data)); + }); + subprocess.on("close", function (code) { + log("[NITRO]::Debug: Nitro exited with code: ".concat(code)); + subprocess = undefined; + reject("child process exited with code ".concat(code)); + }); + tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(function () { + log("[NITRO]::Debug: Nitro is ready"); + resolve({}); + }); + return [2 /*return*/]; + }); + }); }); +} +/** + * Get the system resources information + */ +function getResourcesInfo() { + var _this = this; + return new Promise(function (resolve) { return __awaiter(_this, void 0, void 0, function () { + var cpu, response; + return __generator(this, function (_a) { + cpu = osutils.cpuCount(); + log("[NITRO]::CPU informations - ".concat(cpu)); + response = { + numCpuPhysicalCore: cpu, + memAvailable: 0, + }; + resolve(response); + return [2 /*return*/]; + }); + }); }); +} +var index = { + getNvidiaConfig: getNvidiaConfig, + setNvidiaConfig: setNvidiaConfig, + setLogger: setLogger, + runModel: runModel, + stopModel: stopModel, + loadLLMModel: loadLLMModel, + validateModelStatus: validateModelStatus, + chatCompletion: chatCompletion, + killSubprocess: killSubprocess, + updateNvidiaInfo: function () { return __awaiter(void 0, void 0, void 0, function () { return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, updateNvidiaInfo(nvidiaConfig)]; + case 1: return [2 /*return*/, _a.sent()]; + } + }); }); }, + getCurrentNitroProcessInfo: function () { return getNitroProcessInfo(subprocess); }, +}; + +module.exports = index; +//# sourceMappingURL=index.cjs.js.map diff --git a/nitro-node/dist/index.js b/nitro-node/dist/index.js new file mode 100644 index 000000000..9d3eff8a9 --- /dev/null +++ b/nitro-node/dist/index.js @@ -0,0 +1,448 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +var tslib_1 = require("tslib"); +var node_os_1 = tslib_1.__importDefault(require("node:os")); +var node_fs_1 = tslib_1.__importDefault(require("node:fs")); +var node_path_1 = tslib_1.__importDefault(require("node:path")); +var node_child_process_1 = require("node:child_process"); +var tcp_port_used_1 = tslib_1.__importDefault(require("tcp-port-used")); +var fetch_retry_1 = tslib_1.__importDefault(require("fetch-retry")); +var os_utils_1 = tslib_1.__importDefault(require("os-utils")); +var nvidia_1 = require("./nvidia"); +var execute_1 = require("./execute"); +// Polyfill fetch with retry +var fetchRetry = (0, fetch_retry_1.default)(fetch); +// The PORT to use for the Nitro subprocess +var PORT = 3928; +// The HOST address to use for the Nitro subprocess +var LOCAL_HOST = "127.0.0.1"; +// The URL for the Nitro subprocess +var NITRO_HTTP_SERVER_URL = "http://".concat(LOCAL_HOST, ":").concat(PORT); +// The URL for the Nitro subprocess to load a model +var NITRO_HTTP_LOAD_MODEL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/loadmodel"); +// The URL for the Nitro subprocess to validate a model +var NITRO_HTTP_VALIDATE_MODEL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/modelstatus"); +// The URL for the Nitro subprocess to kill itself +var NITRO_HTTP_KILL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/processmanager/destroy"); +// The URL for the Nitro subprocess to run chat completion +var NITRO_HTTP_CHAT_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/chat_completion"); +// The default config for using Nvidia GPU +var NVIDIA_DEFAULT_CONFIG = { + notify: true, + run_mode: "cpu", + nvidia_driver: { + exist: false, + version: "", + }, + cuda: { + exist: false, + version: "", + }, + gpus: [], + gpu_highest_vram: "", +}; +// The supported model format +// TODO: Should be an array to support more models +var SUPPORTED_MODEL_FORMATS = [".gguf"]; +// The subprocess instance for Nitro +var subprocess = undefined; +// The current model file url +var currentModelFile = ""; +// The current model settings +var currentSettings = undefined; +// The Nvidia info file for checking for CUDA support on the system +var nvidiaConfig = NVIDIA_DEFAULT_CONFIG; +// The logger to use, default to stdout +var log = function (message) { + var _ = []; + for (var _i = 1; _i < arguments.length; _i++) { + _[_i - 1] = arguments[_i]; + } + return process.stdout.write(message + node_os_1.default.EOL); +}; +/** + * Get current Nvidia config + * @returns {NitroNvidiaConfig} A copy of the config object + * The returned object should be used for reading only + * Writing to config should be via the function {@setNvidiaConfig} + */ +function getNvidiaConfig() { + return Object.assign({}, nvidiaConfig); +} +/** + * Set custom Nvidia config for running inference over GPU + * @param {NitroNvidiaConfig} config The new config to apply + */ +function setNvidiaConfig(config) { + nvidiaConfig = config; +} +/** + * Set logger before running nitro + * @param {NitroLogger} logger The logger to use + */ +function setLogger(logger) { + log = logger; +} +/** + * Stops a Nitro subprocess. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +function stopModel() { + return killSubprocess(); +} +/** + * Initializes a Nitro subprocess to load a machine learning model. + * @param modelFullPath - The absolute full path to model directory. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package + */ +function runModel(_a) { + var modelFullPath = _a.modelFullPath, promptTemplate = _a.promptTemplate; + return tslib_1.__awaiter(this, void 0, void 0, function () { + var files, ggufBinFile, nitroResourceProbe, prompt; + return tslib_1.__generator(this, function (_b) { + switch (_b.label) { + case 0: + files = node_fs_1.default.readdirSync(modelFullPath); + ggufBinFile = files.find(function (file) { + return file === node_path_1.default.basename(modelFullPath) || + SUPPORTED_MODEL_FORMATS.some(function (ext) { return file.toLowerCase().endsWith(ext); }); + }); + if (!ggufBinFile) + return [2 /*return*/, Promise.reject("No GGUF model file found")]; + currentModelFile = node_path_1.default.join(modelFullPath, ggufBinFile); + return [4 /*yield*/, getResourcesInfo()]; + case 1: + nitroResourceProbe = _b.sent(); + prompt = {}; + if (promptTemplate) { + try { + Object.assign(prompt, promptTemplateConverter(promptTemplate)); + } + catch (e) { + return [2 /*return*/, Promise.reject(e)]; + } + } + currentSettings = tslib_1.__assign(tslib_1.__assign({}, prompt), { llama_model_path: currentModelFile, + // This is critical and requires real system information + cpu_threads: Math.max(1, Math.round(nitroResourceProbe.numCpuPhysicalCore / 2)) }); + return [2 /*return*/, runNitroAndLoadModel()]; + } + }); + }); +} +/** + * 1. Spawn Nitro process + * 2. Load model into Nitro subprocess + * 3. Validate model status + * @returns + */ +function runNitroAndLoadModel() { + return tslib_1.__awaiter(this, void 0, void 0, function () { + return tslib_1.__generator(this, function (_a) { + // Gather system information for CPU physical cores and memory + return [2 /*return*/, killSubprocess() + .then(function () { return tcp_port_used_1.default.waitUntilFree(PORT, 300, 5000); }) + .then(function () { + /** + * There is a problem with Windows process manager + * Should wait for awhile to make sure the port is free and subprocess is killed + * The tested threshold is 500ms + **/ + if (process.platform === "win32") { + return new Promise(function (resolve) { return setTimeout(function () { return resolve({}); }, 500); }); + } + else { + return Promise.resolve({}); + } + }) + .then(spawnNitroProcess) + .then(function () { return loadLLMModel(currentSettings); }) + .then(validateModelStatus) + .catch(function (err) { + // TODO: Broadcast error so app could display proper error message + log("[NITRO]::Error: ".concat(err)); + return { error: err }; + })]; + }); + }); +} +/** + * Parse prompt template into agrs settings + * @param {string} promptTemplate Template as string + * @returns {(NitroPromptSetting | never)} parsed prompt setting + * @throws {Error} if cannot split promptTemplate + */ +function promptTemplateConverter(promptTemplate) { + // Split the string using the markers + var systemMarker = "{system_message}"; + var promptMarker = "{prompt}"; + if (promptTemplate.includes(systemMarker) && + promptTemplate.includes(promptMarker)) { + // Find the indices of the markers + var systemIndex = promptTemplate.indexOf(systemMarker); + var promptIndex = promptTemplate.indexOf(promptMarker); + // Extract the parts of the string + var system_prompt = promptTemplate.substring(0, systemIndex); + var user_prompt = promptTemplate.substring(systemIndex + systemMarker.length, promptIndex); + var ai_prompt = promptTemplate.substring(promptIndex + promptMarker.length); + // Return the split parts + return { system_prompt: system_prompt, user_prompt: user_prompt, ai_prompt: ai_prompt }; + } + else if (promptTemplate.includes(promptMarker)) { + // Extract the parts of the string for the case where only promptMarker is present + var promptIndex = promptTemplate.indexOf(promptMarker); + var user_prompt = promptTemplate.substring(0, promptIndex); + var ai_prompt = promptTemplate.substring(promptIndex + promptMarker.length); + // Return the split parts + return { user_prompt: user_prompt, ai_prompt: ai_prompt }; + } + // Throw error if none of the conditions are met + throw Error("Cannot split prompt template"); +} +/** + * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + */ +function loadLLMModel(settings) { + return tslib_1.__awaiter(this, void 0, void 0, function () { + var res, err_1; + return tslib_1.__generator(this, function (_a) { + switch (_a.label) { + case 0: + log("[NITRO]::Debug: Loading model with params ".concat(JSON.stringify(settings))); + _a.label = 1; + case 1: + _a.trys.push([1, 4, , 6]); + return [4 /*yield*/, fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(settings), + retries: 3, + retryDelay: 500, + })]; + case 2: + res = _a.sent(); + // FIXME: Actually check response, as the model directory might not exist + log("[NITRO]::Debug: Load model success with response ".concat(JSON.stringify(res))); + return [4 /*yield*/, Promise.resolve(res)]; + case 3: return [2 /*return*/, _a.sent()]; + case 4: + err_1 = _a.sent(); + log("[NITRO]::Error: Load model failed with error ".concat(err_1)); + return [4 /*yield*/, Promise.reject()]; + case 5: return [2 /*return*/, _a.sent()]; + case 6: return [2 /*return*/]; + } + }); + }); +} +/** + * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified + * @param {any} request The request that is then sent to nitro + * @param {WritableStream} outStream Optional stream that consume the response body + * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. + * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data + */ +function chatCompletion(request, outStream) { + return tslib_1.__awaiter(this, void 0, void 0, function () { + var _this = this; + return tslib_1.__generator(this, function (_a) { + if (outStream) { + // Add stream option if there is an outStream specified when calling this function + Object.assign(request, { + stream: true, + }); + } + log("[NITRO]::Debug: Running chat completion with request ".concat(JSON.stringify(request))); + return [2 /*return*/, fetchRetry(NITRO_HTTP_CHAT_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "text/event-stream", + "Access-Control-Allow-Origin": "*", + }, + body: JSON.stringify(request), + retries: 3, + retryDelay: 500, + }) + .then(function (response) { return tslib_1.__awaiter(_this, void 0, void 0, function () { + var outPipe; + return tslib_1.__generator(this, function (_a) { + switch (_a.label) { + case 0: + if (!outStream) return [3 /*break*/, 2]; + if (!response.body) { + throw new Error("Error running chat completion"); + } + outPipe = response.body + .pipeThrough(new TextDecoderStream()) + .pipeTo(outStream); + // Wait for all the streams to complete before returning from async function + return [4 /*yield*/, outPipe]; + case 1: + // Wait for all the streams to complete before returning from async function + _a.sent(); + _a.label = 2; + case 2: + log("[NITRO]::Debug: Chat completion success"); + return [2 /*return*/, response]; + } + }); + }); }) + .catch(function (err) { + log("[NITRO]::Error: Chat completion failed with error ".concat(err)); + throw err; + })]; + }); + }); +} +/** + * Validates the status of a model. + * @returns {Promise} A promise that resolves to an object. + * If the model is loaded successfully, the object is empty. + * If the model is not loaded successfully, the object contains an error message. + */ +function validateModelStatus() { + return tslib_1.__awaiter(this, void 0, void 0, function () { + var _this = this; + return tslib_1.__generator(this, function (_a) { + // Send a GET request to the validation URL. + // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. + return [2 /*return*/, fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + retries: 5, + retryDelay: 500, + }).then(function (res) { return tslib_1.__awaiter(_this, void 0, void 0, function () { + var body; + return tslib_1.__generator(this, function (_a) { + switch (_a.label) { + case 0: + log("[NITRO]::Debug: Validate model state success with response ".concat(JSON.stringify(res))); + if (!res.ok) return [3 /*break*/, 2]; + return [4 /*yield*/, res.json()]; + case 1: + body = _a.sent(); + // If the model is loaded, return an empty object. + // Otherwise, return an object with an error message. + if (body.model_loaded) { + return [2 /*return*/, Promise.resolve({})]; + } + _a.label = 2; + case 2: return [2 /*return*/, Promise.resolve({ error: "Validate model status failed" })]; + } + }); + }); })]; + }); + }); +} +/** + * Terminates the Nitro subprocess. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +function killSubprocess() { + return tslib_1.__awaiter(this, void 0, void 0, function () { + var controller; + return tslib_1.__generator(this, function (_a) { + controller = new AbortController(); + setTimeout(function () { return controller.abort(); }, 5000); + log("[NITRO]::Debug: Request to kill Nitro"); + return [2 /*return*/, fetch(NITRO_HTTP_KILL_URL, { + method: "DELETE", + signal: controller.signal, + }) + .then(function () { + subprocess === null || subprocess === void 0 ? void 0 : subprocess.kill(); + subprocess = undefined; + }) + .catch(function (err) { return ({ error: err }); }) + .then(function () { return tcp_port_used_1.default.waitUntilFree(PORT, 300, 5000); }) + .then(function () { return log("[NITRO]::Debug: Nitro process is terminated"); }) + .then(function () { return Promise.resolve({}); })]; + }); + }); +} +/** + * Spawns a Nitro subprocess. + * @returns A promise that resolves when the Nitro subprocess is started. + */ +function spawnNitroProcess() { + var _this = this; + log("[NITRO]::Debug: Spawning Nitro subprocess..."); + return new Promise(function (resolve, reject) { return tslib_1.__awaiter(_this, void 0, void 0, function () { + var binaryFolder, executableOptions, args; + return tslib_1.__generator(this, function (_a) { + binaryFolder = node_path_1.default.join(__dirname, "..", "bin"); + executableOptions = (0, execute_1.executableNitroFile)(nvidiaConfig); + args = ["1", LOCAL_HOST, PORT.toString()]; + // Execute the binary + log("[NITRO]::Debug: Spawn nitro at path: ".concat(executableOptions.executablePath, ", and args: ").concat(args)); + subprocess = (0, node_child_process_1.spawn)(executableOptions.executablePath, ["1", LOCAL_HOST, PORT.toString()], { + cwd: binaryFolder, + env: tslib_1.__assign(tslib_1.__assign({}, process.env), { CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices }), + }); + // Handle subprocess output + subprocess.stdout.on("data", function (data) { + log("[NITRO]::Debug: ".concat(data)); + }); + subprocess.stderr.on("data", function (data) { + log("[NITRO]::Error: ".concat(data)); + }); + subprocess.on("close", function (code) { + log("[NITRO]::Debug: Nitro exited with code: ".concat(code)); + subprocess = undefined; + reject("child process exited with code ".concat(code)); + }); + tcp_port_used_1.default.waitUntilUsed(PORT, 300, 30000).then(function () { + log("[NITRO]::Debug: Nitro is ready"); + resolve({}); + }); + return [2 /*return*/]; + }); + }); }); +} +/** + * Get the system resources information + */ +function getResourcesInfo() { + var _this = this; + return new Promise(function (resolve) { return tslib_1.__awaiter(_this, void 0, void 0, function () { + var cpu, response; + return tslib_1.__generator(this, function (_a) { + cpu = os_utils_1.default.cpuCount(); + log("[NITRO]::CPU informations - ".concat(cpu)); + response = { + numCpuPhysicalCore: cpu, + memAvailable: 0, + }; + resolve(response); + return [2 /*return*/]; + }); + }); }); +} +exports.default = { + getNvidiaConfig: getNvidiaConfig, + setNvidiaConfig: setNvidiaConfig, + setLogger: setLogger, + runModel: runModel, + stopModel: stopModel, + loadLLMModel: loadLLMModel, + validateModelStatus: validateModelStatus, + chatCompletion: chatCompletion, + killSubprocess: killSubprocess, + updateNvidiaInfo: function () { return tslib_1.__awaiter(void 0, void 0, void 0, function () { return tslib_1.__generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, (0, nvidia_1.updateNvidiaInfo)(nvidiaConfig)]; + case 1: return [2 /*return*/, _a.sent()]; + } + }); }); }, + getCurrentNitroProcessInfo: function () { return (0, nvidia_1.getNitroProcessInfo)(subprocess); }, +}; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/nitro-node/dist/nvidia.js b/nitro-node/dist/nvidia.js new file mode 100644 index 000000000..7157c08c1 --- /dev/null +++ b/nitro-node/dist/nvidia.js @@ -0,0 +1,147 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.updateGpuInfo = exports.updateCudaExistence = exports.checkFileExistenceInPaths = exports.updateNvidiaDriverInfo = exports.getNitroProcessInfo = exports.updateNvidiaInfo = void 0; +var tslib_1 = require("tslib"); +var node_fs_1 = require("node:fs"); +var node_child_process_1 = require("node:child_process"); +var node_path_1 = tslib_1.__importDefault(require("node:path")); +/** + * Current nitro process + */ +var nitroProcessInfo = undefined; +/** + * This will retrive GPU informations and persist settings.json + * Will be called when the extension is loaded to turn on GPU acceleration if supported + */ +function updateNvidiaInfo(nvidiaSettings) { + return tslib_1.__awaiter(this, void 0, void 0, function () { + return tslib_1.__generator(this, function (_a) { + switch (_a.label) { + case 0: + if (!(process.platform !== "darwin")) return [3 /*break*/, 2]; + return [4 /*yield*/, Promise.all([ + updateNvidiaDriverInfo(nvidiaSettings), + updateCudaExistence(nvidiaSettings), + updateGpuInfo(nvidiaSettings), + ])]; + case 1: + _a.sent(); + _a.label = 2; + case 2: return [2 /*return*/]; + } + }); + }); +} +exports.updateNvidiaInfo = updateNvidiaInfo; +/** + * Retrieve current nitro process + */ +var getNitroProcessInfo = function (subprocess) { + nitroProcessInfo = { + isRunning: subprocess != null, + }; + return nitroProcessInfo; +}; +exports.getNitroProcessInfo = getNitroProcessInfo; +/** + * Validate nvidia and cuda for linux and windows + */ +function updateNvidiaDriverInfo(nvidiaSettings) { + return tslib_1.__awaiter(this, void 0, void 0, function () { + return tslib_1.__generator(this, function (_a) { + (0, node_child_process_1.exec)("nvidia-smi --query-gpu=driver_version --format=csv,noheader", function (error, stdout) { + if (!error) { + var firstLine = stdout.split("\n")[0].trim(); + nvidiaSettings["nvidia_driver"].exist = true; + nvidiaSettings["nvidia_driver"].version = firstLine; + } + else { + nvidiaSettings["nvidia_driver"].exist = false; + } + }); + return [2 /*return*/]; + }); + }); +} +exports.updateNvidiaDriverInfo = updateNvidiaDriverInfo; +/** + * Check if file exists in paths + */ +function checkFileExistenceInPaths(file, paths) { + return paths.some(function (p) { return (0, node_fs_1.existsSync)(node_path_1.default.join(p, file)); }); +} +exports.checkFileExistenceInPaths = checkFileExistenceInPaths; +/** + * Validate cuda for linux and windows + */ +function updateCudaExistence(nvidiaSettings) { + var filesCuda12; + var filesCuda11; + var paths; + var cudaVersion = ""; + if (process.platform === "win32") { + filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; + filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; + paths = process.env.PATH ? process.env.PATH.split(node_path_1.default.delimiter) : []; + } + else { + filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; + filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; + paths = process.env.LD_LIBRARY_PATH + ? process.env.LD_LIBRARY_PATH.split(node_path_1.default.delimiter) + : []; + paths.push("/usr/lib/x86_64-linux-gnu/"); + } + var cudaExists = filesCuda12.every(function (file) { return (0, node_fs_1.existsSync)(file) || checkFileExistenceInPaths(file, paths); }); + if (!cudaExists) { + cudaExists = filesCuda11.every(function (file) { return (0, node_fs_1.existsSync)(file) || checkFileExistenceInPaths(file, paths); }); + if (cudaExists) { + cudaVersion = "11"; + } + } + else { + cudaVersion = "12"; + } + nvidiaSettings["cuda"].exist = cudaExists; + nvidiaSettings["cuda"].version = cudaVersion; + if (cudaExists) { + nvidiaSettings.run_mode = "gpu"; + } +} +exports.updateCudaExistence = updateCudaExistence; +/** + * Get GPU information + */ +function updateGpuInfo(nvidiaSettings) { + return tslib_1.__awaiter(this, void 0, void 0, function () { + return tslib_1.__generator(this, function (_a) { + (0, node_child_process_1.exec)("nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", function (error, stdout) { + if (!error) { + // Get GPU info and gpu has higher memory first + var highestVram_1 = 0; + var highestVramId_1 = "0"; + var gpus = stdout + .trim() + .split("\n") + .map(function (line) { + var _a = line.split(", "), id = _a[0], vram = _a[1]; + vram = vram.replace(/\r/g, ""); + if (parseFloat(vram) > highestVram_1) { + highestVram_1 = parseFloat(vram); + highestVramId_1 = id; + } + return { id: id, vram: vram }; + }); + nvidiaSettings["gpus"] = gpus; + nvidiaSettings["gpu_highest_vram"] = highestVramId_1; + } + else { + nvidiaSettings["gpus"] = []; + } + }); + return [2 /*return*/]; + }); + }); +} +exports.updateGpuInfo = updateGpuInfo; +//# sourceMappingURL=nvidia.js.map \ No newline at end of file diff --git a/nitro-node/dist/types/execute.d.ts b/nitro-node/dist/types/execute.d.ts new file mode 100644 index 000000000..753e7a739 --- /dev/null +++ b/nitro-node/dist/types/execute.d.ts @@ -0,0 +1,9 @@ +export interface NitroExecutableOptions { + executablePath: string; + cudaVisibleDevices: string; +} +/** + * Find which executable file to run based on the current platform. + * @returns The name of the executable file to run. + */ +export declare const executableNitroFile: (nvidiaSettings: NitroNvidiaConfig) => NitroExecutableOptions; diff --git a/nitro-node/dist/types/index.d.ts b/nitro-node/dist/types/index.d.ts new file mode 100644 index 000000000..cb5d1d859 --- /dev/null +++ b/nitro-node/dist/types/index.d.ts @@ -0,0 +1,70 @@ +/** + * Get current Nvidia config + * @returns {NitroNvidiaConfig} A copy of the config object + * The returned object should be used for reading only + * Writing to config should be via the function {@setNvidiaConfig} + */ +declare function getNvidiaConfig(): NitroNvidiaConfig; +/** + * Set custom Nvidia config for running inference over GPU + * @param {NitroNvidiaConfig} config The new config to apply + */ +declare function setNvidiaConfig(config: NitroNvidiaConfig): void; +/** + * Set logger before running nitro + * @param {NitroLogger} logger The logger to use + */ +declare function setLogger(logger: NitroLogger): void; +/** + * Stops a Nitro subprocess. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +declare function stopModel(): Promise; +/** + * Initializes a Nitro subprocess to load a machine learning model. + * @param modelFullPath - The absolute full path to model directory. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package + */ +declare function runModel({ modelFullPath, promptTemplate, }: NitroModelInitOptions): Promise; +/** + * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + */ +declare function loadLLMModel(settings: any): Promise; +/** + * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified + * @param {any} request The request that is then sent to nitro + * @param {WritableStream} outStream Optional stream that consume the response body + * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. + * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data + */ +declare function chatCompletion(request: any, outStream?: WritableStream): Promise; +/** + * Validates the status of a model. + * @returns {Promise} A promise that resolves to an object. + * If the model is loaded successfully, the object is empty. + * If the model is not loaded successfully, the object contains an error message. + */ +declare function validateModelStatus(): Promise; +/** + * Terminates the Nitro subprocess. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +declare function killSubprocess(): Promise; +declare const _default: { + getNvidiaConfig: typeof getNvidiaConfig; + setNvidiaConfig: typeof setNvidiaConfig; + setLogger: typeof setLogger; + runModel: typeof runModel; + stopModel: typeof stopModel; + loadLLMModel: typeof loadLLMModel; + validateModelStatus: typeof validateModelStatus; + chatCompletion: typeof chatCompletion; + killSubprocess: typeof killSubprocess; + updateNvidiaInfo: () => Promise; + getCurrentNitroProcessInfo: () => import("./nvidia").NitroProcessInfo; +}; +export default _default; diff --git a/nitro-node/dist/types/nvidia.d.ts b/nitro-node/dist/types/nvidia.d.ts new file mode 100644 index 000000000..45ce2c507 --- /dev/null +++ b/nitro-node/dist/types/nvidia.d.ts @@ -0,0 +1,31 @@ +/** + * Nitro process info + */ +export interface NitroProcessInfo { + isRunning: boolean; +} +/** + * This will retrive GPU informations and persist settings.json + * Will be called when the extension is loaded to turn on GPU acceleration if supported + */ +export declare function updateNvidiaInfo(nvidiaSettings: NitroNvidiaConfig): Promise; +/** + * Retrieve current nitro process + */ +export declare const getNitroProcessInfo: (subprocess: any) => NitroProcessInfo; +/** + * Validate nvidia and cuda for linux and windows + */ +export declare function updateNvidiaDriverInfo(nvidiaSettings: NitroNvidiaConfig): Promise; +/** + * Check if file exists in paths + */ +export declare function checkFileExistenceInPaths(file: string, paths: string[]): boolean; +/** + * Validate cuda for linux and windows + */ +export declare function updateCudaExistence(nvidiaSettings: NitroNvidiaConfig): void; +/** + * Get GPU information + */ +export declare function updateGpuInfo(nvidiaSettings: NitroNvidiaConfig): Promise; diff --git a/nitro-node/dist/types/scripts/download-nitro.d.ts b/nitro-node/dist/types/scripts/download-nitro.d.ts new file mode 100644 index 000000000..862e9b9e8 --- /dev/null +++ b/nitro-node/dist/types/scripts/download-nitro.d.ts @@ -0,0 +1,2 @@ +declare const downloadNitro: () => void; +export default downloadNitro; diff --git a/nitro-node/dist/types/src/execute.d.ts b/nitro-node/dist/types/src/execute.d.ts new file mode 100644 index 000000000..753e7a739 --- /dev/null +++ b/nitro-node/dist/types/src/execute.d.ts @@ -0,0 +1,9 @@ +export interface NitroExecutableOptions { + executablePath: string; + cudaVisibleDevices: string; +} +/** + * Find which executable file to run based on the current platform. + * @returns The name of the executable file to run. + */ +export declare const executableNitroFile: (nvidiaSettings: NitroNvidiaConfig) => NitroExecutableOptions; diff --git a/nitro-node/dist/types/src/index.d.ts b/nitro-node/dist/types/src/index.d.ts new file mode 100644 index 000000000..cb5d1d859 --- /dev/null +++ b/nitro-node/dist/types/src/index.d.ts @@ -0,0 +1,70 @@ +/** + * Get current Nvidia config + * @returns {NitroNvidiaConfig} A copy of the config object + * The returned object should be used for reading only + * Writing to config should be via the function {@setNvidiaConfig} + */ +declare function getNvidiaConfig(): NitroNvidiaConfig; +/** + * Set custom Nvidia config for running inference over GPU + * @param {NitroNvidiaConfig} config The new config to apply + */ +declare function setNvidiaConfig(config: NitroNvidiaConfig): void; +/** + * Set logger before running nitro + * @param {NitroLogger} logger The logger to use + */ +declare function setLogger(logger: NitroLogger): void; +/** + * Stops a Nitro subprocess. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +declare function stopModel(): Promise; +/** + * Initializes a Nitro subprocess to load a machine learning model. + * @param modelFullPath - The absolute full path to model directory. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package + */ +declare function runModel({ modelFullPath, promptTemplate, }: NitroModelInitOptions): Promise; +/** + * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + */ +declare function loadLLMModel(settings: any): Promise; +/** + * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified + * @param {any} request The request that is then sent to nitro + * @param {WritableStream} outStream Optional stream that consume the response body + * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. + * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data + */ +declare function chatCompletion(request: any, outStream?: WritableStream): Promise; +/** + * Validates the status of a model. + * @returns {Promise} A promise that resolves to an object. + * If the model is loaded successfully, the object is empty. + * If the model is not loaded successfully, the object contains an error message. + */ +declare function validateModelStatus(): Promise; +/** + * Terminates the Nitro subprocess. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +declare function killSubprocess(): Promise; +declare const _default: { + getNvidiaConfig: typeof getNvidiaConfig; + setNvidiaConfig: typeof setNvidiaConfig; + setLogger: typeof setLogger; + runModel: typeof runModel; + stopModel: typeof stopModel; + loadLLMModel: typeof loadLLMModel; + validateModelStatus: typeof validateModelStatus; + chatCompletion: typeof chatCompletion; + killSubprocess: typeof killSubprocess; + updateNvidiaInfo: () => Promise; + getCurrentNitroProcessInfo: () => import("./nvidia").NitroProcessInfo; +}; +export default _default; diff --git a/nitro-node/dist/types/src/nvidia.d.ts b/nitro-node/dist/types/src/nvidia.d.ts new file mode 100644 index 000000000..45ce2c507 --- /dev/null +++ b/nitro-node/dist/types/src/nvidia.d.ts @@ -0,0 +1,31 @@ +/** + * Nitro process info + */ +export interface NitroProcessInfo { + isRunning: boolean; +} +/** + * This will retrive GPU informations and persist settings.json + * Will be called when the extension is loaded to turn on GPU acceleration if supported + */ +export declare function updateNvidiaInfo(nvidiaSettings: NitroNvidiaConfig): Promise; +/** + * Retrieve current nitro process + */ +export declare const getNitroProcessInfo: (subprocess: any) => NitroProcessInfo; +/** + * Validate nvidia and cuda for linux and windows + */ +export declare function updateNvidiaDriverInfo(nvidiaSettings: NitroNvidiaConfig): Promise; +/** + * Check if file exists in paths + */ +export declare function checkFileExistenceInPaths(file: string, paths: string[]): boolean; +/** + * Validate cuda for linux and windows + */ +export declare function updateCudaExistence(nvidiaSettings: NitroNvidiaConfig): void; +/** + * Get GPU information + */ +export declare function updateGpuInfo(nvidiaSettings: NitroNvidiaConfig): Promise; diff --git a/nitro-node/download-nitro.js b/nitro-node/download-nitro.js deleted file mode 100644 index 71eec4f42..000000000 --- a/nitro-node/download-nitro.js +++ /dev/null @@ -1,92 +0,0 @@ -const path = require("path"); -const download = require("download"); - -// Define nitro version to download in env variable -const NITRO_VERSION = process.env.NITRO_VERSION || "0.2.11"; -// The platform OS to download nitro for -const PLATFORM = process.env.npm_config_platform || process.platform; -// The platform architecture -//const ARCH = process.env.npm_config_arch || process.arch; - -const linuxVariants = { - "linux-amd64": path.join(__dirname, "bin", "linux-cpu"), - "linux-amd64-cuda-12-0": path.join(__dirname, "bin", "linux-cuda-12-0"), - "linux-amd64-cuda-11-7": path.join(__dirname, "bin", "linux-cuda-11-7"), -}; - -const darwinVariants = { - "mac-arm64": path.join(__dirname, "bin", "mac-arm64"), - "mac-amd64": path.join(__dirname, "bin", "mac-x64"), -}; - -const win32Variants = { - "win-amd64-cuda-12-0": path.join(__dirname, "bin", "win-cuda-12-0"), - "win-amd64-cuda-11-7": path.join(__dirname, "bin", "win-cuda-11-7"), - "win-amd64": path.join(__dirname, "bin", "win-cpu"), -}; - -// Mapping to installation variants -const variantMapping = { - darwin: darwinVariants, - linux: linuxVariants, - win32: win32Variants, -}; - -if (!(PLATFORM in variantMapping)) { - throw Error(`Invalid platform: ${PLATFORM}`); -} -// Get download config for this platform -const variantConfig = variantMapping[PLATFORM]; - -// Generate download link for each tarball -const getTarUrl = (version, suffix) => - `https://github.com/janhq/nitro/releases/download/v${version}/nitro-${version}-${suffix}.tar.gz`; - -// Report download progress -const createProgressReporter = (variant) => (stream) => - stream - .on("downloadProgress", (progress) => { - // Print and update progress on a single line of terminal - process.stdout.write( - `\r\x1b[K[${variant}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`, - ); - }) - .on("end", () => { - // Jump to new line to log next message - console.log(); - console.log(`[${variant}] Finished downloading!`); - }); - -// Download single binary -const downloadBinary = (version, suffix, filePath) => { - const tarUrl = getTarUrl(version, suffix); - console.log(`Downloading ${tarUrl} to ${filePath}`); - const progressReporter = createProgressReporter(suffix); - return progressReporter( - download(tarUrl, filePath, { - strip: 1, - extract: true, - }), - ); -}; - -// Download the binaries -const downloadBinaries = (version, config) => { - return Object.entries(config).reduce( - (p, [k, v]) => p.then(() => downloadBinary(version, k, v)), - Promise.resolve(), - ); -}; - -// Call the download function with version and config -const run = () => { - downloadBinaries(NITRO_VERSION, variantConfig); -}; - -module.exports = run; -module.exports.createProgressReporter = createProgressReporter; - -// Run script if called directly instead of import as module -if (require.main === module) { - run(); -} diff --git a/nitro-node/package.json b/nitro-node/package.json index caf9ee7e0..59cb02ebc 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -8,15 +8,16 @@ "license": "AGPL-3.0", "scripts": { "test": "jest --verbose --detectOpenHandles", - "pretest": "node download-nitro.js", + "pretest": "ts-node scripts/download-nitro.ts", "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "downloadnitro": "node download-nitro.js", + "downloadnitro": "ts-node scripts/download-nitro.ts", "build:publish": "npm pack", - "postinstall": "node postinstall.js" + "postinstall": "node -r @janhq/nitro-node/postinstall" }, "exports": { ".": "./dist/index.js", - "./main": "./dist/index.cjs.js" + "./main": "./dist/index.cjs.js", + "./postinstall": "./postinstall.js" }, "devDependencies": { "@rollup/plugin-commonjs": "^25.0.7", @@ -45,15 +46,9 @@ "node": ">=18.0.0" }, "files": [ - "download-nitro.js", "postinstall.js", "dist/*", "package.json", "README.md" - ], - "bundleDependencies": [ - "tcp-port-used", - "fetch-retry", - "os-utils" ] } diff --git a/nitro-node/postinstall.js b/nitro-node/postinstall.js index 18b9457af..18f6acbe1 100644 --- a/nitro-node/postinstall.js +++ b/nitro-node/postinstall.js @@ -1,6 +1,5 @@ // Only run if this package is installed as dependency -if (process.env.INIT_CWD === process.cwd()) - process.exit() +if (process.env.INIT_CWD === process.cwd()) process.exit(); -const downloadNitro = require('./download-nitro'); +const downloadNitro = require("./dist/download-nitro.cjs"); downloadNitro(); diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts index 922a1fdfe..cbfa184d2 100644 --- a/nitro-node/rollup.config.ts +++ b/nitro-node/rollup.config.ts @@ -7,9 +7,7 @@ import json from "@rollup/plugin-json"; export default [ { input: `src/index.ts`, - output: [ - { file: "dist/index.cjs.js", format: "cjs", sourcemap: true }, - ], + output: [{ file: "dist/index.cjs.js", format: "cjs", sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { @@ -29,6 +27,35 @@ export default [ extensions: [".ts", ".js", ".json"], }), + // Resolve source maps to the original source + sourceMaps(), + ], + }, + { + input: `scripts/download-nitro.ts`, + output: [ + { file: "dist/download-nitro.cjs.js", format: "cjs", sourcemap: true }, + ], + // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') + external: [], + watch: { + include: "scripts/**", + }, + plugins: [ + // Allow json resolution + json(), + // Compile TypeScript files + typescript({ useTsconfigDeclarationDir: true }), + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + commonjs(), + // Allow node_modules resolution, so you can use 'external' to control + // which external modules to include in the bundle + // https://github.com/rollup/rollup-plugin-node-resolve#usage + //resolve({ + // extensions: [".ts", ".js", ".json"], + // preferBuiltins: false, + //}), + // Resolve source maps to the original source sourceMaps(), ], diff --git a/nitro-node/scripts/download-nitro.ts b/nitro-node/scripts/download-nitro.ts new file mode 100644 index 000000000..ed0f42b29 --- /dev/null +++ b/nitro-node/scripts/download-nitro.ts @@ -0,0 +1,102 @@ +import path from "node:path"; +import download from "download"; + +// Define nitro version to download in env variable +const NITRO_VERSION = process.env.NITRO_VERSION || "0.2.11"; +// The platform OS to download nitro for +const PLATFORM = process.env.npm_config_platform || process.platform; +// The platform architecture +//const ARCH = process.env.npm_config_arch || process.arch; + +const linuxVariants = { + "linux-amd64": path.normalize(path.join(__dirname, "..", "bin", "linux-cpu")), + "linux-amd64-cuda-12-0": path.normalize( + path.join(__dirname, "..", "bin", "linux-cuda-12-0"), + ), + "linux-amd64-cuda-11-7": path.normalize( + path.join(__dirname, "..", "bin", "linux-cuda-11-7"), + ), +}; + +const darwinVariants = { + "mac-arm64": path.normalize(path.join(__dirname, "..", "bin", "mac-arm64")), + "mac-amd64": path.normalize(path.join(__dirname, "..", "bin", "mac-x64")), +}; + +const win32Variants = { + "win-amd64-cuda-12-0": path.normalize( + path.join(__dirname, "..", "bin", "win-cuda-12-0"), + ), + "win-amd64-cuda-11-7": path.normalize( + path.join(__dirname, "..", "bin", "win-cuda-11-7"), + ), + "win-amd64": path.normalize(path.join(__dirname, "..", "bin", "win-cpu")), +}; + +// Mapping to installation variants +const variantMapping: Record> = { + darwin: darwinVariants, + linux: linuxVariants, + win32: win32Variants, +}; + +if (!(PLATFORM in variantMapping)) { + throw Error(`Invalid platform: ${PLATFORM}`); +} +// Get download config for this platform +const variantConfig: Record = variantMapping[PLATFORM]; + +// Generate download link for each tarball +const getTarUrl = (version: string, suffix: string) => + `https://github.com/janhq/nitro/releases/download/v${version}/nitro-${version}-${suffix}.tar.gz`; + +// Report download progress +const createProgressReporter = (variant: string) => (stream: any) => + stream + .on( + "downloadProgress", + (progress: { transferred: any; total: any; percent: number }) => { + // Print and update progress on a single line of terminal + process.stdout.write( + `\r\x1b[K[${variant}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`, + ); + }, + ) + .on("end", () => { + // Jump to new line to log next message + console.log(); + console.log(`[${variant}] Finished downloading!`); + }); + +// Download single binary +const downloadBinary = (version: string, suffix: string, filePath: string) => { + const tarUrl = getTarUrl(version, suffix); + console.log(`Downloading ${tarUrl} to ${filePath}`); + const progressReporter = createProgressReporter(suffix); + return progressReporter( + download(tarUrl, filePath, { + strip: 1, + extract: true, + }), + ); +}; + +// Download the binaries +const downloadBinaries = (version: string, config: Record) => { + return Object.entries(config).reduce( + (p: Promise, [k, v]) => p.then(() => downloadBinary(version, k, v)), + Promise.resolve(), + ); +}; + +// Call the download function with version and config +const downloadNitro = () => { + downloadBinaries(NITRO_VERSION, variantConfig); +}; + +export default downloadNitro; + +// Run script if called directly instead of import as module +if (require.main === module) { + downloadNitro(); +} diff --git a/nitro-node/tsconfig.json b/nitro-node/tsconfig.json index c4ccd8b7f..a055f5325 100644 --- a/nitro-node/tsconfig.json +++ b/nitro-node/tsconfig.json @@ -22,7 +22,18 @@ "node_modules/@types" ] }, + "ts-node": { + "compilerOptions": { + "module": "commonjs" + }, + "include": [ + "scripts" + ] + }, "include": [ "src" + ], + "exclude": [ + "node_modules" ] } From 152f4b243eef6bac51ce9d19073b4cad87a41e04 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Fri, 26 Jan 2024 05:12:20 +0700 Subject: [PATCH 17/49] chore(nitro-node/ci): fix job name for github action workflow --- .github/workflows/test-install-nitro-node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-install-nitro-node.yml b/.github/workflows/test-install-nitro-node.yml index 6ec9db7fd..e27cf9114 100644 --- a/.github/workflows/test-install-nitro-node.yml +++ b/.github/workflows/test-install-nitro-node.yml @@ -61,7 +61,7 @@ jobs: cd scripts node e2e-test-install-nitro-node.js - windows-build: + windows-install: runs-on: windows-latest steps: - name: Clone From c81ff904c8bbca6e4e17a662f155fe94a5274fa1 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Fri, 26 Jan 2024 05:20:22 +0700 Subject: [PATCH 18/49] fix(nitro-node/ci): correct check condition for existence of bin dirs --- .github/scripts/e2e-test-install-nitro-node.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/scripts/e2e-test-install-nitro-node.js b/.github/scripts/e2e-test-install-nitro-node.js index 4b53e5df6..1dc050bbe 100644 --- a/.github/scripts/e2e-test-install-nitro-node.js +++ b/.github/scripts/e2e-test-install-nitro-node.js @@ -41,10 +41,7 @@ const checkBinaries = (repoDir) => { console.log(`Downloaded bin paths:`, matched); // Must have both the directory for the platform and the binary - return ( - matched.some((fname) => fname.startsWith(binDirPrefix)) && - matched.some((fname) => fname.startsWith(binDirPrefix)) - ); + return matched.length > 1; }; // Wrapper to wait for child process to finish From 75d8907faa0b4285aeac834edbadb6e321730e69 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Fri, 26 Jan 2024 05:32:09 +0700 Subject: [PATCH 19/49] chore(nitro-node/test): cleanup test assets Remove unused fields in model config --- nitro-node/test/nitro-process.test.ts | 217 ++++++++++++++----------- nitro-node/test/test_assets/model.json | 26 +-- 2 files changed, 124 insertions(+), 119 deletions(-) diff --git a/nitro-node/test/nitro-process.test.ts b/nitro-node/test/nitro-process.test.ts index e2d02ebb9..778d71c2b 100644 --- a/nitro-node/test/nitro-process.test.ts +++ b/nitro-node/test/nitro-process.test.ts @@ -1,66 +1,84 @@ -import { describe, test } from '@jest/globals' +import { describe, test } from "@jest/globals"; -import fs from 'node:fs' -import os from 'node:os' -import path from 'node:path' +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; -import download from 'download' +import download from "download"; -import { Duplex } from 'node:stream' -import { default as nitro } from '../src/index' -const { stopModel, runModel, loadLLMModel, validateModelStatus, chatCompletion } = nitro +import { Duplex } from "node:stream"; +import { default as nitro } from "../src/index"; +const { + stopModel, + runModel, + loadLLMModel, + validateModelStatus, + chatCompletion, +} = nitro; // FIXME: Shorthand only possible for es6 targets and up //import * as model from './model.json' assert {type: 'json'} // Test assets dir -const TEST_ASSETS_PATH = path.join(__dirname, 'test_assets') -const MODEL_CONFIG_PATH = path.join(TEST_ASSETS_PATH, 'model.json') +const TEST_ASSETS_PATH = path.join(__dirname, "test_assets"); +const MODEL_CONFIG_PATH = path.join(TEST_ASSETS_PATH, "model.json"); // Get model config const getModelConfigHook = (callback: (modelCfg: any) => void) => () => { - const modelJson = fs.readFileSync(MODEL_CONFIG_PATH, { encoding: 'utf8' }) - const modelCfg = JSON.parse(modelJson) - callback(modelCfg) -} + const modelJson = fs.readFileSync(MODEL_CONFIG_PATH, { encoding: "utf8" }); + const modelCfg = JSON.parse(modelJson); + callback(modelCfg); +}; // Report download progress -const createProgressReporter = (name: string) => (stream: Promise & Duplex) => stream.on( - 'downloadProgress', - (progress: { transferred: any; total: any; percent: number }) => { - // Print and update progress on a single line of terminal - process.stdout.write(`\r\x1b[K[${name}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`); - }).on('end', () => { - // Jump to new line to log next message - process.stdout.write(`${os.EOL}[${name}] Finished downloading!`); - }) +const createProgressReporter = + (name: string) => (stream: Promise & Duplex) => + stream + .on( + "downloadProgress", + (progress: { transferred: any; total: any; percent: number }) => { + // Print and update progress on a single line of terminal + process.stdout.write( + `\r\x1b[K[${name}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`, + ); + }, + ) + .on("end", () => { + // Jump to new line to log next message + process.stdout.write(`${os.EOL}[${name}] Finished downloading!`); + }); // Download model file const downloadModelHook = (modelCfg: any, targetDir: string) => async () => { - const fileName = modelCfg.source_url.split('/')?.pop() ?? 'model.gguf' + const fileName = modelCfg.source_url.split("/")?.pop() ?? "model.gguf"; // Check if there is a downloaded model at TEST_ASSETS_PATH - const downloadedModelFile = fs.readdirSync(TEST_ASSETS_PATH).find((fname) => fname.match(/.*\.gguf/ig)) + const downloadedModelFile = fs + .readdirSync(TEST_ASSETS_PATH) + .find((fname) => fname.match(/.*\.gguf/gi)); if (downloadedModelFile) { - const downloadedModelPath = path.join(TEST_ASSETS_PATH, downloadedModelFile) + const downloadedModelPath = path.join( + TEST_ASSETS_PATH, + downloadedModelFile, + ); // Copy model files to targetDir and return - fs.cpSync(downloadedModelPath, path.join(targetDir, fileName)) - console.log(`Reuse cached model ${modelCfg.name} from path ${downloadedModelPath} => ${targetDir}`) - return + fs.cpSync(downloadedModelPath, path.join(targetDir, fileName)); + console.log( + `Reuse cached model ${modelCfg.name} from path ${downloadedModelPath} => ${targetDir}`, + ); + return; } - const progressReporter = createProgressReporter(modelCfg.name) + const progressReporter = createProgressReporter(modelCfg.name); await progressReporter( - download( - modelCfg.source_url, - targetDir, - { - filename: fileName, - strip: 1, - extract: true, - }, - ) - ) - console.log(`Downloaded model ${modelCfg.name} at path ${path.join(targetDir, fileName)}`) -} + download(modelCfg.source_url, targetDir, { + filename: fileName, + strip: 1, + extract: true, + }), + ); + console.log( + `Downloaded model ${modelCfg.name} at path ${path.join(targetDir, fileName)}`, + ); +}; // Cleanup tmp directory that is used during tests const cleanupTargetDirHook = (targetDir: string) => () => { @@ -68,23 +86,26 @@ const cleanupTargetDirHook = (targetDir: string) => () => { recursive: true, // Remove whole directory maxRetries: 3, // Retry 3 times on error retryDelay: 250, // Back-off with 250ms delay - }) -} + }); +}; /** * Sleep for the specified milliseconds * @param {number} ms milliseconds to sleep for * @returns {Promise} */ -const sleep = async (ms: number): Promise => Promise.resolve().then(() => setTimeout(() => void (0), ms)) +const sleep = async (ms: number): Promise => + Promise.resolve().then(() => setTimeout(() => void 0, ms)); /** - * Basic test suite - */ -describe('Manage nitro process', () => { + * Basic test suite + */ +describe("Manage nitro process", () => { /// BEGIN SUITE CONFIG - const modelFullPath = fs.mkdtempSync(path.join(os.tmpdir(), 'nitro-node-test')); - let modelCfg: any = {} + const modelFullPath = fs.mkdtempSync( + path.join(os.tmpdir(), "nitro-node-test"), + ); + let modelCfg: any = {}; // Setup steps before running the suite const setupHooks = [ @@ -92,14 +113,14 @@ describe('Manage nitro process', () => { getModelConfigHook((cfg) => Object.assign(modelCfg, cfg)), // Download model before starting tests downloadModelHook(modelCfg, modelFullPath), - ] + ]; // Teardown steps after running the suite const teardownHooks = [ // Stop nitro after running, regardless of error or not () => stopModel(), // On teardown, cleanup tmp directory that was created earlier cleanupTargetDirHook(modelFullPath), - ] + ]; /// END SUITE CONFIG /// BEGIN HOOKS REGISTERING @@ -108,100 +129,102 @@ describe('Manage nitro process', () => { async () => setupHooks.reduce((p, fn) => p.then(fn), Promise.resolve()), // Set timeout for tests to wait for downloading model before run 10 * 60 * 1000, - ) + ); afterAll( // Run all the hooks sequentially async () => teardownHooks.reduce((p, fn) => p.then(fn), Promise.resolve()), // Set timeout for cleaning up 10 * 60 * 1000, - ) + ); /// END HOOKS REGISTERING /// BEGIN TESTS - test('start/stop nitro process normally', + test( + "start/stop nitro process normally", async () => { // Start nitro await runModel({ modelFullPath, promptTemplate: modelCfg.settings.prompt_template, - }) + }); // Wait 5s for nitro to start - await sleep(5 * 1000) + await sleep(5 * 1000); // Stop nitro - await stopModel() + await stopModel(); }, // Set timeout to 30 seconds 30 * 1000, - ) - test('chat completion', + ); + test( + "chat completion", async () => { // Start nitro await runModel({ modelFullPath, promptTemplate: modelCfg.settings.prompt_template, - }) + }); // Wait 5s for nitro to start - await sleep(5 * 1000) + await sleep(5 * 1000); // Load LLM model await loadLLMModel({ - "llama_model_path": modelFullPath, - "ctx_len": 50, - "ngl": 32, - "embedding": false - }) + llama_model_path: modelFullPath, + ctx_len: modelCfg.settings.ctx_len, + ngl: modelCfg.settings.ngl, + embedding: false, + }); // Validate model status - await validateModelStatus() + await validateModelStatus(); // Arrays of all the chunked response - let streamedContent: string[] = [] + let streamedContent: string[] = []; // Run chat completion with stream const response = await chatCompletion( { - "messages": [ - { "content": "Hello there", "role": "assistant" }, - { "content": "Write a long and sad story for me", "role": "user" } + messages: [ + { content: "Hello there", role: "assistant" }, + { content: "Write a long and sad story for me", role: "user" }, ], - "model": "gpt-3.5-turbo", - "max_tokens": 50, - "stop": ["hello"], - "frequency_penalty": 0, - "presence_penalty": 0, - "temperature": 0.1 + model: "gpt-3.5-turbo", + max_tokens: 50, + stop: ["hello"], + frequency_penalty: 0, + presence_penalty: 0, + temperature: 0.1, }, new WritableStream({ write(chunk: string) { - if (chunk.trim() == 'data: [DONE]') { - return + if (chunk.trim() == "data: [DONE]") { + return; } return new Promise((resolve) => { - streamedContent.push( - chunk.slice('data:'.length).trim() - ) - resolve() - }) + streamedContent.push(chunk.slice("data:".length).trim()); + resolve(); + }); }, //close() {}, //abort(_err) {} - }) - ) + }), + ); // Remove the [DONE] message - streamedContent.pop() + streamedContent.pop(); // Parse json - streamedContent = streamedContent.map((str) => JSON.parse(str)) + streamedContent = streamedContent.map((str) => JSON.parse(str)); // Show the streamed content - console.log(`[Streamed response] ${JSON.stringify(streamedContent, null, 2)}`) + console.log( + `[Streamed response] ${JSON.stringify(streamedContent, null, 2)}`, + ); // The response body is unusable if consumed by out stream - await expect(response.text).rejects.toThrow() - await expect(response.json).rejects.toThrow() + await expect(response.text).rejects.toThrow(); + await expect(response.json).rejects.toThrow(); // Response body should be used already - expect(response.bodyUsed).toBeTruthy() + expect(response.bodyUsed).toBeTruthy(); // There should be multiple chunks of json data - expect(streamedContent.length).toBeGreaterThan(0) + expect(streamedContent.length).toBeGreaterThan(0); // Stop nitro - await stopModel() + await stopModel(); }, // Set timeout to 1 minutes 1 * 60 * 1000, - ) + ); /// END TESTS -}) +}); diff --git a/nitro-node/test/test_assets/model.json b/nitro-node/test/test_assets/model.json index 641511569..1e12aca81 100644 --- a/nitro-node/test/test_assets/model.json +++ b/nitro-node/test/test_assets/model.json @@ -1,28 +1,10 @@ { "source_url": "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", "id": "tinyllama-1.1b", - "object": "model", "name": "TinyLlama Chat 1.1B Q4", - "version": "1.0", - "description": "TinyLlama is a tiny model with only 1.1B. It's a good model for less powerful computers.", - "format": "gguf", "settings": { - "ctx_len": 2048, - "prompt_template": "<|system|>\n{system_message}<|user|>\n{prompt}<|assistant|>" - }, - "parameters": { - "temperature": 0.7, - "top_p": 0.95, - "stream": true, - "max_tokens": 2048, - "stop": [], - "frequency_penalty": 0, - "presence_penalty": 0 - }, - "metadata": { - "author": "TinyLlama", - "tags": ["Tiny", "Foundation Model"], - "size": 669000000 - }, - "engine": "nitro" + "ctx_len": 50, + "ngl": 32, + "prompt_template": "<|system|>\n{system_message}<|user|>\n{prompt}<|assistant|>" + } } \ No newline at end of file From 029a142d9036453e43d04c404174f25c2e66a423 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Fri, 26 Jan 2024 13:09:58 +0700 Subject: [PATCH 20/49] fix(nitro-node/ci): remove nitro-node/dist from workflow dependency - Add a GiHub workflow job to build tarball for nitro-node instead - The tarball is uploaded as artifact of the run - In the test install jobs, the artifact is referenced by URL --- .../scripts/e2e-test-install-nitro-node.js | 16 +- .github/workflows/test-install-nitro-node.yml | 81 + nitro-node/.gitignore | 2 +- nitro-node/Makefile | 4 +- nitro-node/dist/download-nitro.cjs.js | 86 - nitro-node/dist/execute.js | 66 - nitro-node/dist/index.cjs.js | 4209 ----------------- nitro-node/dist/index.js | 448 -- nitro-node/dist/nvidia.js | 147 - nitro-node/dist/types/execute.d.ts | 9 - nitro-node/dist/types/index.d.ts | 70 - nitro-node/dist/types/nvidia.d.ts | 31 - .../dist/types/scripts/download-nitro.d.ts | 2 - nitro-node/dist/types/src/execute.d.ts | 9 - nitro-node/dist/types/src/index.d.ts | 70 - nitro-node/dist/types/src/nvidia.d.ts | 31 - 16 files changed, 92 insertions(+), 5189 deletions(-) delete mode 100644 nitro-node/dist/download-nitro.cjs.js delete mode 100644 nitro-node/dist/execute.js delete mode 100644 nitro-node/dist/index.cjs.js delete mode 100644 nitro-node/dist/index.js delete mode 100644 nitro-node/dist/nvidia.js delete mode 100644 nitro-node/dist/types/execute.d.ts delete mode 100644 nitro-node/dist/types/index.d.ts delete mode 100644 nitro-node/dist/types/nvidia.d.ts delete mode 100644 nitro-node/dist/types/scripts/download-nitro.d.ts delete mode 100644 nitro-node/dist/types/src/execute.d.ts delete mode 100644 nitro-node/dist/types/src/index.d.ts delete mode 100644 nitro-node/dist/types/src/nvidia.d.ts diff --git a/.github/scripts/e2e-test-install-nitro-node.js b/.github/scripts/e2e-test-install-nitro-node.js index 1dc050bbe..ae2339aec 100644 --- a/.github/scripts/e2e-test-install-nitro-node.js +++ b/.github/scripts/e2e-test-install-nitro-node.js @@ -11,9 +11,9 @@ const ADD_DEP_CMDS = { yarn: "add", }; // Path to the package to install -const NITRO_NODE_PKG = path.resolve( - path.normalize(path.join(__dirname, "..", "..", "nitro-node")), -); +const NITRO_NODE_PKG = + process.env.NITRO_NODE_PKG || + path.resolve(path.normalize(path.join(__dirname, "..", "..", "nitro-node"))); // Prefixes of downloaded nitro bin subdirectories const BIN_DIR_PREFIXES = { darwin: "mac", @@ -33,11 +33,11 @@ const checkBinaries = (repoDir) => { "bin", ); // Get the dir and files that indicate successful download of binaries - const matched = fs - .readdirSync(searchRoot, { recursive: true }) - .filter( - (fname) => fname.startsWith(binDirPrefix) || fname.startsWith("nitro"), - ); + const matched = fs.readdirSync(searchRoot, { recursive: true }).filter( + // FIXME: the result of readdirSync with recursive option is filename + // with intermediate subdirectories so this logic might not be correct + (fname) => fname.startsWith(binDirPrefix) || fname.includes("nitro"), + ); console.log(`Downloaded bin paths:`, matched); // Must have both the directory for the platform and the binary diff --git a/.github/workflows/test-install-nitro-node.yml b/.github/workflows/test-install-nitro-node.yml index e27cf9114..5c0a010ea 100644 --- a/.github/workflows/test-install-nitro-node.yml +++ b/.github/workflows/test-install-nitro-node.yml @@ -21,8 +21,40 @@ on: workflow_dispatch: jobs: + linux-pack-tarball: + runs-on: ubuntu-latest + outputs: + tarball-url: ${{ steps.upload.outputs.artifact-url }} + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Build tarball + id: build + run: | + cd nitro-node + make pack + find . -type f -name 'janhq-nitro-node-*.tgz' -exec mv {} janhq-nitro-node.tgz \; + + - name: Upload tarball as artifact + id: upload + uses: actions/upload-artifact@master + with: + name: janhq-nitro-node + path: nitro-node/janhq-nitro-node.tgz + if-no-files-found: error + ubuntu-install: runs-on: ubuntu-latest + needs: [linux-pack-tarball] + if: always() && needs.linux-pack-tarball.result == 'success' steps: - name: Clone id: checkout @@ -34,8 +66,23 @@ jobs: with: node-version: 18 + - name: Download prebuilt tarball + uses: actions/download-artifact@master + with: + name: janhq-nitro-node + path: .github/scripts/ + + - name: List tarball content + id: tar-tf + run: | + cd .github + cd scripts + tar tf janhq-nitro-node.tgz + - name: Run tests id: test_install_nitro_node + env: + NITRO_NODE_PKG: ${{ github.workspace }}/.github/scripts/janhq-nitro-node.tgz run: | cd .github cd scripts @@ -43,6 +90,8 @@ jobs: macOS-install: runs-on: macos-latest + needs: [linux-pack-tarball] + if: always() && needs.linux-pack-tarball.result == 'success' steps: - name: Clone id: checkout @@ -54,8 +103,23 @@ jobs: with: node-version: 18 + - name: Download prebuilt tarball + uses: actions/download-artifact@master + with: + name: janhq-nitro-node + path: .github/scripts/ + + - name: List tarball content + id: tar-tf + run: | + cd .github + cd scripts + tar tf janhq-nitro-node.tgz + - name: Run tests id: test_install_nitro_node + env: + NITRO_NODE_PKG: ${{ github.workspace }}/.github/scripts/janhq-nitro-node.tgz run: | cd .github cd scripts @@ -63,6 +127,8 @@ jobs: windows-install: runs-on: windows-latest + needs: [linux-pack-tarball] + if: always() && needs.linux-pack-tarball.result == 'success' steps: - name: Clone @@ -75,8 +141,23 @@ jobs: with: node-version: 18 + - name: Download prebuilt tarball + uses: actions/download-artifact@master + with: + name: janhq-nitro-node + path: .github/scripts/ + + - name: List tarball content + id: tar-tf + run: | + cd .github + cd scripts + tar tf janhq-nitro-node.tgz + - name: Run tests id: test_install_nitro_node + env: + NITRO_NODE_PKG: ${{ github.workspace }}\.github\scripts\janhq-nitro-node.tgz run: | cd .github cd scripts diff --git a/nitro-node/.gitignore b/nitro-node/.gitignore index b671335b4..446285591 100644 --- a/nitro-node/.gitignore +++ b/nitro-node/.gitignore @@ -6,7 +6,7 @@ node_modules *.tgz yarn.lock -dist/bin/ +dist build .DS_Store package-lock.json diff --git a/nitro-node/Makefile b/nitro-node/Makefile index e58f9afa0..8639f21f5 100644 --- a/nitro-node/Makefile +++ b/nitro-node/Makefile @@ -29,8 +29,8 @@ test: install @test -e test/test_assets/*.gguf && echo "test/test_assets/*.gguf is already downloaded" || (mkdir -p test/test_assets && cd test/test_assets/ && curl -JLO "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf") yarn test -# Builds and publishes the extension -publish: build +# Builds and pack +pack: build yarn run build:publish clean: diff --git a/nitro-node/dist/download-nitro.cjs.js b/nitro-node/dist/download-nitro.cjs.js deleted file mode 100644 index cad272c3d..000000000 --- a/nitro-node/dist/download-nitro.cjs.js +++ /dev/null @@ -1,86 +0,0 @@ -'use strict'; - -var path = require('node:path'); -var download = require('download'); - -function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; } - -var path__default = /*#__PURE__*/_interopDefaultLegacy(path); -var download__default = /*#__PURE__*/_interopDefaultLegacy(download); - -// Define nitro version to download in env variable -var NITRO_VERSION = process.env.NITRO_VERSION || "0.2.11"; -// The platform OS to download nitro for -var PLATFORM = process.env.npm_config_platform || process.platform; -// The platform architecture -//const ARCH = process.env.npm_config_arch || process.arch; -var linuxVariants = { - "linux-amd64": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "linux-cpu")), - "linux-amd64-cuda-12-0": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "linux-cuda-12-0")), - "linux-amd64-cuda-11-7": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "linux-cuda-11-7")), -}; -var darwinVariants = { - "mac-arm64": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "mac-arm64")), - "mac-amd64": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "mac-x64")), -}; -var win32Variants = { - "win-amd64-cuda-12-0": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "win-cuda-12-0")), - "win-amd64-cuda-11-7": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "win-cuda-11-7")), - "win-amd64": path__default["default"].normalize(path__default["default"].join(__dirname, "..", "bin", "win-cpu")), -}; -// Mapping to installation variants -var variantMapping = { - darwin: darwinVariants, - linux: linuxVariants, - win32: win32Variants, -}; -if (!(PLATFORM in variantMapping)) { - throw Error("Invalid platform: ".concat(PLATFORM)); -} -// Get download config for this platform -var variantConfig = variantMapping[PLATFORM]; -// Generate download link for each tarball -var getTarUrl = function (version, suffix) { - return "https://github.com/janhq/nitro/releases/download/v".concat(version, "/nitro-").concat(version, "-").concat(suffix, ".tar.gz"); -}; -// Report download progress -var createProgressReporter = function (variant) { return function (stream) { - return stream - .on("downloadProgress", function (progress) { - // Print and update progress on a single line of terminal - process.stdout.write("\r\u001B[K[".concat(variant, "] ").concat(progress.transferred, "/").concat(progress.total, " ").concat(Math.floor(progress.percent * 100), "%...")); - }) - .on("end", function () { - // Jump to new line to log next message - console.log(); - console.log("[".concat(variant, "] Finished downloading!")); - }); -}; }; -// Download single binary -var downloadBinary = function (version, suffix, filePath) { - var tarUrl = getTarUrl(version, suffix); - console.log("Downloading ".concat(tarUrl, " to ").concat(filePath)); - var progressReporter = createProgressReporter(suffix); - return progressReporter(download__default["default"](tarUrl, filePath, { - strip: 1, - extract: true, - })); -}; -// Download the binaries -var downloadBinaries = function (version, config) { - return Object.entries(config).reduce(function (p, _a) { - var k = _a[0], v = _a[1]; - return p.then(function () { return downloadBinary(version, k, v); }); - }, Promise.resolve()); -}; -// Call the download function with version and config -var downloadNitro = function () { - downloadBinaries(NITRO_VERSION, variantConfig); -}; -// Run script if called directly instead of import as module -if (require.main === module) { - downloadNitro(); -} - -module.exports = downloadNitro; -//# sourceMappingURL=download-nitro.cjs.js.map diff --git a/nitro-node/dist/execute.js b/nitro-node/dist/execute.js deleted file mode 100644 index 098649f31..000000000 --- a/nitro-node/dist/execute.js +++ /dev/null @@ -1,66 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.executableNitroFile = void 0; -var tslib_1 = require("tslib"); -var node_path_1 = tslib_1.__importDefault(require("node:path")); -/** - * Find which executable file to run based on the current platform. - * @returns The name of the executable file to run. - */ -var executableNitroFile = function (nvidiaSettings) { - var binaryFolder = node_path_1.default.join(__dirname, "..", "bin"); // Current directory by default - var cudaVisibleDevices = ""; - var binaryName = "nitro"; - /** - * The binary folder is different for each platform. - */ - if (process.platform === "win32") { - /** - * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 - */ - if (nvidiaSettings["run_mode"] === "cpu") { - binaryFolder = node_path_1.default.join(binaryFolder, "win-cpu"); - } - else { - if (nvidiaSettings["cuda"].version === "12") { - binaryFolder = node_path_1.default.join(binaryFolder, "win-cuda-12-0"); - } - else { - binaryFolder = node_path_1.default.join(binaryFolder, "win-cuda-11-7"); - } - cudaVisibleDevices = nvidiaSettings["gpu_highest_vram"]; - } - binaryName = "nitro.exe"; - } - else if (process.platform === "darwin") { - /** - * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) - */ - if (process.arch === "arm64") { - binaryFolder = node_path_1.default.join(binaryFolder, "mac-arm64"); - } - else { - binaryFolder = node_path_1.default.join(binaryFolder, "mac-x64"); - } - } - else { - if (nvidiaSettings["run_mode"] === "cpu") { - binaryFolder = node_path_1.default.join(binaryFolder, "linux-cpu"); - } - else { - if (nvidiaSettings["cuda"].version === "12") { - binaryFolder = node_path_1.default.join(binaryFolder, "linux-cuda-12-0"); - } - else { - binaryFolder = node_path_1.default.join(binaryFolder, "linux-cuda-11-7"); - } - cudaVisibleDevices = nvidiaSettings["gpu_highest_vram"]; - } - } - return { - executablePath: node_path_1.default.join(binaryFolder, binaryName), - cudaVisibleDevices: cudaVisibleDevices, - }; -}; -exports.executableNitroFile = executableNitroFile; -//# sourceMappingURL=execute.js.map \ No newline at end of file diff --git a/nitro-node/dist/index.cjs.js b/nitro-node/dist/index.cjs.js deleted file mode 100644 index 17273aa89..000000000 --- a/nitro-node/dist/index.cjs.js +++ /dev/null @@ -1,4209 +0,0 @@ -'use strict'; - -var os = require('node:os'); -var fs = require('node:fs'); -var path = require('node:path'); -var node_child_process = require('node:child_process'); -var require$$1$2 = require('net'); -var require$$1$1 = require('util'); -var require$$1 = require('tty'); -var require$$0 = require('os'); -var require$$1$3 = require('child_process'); - -function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; } - -var os__default = /*#__PURE__*/_interopDefaultLegacy(os); -var fs__default = /*#__PURE__*/_interopDefaultLegacy(fs); -var path__default = /*#__PURE__*/_interopDefaultLegacy(path); -var require$$1__default$2 = /*#__PURE__*/_interopDefaultLegacy(require$$1$2); -var require$$1__default$1 = /*#__PURE__*/_interopDefaultLegacy(require$$1$1); -var require$$1__default = /*#__PURE__*/_interopDefaultLegacy(require$$1); -var require$$0__default = /*#__PURE__*/_interopDefaultLegacy(require$$0); -var require$$1__default$3 = /*#__PURE__*/_interopDefaultLegacy(require$$1$3); - -/****************************************************************************** -Copyright (c) Microsoft Corporation. - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. -***************************************************************************** */ - -var __assign = function() { - __assign = Object.assign || function __assign(t) { - for (var s, i = 1, n = arguments.length; i < n; i++) { - s = arguments[i]; - for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p]; - } - return t; - }; - return __assign.apply(this, arguments); -}; - -function __awaiter(thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -} - -function __generator(thisArg, body) { - var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; - return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; - function verb(n) { return function (v) { return step([n, v]); }; } - function step(op) { - if (f) throw new TypeError("Generator is already executing."); - while (g && (g = 0, op[0] && (_ = 0)), _) try { - if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; - if (y = 0, t) op = [op[0] & 2, t.value]; - switch (op[0]) { - case 0: case 1: t = op; break; - case 4: _.label++; return { value: op[1], done: false }; - case 5: _.label++; y = op[1]; op = [0]; continue; - case 7: op = _.ops.pop(); _.trys.pop(); continue; - default: - if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } - if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } - if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } - if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } - if (t[2]) _.ops.pop(); - _.trys.pop(); continue; - } - op = body.call(thisArg, _); - } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } - if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; - } -} - -typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) { - var e = new Error(message); - return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e; -}; - -function getDefaultExportFromCjs (x) { - return x && x.__esModule && Object.prototype.hasOwnProperty.call(x, 'default') ? x['default'] : x; -} - -var tcpPortUsed = {}; - -var is2 = {}; - -var deepIs = {exports: {}}; - -var pSlice = Array.prototype.slice; -var Object_keys = typeof Object.keys === 'function' - ? Object.keys - : function (obj) { - var keys = []; - for (var key in obj) keys.push(key); - return keys; - } -; - -var deepEqual = deepIs.exports = function (actual, expected) { - // enforce Object.is +0 !== -0 - if (actual === 0 && expected === 0) { - return areZerosEqual(actual, expected); - - // 7.1. All identical values are equivalent, as determined by ===. - } else if (actual === expected) { - return true; - - } else if (actual instanceof Date && expected instanceof Date) { - return actual.getTime() === expected.getTime(); - - } else if (isNumberNaN(actual)) { - return isNumberNaN(expected); - - // 7.3. Other pairs that do not both pass typeof value == 'object', - // equivalence is determined by ==. - } else if (typeof actual != 'object' && typeof expected != 'object') { - return actual == expected; - - // 7.4. For all other Object pairs, including Array objects, equivalence is - // determined by having the same number of owned properties (as verified - // with Object.prototype.hasOwnProperty.call), the same set of keys - // (although not necessarily the same order), equivalent values for every - // corresponding key, and an identical 'prototype' property. Note: this - // accounts for both named and indexed properties on Arrays. - } else { - return objEquiv(actual, expected); - } -}; - -function isUndefinedOrNull(value) { - return value === null || value === undefined; -} - -function isArguments(object) { - return Object.prototype.toString.call(object) == '[object Arguments]'; -} - -function isNumberNaN(value) { - // NaN === NaN -> false - return typeof value == 'number' && value !== value; -} - -function areZerosEqual(zeroA, zeroB) { - // (1 / +0|0) -> Infinity, but (1 / -0) -> -Infinity and (Infinity !== -Infinity) - return (1 / zeroA) === (1 / zeroB); -} - -function objEquiv(a, b) { - if (isUndefinedOrNull(a) || isUndefinedOrNull(b)) - return false; - - // an identical 'prototype' property. - if (a.prototype !== b.prototype) return false; - //~~~I've managed to break Object.keys through screwy arguments passing. - // Converting to array solves the problem. - if (isArguments(a)) { - if (!isArguments(b)) { - return false; - } - a = pSlice.call(a); - b = pSlice.call(b); - return deepEqual(a, b); - } - try { - var ka = Object_keys(a), - kb = Object_keys(b), - key, i; - } catch (e) {//happens when one is a string literal and the other isn't - return false; - } - // having the same number of owned properties (keys incorporates - // hasOwnProperty) - if (ka.length != kb.length) - return false; - //the same set of keys (although not necessarily the same order), - ka.sort(); - kb.sort(); - //~~~cheap key test - for (i = ka.length - 1; i >= 0; i--) { - if (ka[i] != kb[i]) - return false; - } - //equivalent values for every corresponding key, and - //~~~possibly expensive deep test - for (i = ka.length - 1; i >= 0; i--) { - key = ka[i]; - if (!deepEqual(a[key], b[key])) return false; - } - return true; -} - -var deepIsExports = deepIs.exports; - -const word = '[a-fA-F\\d:]'; -const b = options => options && options.includeBoundaries ? - `(?:(?<=\\s|^)(?=${word})|(?<=${word})(?=\\s|$))` : - ''; - -const v4 = '(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)(?:\\.(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)){3}'; - -const v6seg = '[a-fA-F\\d]{1,4}'; -const v6 = ` -(?: -(?:${v6seg}:){7}(?:${v6seg}|:)| // 1:2:3:4:5:6:7:: 1:2:3:4:5:6:7:8 -(?:${v6seg}:){6}(?:${v4}|:${v6seg}|:)| // 1:2:3:4:5:6:: 1:2:3:4:5:6::8 1:2:3:4:5:6::8 1:2:3:4:5:6::1.2.3.4 -(?:${v6seg}:){5}(?::${v4}|(?::${v6seg}){1,2}|:)| // 1:2:3:4:5:: 1:2:3:4:5::7:8 1:2:3:4:5::8 1:2:3:4:5::7:1.2.3.4 -(?:${v6seg}:){4}(?:(?::${v6seg}){0,1}:${v4}|(?::${v6seg}){1,3}|:)| // 1:2:3:4:: 1:2:3:4::6:7:8 1:2:3:4::8 1:2:3:4::6:7:1.2.3.4 -(?:${v6seg}:){3}(?:(?::${v6seg}){0,2}:${v4}|(?::${v6seg}){1,4}|:)| // 1:2:3:: 1:2:3::5:6:7:8 1:2:3::8 1:2:3::5:6:7:1.2.3.4 -(?:${v6seg}:){2}(?:(?::${v6seg}){0,3}:${v4}|(?::${v6seg}){1,5}|:)| // 1:2:: 1:2::4:5:6:7:8 1:2::8 1:2::4:5:6:7:1.2.3.4 -(?:${v6seg}:){1}(?:(?::${v6seg}){0,4}:${v4}|(?::${v6seg}){1,6}|:)| // 1:: 1::3:4:5:6:7:8 1::8 1::3:4:5:6:7:1.2.3.4 -(?::(?:(?::${v6seg}){0,5}:${v4}|(?::${v6seg}){1,7}|:)) // ::2:3:4:5:6:7:8 ::2:3:4:5:6:7:8 ::8 ::1.2.3.4 -)(?:%[0-9a-zA-Z]{1,})? // %eth0 %1 -`.replace(/\s*\/\/.*$/gm, '').replace(/\n/g, '').trim(); - -// Pre-compile only the exact regexes because adding a global flag make regexes stateful -const v46Exact = new RegExp(`(?:^${v4}$)|(?:^${v6}$)`); -const v4exact = new RegExp(`^${v4}$`); -const v6exact = new RegExp(`^${v6}$`); - -const ip = options => options && options.exact ? - v46Exact : - new RegExp(`(?:${b(options)}${v4}${b(options)})|(?:${b(options)}${v6}${b(options)})`, 'g'); - -ip.v4 = options => options && options.exact ? v4exact : new RegExp(`${b(options)}${v4}${b(options)}`, 'g'); -ip.v6 = options => options && options.exact ? v6exact : new RegExp(`${b(options)}${v6}${b(options)}`, 'g'); - -var ipRegex = ip; - -var name = "is2"; -var version = "2.0.9"; -var description = "A type checking library where each exported function returns either true or false and does not throw. Also added tests."; -var license = "MIT"; -var tags = [ - "type", - "check", - "checker", - "checking", - "utilities", - "network", - "networking", - "credit", - "card", - "validation" -]; -var keywords = [ - "type", - "check", - "checker", - "checking", - "utilities", - "network", - "networking", - "credit", - "card", - "validation" -]; -var author = "Enrico Marino "; -var maintainers = "Edmond Meinfelder , Chris Oyler "; -var homepage = "http://github.com/stdarg/is2"; -var repository = { - type: "git", - url: "git@github.com:stdarg/is2.git" -}; -var bugs = { - url: "http://github.com/stdarg/is/issues" -}; -var main = "./index.js"; -var scripts = { - test: "./node_modules/.bin/mocha -C --reporter list tests.js" -}; -var engines = { - node: ">=v0.10.0" -}; -var dependencies = { - "deep-is": "^0.1.3", - "ip-regex": "^4.1.0", - "is-url": "^1.2.4" -}; -var devDependencies = { - mocha: "6.2.3", - mongodb: "3.2.4" -}; -var require$$2 = { - name: name, - version: version, - description: description, - license: license, - tags: tags, - keywords: keywords, - author: author, - maintainers: maintainers, - homepage: homepage, - repository: repository, - bugs: bugs, - main: main, - scripts: scripts, - engines: engines, - dependencies: dependencies, - devDependencies: devDependencies -}; - -var isUrl_1; -var hasRequiredIsUrl; - -function requireIsUrl () { - if (hasRequiredIsUrl) return isUrl_1; - hasRequiredIsUrl = 1; - /** - * Expose `isUrl`. - */ - - isUrl_1 = isUrl; - - /** - * RegExps. - * A URL must match #1 and then at least one of #2/#3. - * Use two levels of REs to avoid REDOS. - */ - - var protocolAndDomainRE = /^(?:\w+:)?\/\/(\S+)$/; - - var localhostDomainRE = /^localhost[\:?\d]*(?:[^\:?\d]\S*)?$/; - var nonLocalhostDomainRE = /^[^\s\.]+\.\S{2,}$/; - - /** - * Loosely validate a URL `string`. - * - * @param {String} string - * @return {Boolean} - */ - - function isUrl(string){ - if (typeof string !== 'string') { - return false; - } - - var match = string.match(protocolAndDomainRE); - if (!match) { - return false; - } - - var everythingAfterProtocol = match[1]; - if (!everythingAfterProtocol) { - return false; - } - - if (localhostDomainRE.test(everythingAfterProtocol) || - nonLocalhostDomainRE.test(everythingAfterProtocol)) { - return true; - } - - return false; - } - return isUrl_1; -} - -/** - * @fileOverview - * is2 derived from is by Enrico Marino, adapted for Node.js. - * Slightly modified by Edmond Meinfelder - * - * is - * the definitive JavaScript type testing library - * Copyright(c) 2013,2014 Edmond Meinfelder - * Copyright(c) 2011 Enrico Marino - * MIT license - */ - -(function (exports) { - const owns = {}.hasOwnProperty; - const toString = {}.toString; - const is = exports; - const deepIs = deepIsExports; - const ipRegEx = ipRegex; - is.version = require$$2.version; - - //////////////////////////////////////////////////////////////////////////////// - // Environment - - /** - * Tests if is is running under a browser. - * @return {Boolean} true if the environment has process, process.version and process.versions. - */ - is.browser = function() { - return (!is.node() && typeof window !== 'undefined' && toString.call(window) === '[object global]'); - }; - - /** - * Test if 'value' is defined. - * Alias: def - * @param {Any} value The value to test. - * @return {Boolean} true if 'value' is defined, false otherwise. - */ - is.defined = function(value) { - return typeof value !== 'undefined'; - }; - is.def = is.defined; - - /** - * Tests if is is running under node.js - * @return {Boolean} true if the environment has process, process.version and process.versions. - */ - is.nodejs = function() { - return (process && process.hasOwnProperty('version') && - process.hasOwnProperty('versions')); - }; - is.node = is.nodejs; - - /** - * Test if 'value' is undefined. - * Aliases: undef, udef - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is undefined, false otherwise. - */ - is.undefined = function(value) { - return value === undefined; - }; - is.udef = is.undef = is.undefined; - - - //////////////////////////////////////////////////////////////////////////////// - // Types - - /** - * Test if 'value' is an array. - * Alias: ary, arry - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is an array, false otherwise. - */ - is.array = function(value) { - return '[object Array]' === toString.call(value); - }; - is.arr = is.ary = is.arry = is.array; - - /** - * Test if 'value' is an arraylike object (i.e. it has a length property with a valid value) - * Aliases: arraylike, arryLike, aryLike - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is an arguments object, false otherwise. - */ - is.arrayLike = function(value) { - if (is.nullOrUndef(value)) - return false; - return value !== undefined && - owns.call(value, 'length') && - isFinite(value.length); - }; - is.arrLike = is.arryLike = is.aryLike = is.arraylike = is.arrayLike; - - /** - * Test if 'value' is an arguments object. - * Alias: args - * @param {Any} value value to test - * @return {Boolean} true if 'value' is an arguments object, false otherwise - */ - is.arguments = function(value) { - return '[object Arguments]' === toString.call(value); - }; - is.args = is.arguments; - - /** - * Test if 'value' is a boolean. - * Alias: bool - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is a boolean, false otherwise. - */ - is.boolean = function(value) { - return '[object Boolean]' === toString.call(value); - }; - is.bool = is.boolean; - - /** - * Test if 'value' is an instance of Buffer. - * Aliases: instOf, instanceof - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is an instance of 'constructor'. - */ - is.buffer = function(value) { - return is.nodejs() && Buffer && Buffer.hasOwnProperty('isBuffer') && Buffer.isBuffer(value); - }; - is.buff = is.buf = is.buffer; - - /** - * Test if 'value' is a date. - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is a date, false otherwise. - */ - is.date = function(value) { - return '[object Date]' === toString.call(value); - }; - - /** - * Test if 'value' is an error object. - * Alias: err - * @param value value to test. - * @return {Boolean} true if 'value' is an error object, false otherwise. - */ - is.error = function(value) { - return '[object Error]' === toString.call(value); - }; - is.err = is.error; - - /** - * Test if 'value' is false. - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is false, false otherwise - */ - is.false = function(value) { - return value === false; - }; - - /** - * Test if 'value' is a function or async function. - * Alias: func - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is a function, false otherwise. - */ - is.function = function(value) { - return is.syncFunction(value) || is.asyncFunction(value) - }; - is.fun = is.func = is.function; - - /** - * Test if 'value' is an async function using `async () => {}` or `async function () {}`. - * Alias: func - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is a function, false otherwise. - */ - is.asyncFunction = function(value) { - return '[object AsyncFunction]' === toString.call(value); - }; - is.asyncFun = is.asyncFunc = is.asyncFunction; - - /** - * Test if 'value' is a synchronous function. - * Alias: syncFunc - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is a function, false otherwise. - */ - is.syncFunction = function (value) { - return '[object Function]' === toString.call(value); - }; - is.syncFun = is.syncFunc = is.syncFunction; - /** - * Test if 'value' is null. - * @param {Any} value to test. - * @return {Boolean} true if 'value' is null, false otherwise. - */ - is.null = function(value) { - return value === null; - }; - - /** - * Test is 'value' is either null or undefined. - * Alias: nullOrUndef - * @param {Any} value value to test. - * @return {Boolean} True if value is null or undefined, false otherwise. - */ - is.nullOrUndefined = function(value) { - return value === null || typeof value === 'undefined'; - }; - is.nullOrUndef = is.nullOrUndefined; - - /** - * Test if 'value' is a number. - * Alias: num - * @param {Any} value to test. - * @return {Boolean} true if 'value' is a number, false otherwise. - */ - is.number = function(value) { - return '[object Number]' === toString.call(value); - }; - is.num = is.number; - - /** - * Test if 'value' is an object. Note: Arrays, RegExps, Date, Error, etc all return false. - * Alias: obj - * @param {Any} value to test. - * @return {Boolean} true if 'value' is an object, false otherwise. - */ - is.object = function(value) { - return '[object Object]' === toString.call(value); - }; - is.obj = is.object; - - /** - * Test if 'value' is a regular expression. - * Alias: regexp - * @param {Any} value to test. - * @return {Boolean} true if 'value' is a regexp, false otherwise. - */ - is.regExp = function(value) { - return '[object RegExp]' === toString.call(value); - }; - is.re = is.regexp = is.regExp; - - /** - * Test if 'value' is a string. - * Alias: str - * @param {Any} value to test. - * @return {Boolean} true if 'value' is a string, false otherwise. - */ - is.string = function(value) { - return '[object String]' === toString.call(value); - }; - is.str = is.string; - - /** - * Test if 'value' is true. - * @param {Any} value to test. - * @return {Boolean} true if 'value' is true, false otherwise. - */ - is.true = function(value) { - return value === true; - }; - - /** - * Test if 'value' is a uuid (v1-v5) - * @param {Any} value to test. - * @return {Boolean} true if 'value is a valid RFC4122 UUID. Case non-specific. - */ - var uuidRegExp = new RegExp('[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab]'+ - '[0-9a-f]{3}-[0-9a-f]{12}', 'i'); - is.uuid = function(value) { - return uuidRegExp.test(value); - }; - - //////////////////////////////////////////////////////////////////////////////// - // Object Relationships - - /** - * Test if 'value' is equal to 'other'. Works for objects and arrays and will do deep comparisions, - * using recursion. - * Alias: eq - * @param {Any} value value. - * @param {Any} other value to compare with. - * @return {Boolean} true if 'value' is equal to 'other', false otherwise - */ - is.equal = function(value, other) { - var type = toString.call(value); - - if (typeof value !== typeof other) { - return false; - } - - if (type !== toString.call(other)) { - return false; - } - - if ('[object Object]' === type || '[object Array]' === type) { - return deepIs(value, other); - } else if ('[object Function]' === type) { - return value.prototype === other.prototype; - } else if ('[object Date]' === type) { - return value.getTime() === other.getTime(); - } - - return value === other; - }; - is.objEquals = is.eq = is.equal; - - /** - * JS Type definitions which cannot host values. - * @api private - */ - var NON_HOST_TYPES = { - 'boolean': 1, - 'number': 1, - 'string': 1, - 'undefined': 1 - }; - - /** - * Test if 'key' in host is an object. To be hosted means host[value] is an object. - * @param {Any} value The value to test. - * @param {Any} host Host that may contain value. - * @return {Boolean} true if 'value' is hosted by 'host', false otherwise. - */ - is.hosted = function(value, host) { - if (is.nullOrUndef(value)) - return false; - var type = typeof host[value]; - return type === 'object' ? !!host[value] : !NON_HOST_TYPES[type]; - }; - - /** - * Test if 'value' is an instance of 'constructor'. - * Aliases: instOf, instanceof - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is an instance of 'constructor'. - */ - is.instanceOf = function(value, constructor) { - if (is.nullOrUndef(value) || is.nullOrUndef(constructor)) - return false; - return (value instanceof constructor); - }; - is.instOf = is.instanceof = is.instanceOf; - - /** - * Test if 'value' is an instance type objType. - * Aliases: objInstOf, objectinstanceof, instOf, instanceOf - * @param {object} objInst an object to testfor type. - * @param {object} objType an object type to compare. - * @return {Boolean} true if 'value' is an object, false otherwise. - */ - is.objectInstanceOf = function(objInst, objType) { - try { - return '[object Object]' === toString.call(objInst) && (objInst instanceof objType); - } catch(err) { - return false; - } - }; - is.instOf = is.instanceOf = is.objInstOf = is.objectInstanceOf; - - /** - * Test if 'value' is a type of 'type'. - * Alias: a - * @param value value to test. - * @param {String} type The name of the type. - * @return {Boolean} true if 'value' is an arguments object, false otherwise. - */ - is.type = function(value, type) { - return typeof value === type; - }; - is.a = is.type; - - //////////////////////////////////////////////////////////////////////////////// - // Object State - - /** - * Test if 'value' is empty. To be empty means to be an array, object or string with nothing contained. - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is empty, false otherwise. - */ - is.empty = function(value) { - var type = toString.call(value); - - if ('[object Array]' === type || '[object Arguments]' === type) { - return value.length === 0; - } - - if ('[object Object]' === type) { - for (var key in value) if (owns.call(value, key)) return false; - return true; - } - - if ('[object String]' === type) { - return value === ''; - } - - return false; - }; - - /** - * Test if 'value' is an arguments object that is empty. - * Alias: args - * @param {Any} value value to test - * @return {Boolean} true if 'value' is an arguments object with no args, false otherwise - */ - is.emptyArguments = function(value) { - return '[object Arguments]' === toString.call(value) && value.length === 0; - }; - is.noArgs = is.emptyArgs = is.emptyArguments; - - /** - * Test if 'value' is an array containing no entries. - * Aliases: emptyArry, emptyAry - * @param {Any} value The value to test. - * @return {Boolean} true if 'value' is an array with no elemnets. - */ - is.emptyArray = function(value) { - return '[object Array]' === toString.call(value) && value.length === 0; - }; - is.emptyArry = is.emptyAry = is.emptyArray; - - /** - * Test if 'value' is an empty array(like) object. - * Aliases: arguents.empty, args.empty, ary.empty, arry.empty - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is an empty array(like), false otherwise. - */ - is.emptyArrayLike = function(value) { - return value.length === 0; - }; - is.emptyArrLike = is.emptyArrayLike; - - /** - * Test if 'value' is an empty string. - * Alias: emptyStr - * @param {Any} value to test. - * @return {Boolean} true if 'value' is am empty string, false otherwise. - */ - is.emptyString = function(value) { - return is.string(value) && value.length === 0; - }; - is.emptyStr = is.emptyString; - - /** - * Test if 'value' is an array containing at least 1 entry. - * Aliases: nonEmptyArry, nonEmptyAry - * @param {Any} value The value to test. - * @return {Boolean} true if 'value' is an array with at least 1 value, false otherwise. - */ - is.nonEmptyArray = function(value) { - return '[object Array]' === toString.call(value) && value.length > 0; - }; - is.nonEmptyArr = is.nonEmptyArry = is.nonEmptyAry = is.nonEmptyArray; - - /** - * Test if 'value' is an object with properties. Note: Arrays are objects. - * Alias: nonEmptyObj - * @param {Any} value to test. - * @return {Boolean} true if 'value' is an object, false otherwise. - */ - is.nonEmptyObject = function(value) { - return '[object Object]' === toString.call(value) && Object.keys(value).length > 0; - }; - is.nonEmptyObj = is.nonEmptyObject; - - /** - * Test if 'value' is an object with no properties. Note: Arrays are objects. - * Alias: nonEmptyObj - * @param {Any} value to test. - * @return {Boolean} true if 'value' is an object, false otherwise. - */ - is.emptyObject = function(value) { - return '[object Object]' === toString.call(value) && Object.keys(value).length === 0; - }; - is.emptyObj = is.emptyObject; - - /** - * Test if 'value' is a non-empty string. - * Alias: nonEmptyStr - * @param {Any} value to test. - * @return {Boolean} true if 'value' is a non-empty string, false otherwise. - */ - is.nonEmptyString = function(value) { - return is.string(value) && value.length > 0; - }; - is.nonEmptyStr = is.nonEmptyString; - - //////////////////////////////////////////////////////////////////////////////// - // Numeric Types within Number - - /** - * Test if 'value' is an even number. - * @param {Number} value to test. - * @return {Boolean} true if 'value' is an even number, false otherwise. - */ - is.even = function(value) { - return '[object Number]' === toString.call(value) && value % 2 === 0; - }; - - /** - * Test if 'value' is a decimal number. - * Aliases: decimalNumber, decNum - * @param {Any} value value to test. - * @return {Boolean} true if 'value' is a decimal number, false otherwise. - */ - is.decimal = function(value) { - return '[object Number]' === toString.call(value) && value % 1 !== 0; - }; - is.dec = is.decNum = is.decimal; - - /** - * Test if 'value' is an integer. - * Alias: integer - * @param {Any} value to test. - * @return {Boolean} true if 'value' is an integer, false otherwise. - */ - is.integer = function(value) { - return '[object Number]' === toString.call(value) && value % 1 === 0; - }; - is.int = is.integer; - - /** - * is.nan - * Test if `value` is not a number. - * - * @param {Mixed} value value to test - * @return {Boolean} true if `value` is not a number, false otherwise - * @api public - */ - is.notANumber = function(value) { - return !is.num(value) || value !== value; - }; - is.nan = is.notANum = is.notANumber; - - /** - * Test if 'value' is an odd number. - * @param {Number} value to test. - * @return {Boolean} true if 'value' is an odd number, false otherwise. - */ - is.odd = function(value) { - return !is.decimal(value) && '[object Number]' === toString.call(value) && value % 2 !== 0; - }; - is.oddNumber = is.oddNum = is.odd; - - //////////////////////////////////////////////////////////////////////////////// - // Numeric Type & State - - /** - * Test if 'value' is a positive number. - * Alias: positiveNum, posNum - * @param {Any} value to test. - * @return {Boolean} true if 'value' is a number, false otherwise. - */ - is.positiveNumber = function(value) { - return '[object Number]' === toString.call(value) && value > 0; - }; - is.pos = is.positive = is.posNum = is.positiveNum = is.positiveNumber; - - /** - * Test if 'value' is a negative number. - * Aliases: negNum, negativeNum - * @param {Any} value to test. - * @return {Boolean} true if 'value' is a number, false otherwise. - */ - is.negativeNumber = function(value) { - return '[object Number]' === toString.call(value) && value < 0; - }; - is.neg = is.negNum = is.negativeNum = is.negativeNumber; - - /** - * Test if 'value' is a negative integer. - * Aliases: negInt, negativeInteger - * @param {Any} value to test. - * @return {Boolean} true if 'value' is a negative integer, false otherwise. - */ - is.negativeInteger = function(value) { - return '[object Number]' === toString.call(value) && value % 1 === 0 && value < 0; - }; - is.negativeInt = is.negInt = is.negativeInteger; - - /** - * Test if 'value' is a positive integer. - * Alias: posInt - * @param {Any} value to test. - * @return {Boolean} true if 'value' is a positive integer, false otherwise. - */ - is.positiveInteger = function(value) { - return '[object Number]' === toString.call(value) && value % 1 === 0 && value > 0; - }; - is.posInt = is.positiveInt = is.positiveInteger; - - //////////////////////////////////////////////////////////////////////////////// - // Numeric Relationships - - /** - * Test if 'value' is divisible by 'n'. - * Alias: divisBy - * @param {Number} value value to test. - * @param {Number} n dividend. - * @return {Boolean} true if 'value' is divisible by 'n', false otherwise. - */ - is.divisibleBy = function(value, n) { - if (value === 0) - return false; - return '[object Number]' === toString.call(value) && - n !== 0 && - value % n === 0; - }; - is.divBy = is.divisBy = is.divisibleBy; - - /** - * Test if 'value' is greater than or equal to 'other'. - * Aliases: greaterOrEq, greaterOrEqual - * @param {Number} value value to test. - * @param {Number} other value to compare with. - * @return {Boolean} true, if value is greater than or equal to other, false otherwise. - */ - is.greaterOrEqualTo = function(value, other) { - return value >= other; - }; - is.greaterOrEqual = is.ge = is.greaterOrEqualTo; - - /** - * Test if 'value' is greater than 'other'. - * Aliases: greaterThan - * @param {Number} value value to test. - * @param {Number} other value to compare with. - * @return {Boolean} true, if value is greater than other, false otherwise. - */ - is.greaterThan = function(value, other) { - return value > other; - }; - is.gt = is.greaterThan; - - /** - * Test if 'value' is less than or equal to 'other'. - * Alias: lessThanOrEq, lessThanOrEqual - * @param {Number} value value to test - * @param {Number} other value to compare with - * @return {Boolean} true, if 'value' is less than or equal to 'other', false otherwise. - */ - is.lessThanOrEqualTo = function(value, other) { - return value <= other; - }; - is.lessThanOrEq = is.lessThanOrEqual = is.le = is.lessThanOrEqualTo; - - /** - * Test if 'value' is less than 'other'. - * Alias: lessThan - * @param {Number} value value to test - * @param {Number} other value to compare with - * @return {Boolean} true, if 'value' is less than 'other', false otherwise. - */ - is.lessThan = function(value, other) { - return value < other; - }; - is.lt = is.lessThan; - - /** - * Test if 'value' is greater than 'others' values. - * Alias: max - * @param {Number} value value to test. - * @param {Array} others values to compare with. - * @return {Boolean} true if 'value' is greater than 'others' values. - */ - is.maximum = function(value, others) { - if (!is.arrayLike(others) || !is.number(value)) - return false; - - var len = others.length; - while (--len > -1) { - if (value < others[len]) { - return false; - } - } - - return true; - }; - is.max = is.maximum; - - /** - * Test if 'value' is less than 'others' values. - * Alias: min - * @param {Number} value value to test. - * @param {Array} others values to compare with. - * @return {Boolean} true if 'value' is less than 'others' values. - */ - is.minimum = function(value, others) { - if (!is.arrayLike(others) || !is.number(value)) - return false; - - var len = others.length; - while (--len > -1) { - if (value > others[len]) { - return false; - } - } - - return true; - }; - is.min = is.minimum; - - /** - * Test if 'value' is within 'start' and 'finish'. - * Alias: withIn - * @param {Number} value value to test. - * @param {Number} start lower bound. - * @param {Number} finish upper bound. - * @return {Boolean} true if 'value' is is within 'start' and 'finish', false otherwise. - */ - is.within = function(value, start, finish) { - return value >= start && value <= finish; - }; - is.withIn = is.within; - - /** - * Test if 'value' is within 'precision' decimal places from 'comparitor'. - * Alias: closish, near. - * @param {Number} value value to test - * @param {Number} comparitor value to test 'value' against - * @param {Number} precision number of decimals to compare floating points, defaults to 2 - * @return {Boolean} true if 'value' is within 'precision' decimal places from 'comparitor', false otherwise. - */ - is.prettyClose = function(value, comparitor, precision) { - if (!is.number(value) || !is.number(comparitor)) return false; - if (is.defined(precision) && !is.posInt(precision)) return false; - if (is.undefined(precision)) precision = 2; - - return value.toFixed(precision) === comparitor.toFixed(precision); - }; - is.closish = is.near = is.prettyClose; - //////////////////////////////////////////////////////////////////////////////// - // Networking - - /** - * Test if a value is a valid DNS address. eg www.stdarg.com is true while - * 127.0.0.1 is false. - * @param {Any} value to test if a DNS address. - * @return {Boolean} true if a DNS address, false otherwise. - * DNS Address is made up of labels separated by '.' - * Each label must be between 1 and 63 characters long - * The entire hostname (including the delimiting dots) has a maximum of 255 characters. - * Hostname may not contain other characters, such as the underscore character (_) - * other DNS names may contain the underscore. - */ - is.dnsAddress = function(value) { - if (!is.nonEmptyStr(value)) return false; - if (value.length > 255) return false; - if (numbersLabel.test(value)) return false; - if (!dnsLabel.test(value)) return false; - return true; - //var names = value.split('.'); - //if (!is.array(names) || !names.length) return false; - //if (names[0].indexOf('_') > -1) return false; - //for (var i=0; i 15) return false; - var octets = value.split('.'); - if (!is.array(octets) || octets.length !== 4) return false; - for (var i=0; i 255) return false; - } - return true; - }; - is.ipv4 = is.ipv4Addr = is.ipv4Address; - - /** - * Test if a value is either an IPv6 numeric IP address. - * @param {Any} value to test if an ip address. - * @return {Boolean} true if an ip address, false otherwise. - */ - is.ipv6Address = function(value) { - if (!is.nonEmptyStr(value)) return false; - return ipRegEx.v6({extract: true}).test(value); - }; - is.ipv6 = is.ipv6Addr = is.ipv6Address; - - /** - * Test if a value is either an IPv4 or IPv6 numeric IP address. - * @param {Any} value to test if an ip address. - * @return {Boolean} true if an ip address, false otherwise. - */ - is.ipAddress = function(value) { - if (!is.nonEmptyStr(value)) return false; - return is.ipv4Address(value) || is.ipv6Address(value) - }; - is.ip = is.ipAddr = is.ipAddress; - - /** - * Test is a value is a valid ipv4, ipv6 or DNS name. - * Aliases: host, hostAddr, hostAddress. - * @param {Any} value to test if a host address. - * @return {Boolean} true if a host address, false otherwise. - */ - is.hostAddress = function(value) { - if (!is.nonEmptyStr(value)) return false; - return is.dns(value) || is.ipv4(value) || is.ipv6(value); - }; - is.host = is.hostIp = is.hostAddr = is.hostAddress; - - /** - * Test if a number is a valid TCP port - * @param {Any} value to test if its a valid TCP port - */ - is.port = function(value) { - if (!is.num(value) || is.negativeInt(value) || value > 65535) - return false; - return true; - }; - - /** - * Test if a number is a valid TCP port in the range 0-1023. - * Alias: is.sysPort. - * @param {Any} value to test if its a valid TCP port - */ - is.systemPort = function(value) { - if (is.port(value) && value < 1024) - return true; - return false; - }; - is.sysPort = is.systemPort; - - /** - * Test if a number is a valid TCP port in the range 1024-65535. - * @param {Any} value to test if its a valid TCP port - */ - is.userPort = function(value) { - if (is.port(value) && value > 1023) - return true; - return false; - }; - - /* - function sumDigits(num) { - var str = num.toString(); - var sum = 0; - for (var i = 0; i < str.length; i++) - sum += (str[i]-0); - return sum; - } - */ - - /** - * Test if a string is a credit card. - * From http://en.wikipedia.org/wiki/Luhn_algorithm - * @param {String} value to test if a credit card. - * @return true if the string is the correct format, false otherwise - */ - is.creditCardNumber = function(str) { - if (!is.str(str)) - return false; - - var ary = str.split(''); - var i, cnt; - // From the rightmost digit, which is the check digit, moving left, double - // the value of every second digit; - for (i=ary.length-1, cnt=1; i>-1; i--, cnt++) { - if (cnt%2 === 0) - ary[i] *= 2; - } - - str = ary.join(''); - var sum = 0; - // if the product of the previous doubling operation is greater than 9 - // (e.g., 7 * 2 = 14), then sum the digits of the products (e.g., 10: 1 + 0 - // = 1, 14: 1 + 4 = 5). We do the this by joining the array of numbers and - // add adding the int value of all the characters in the string. - for (i=0; i 19)) - return false; - - var prefix = Math.floor(str.slice(0,2)); - if (prefix !== 62 && prefix !== 88) - return false; - - // no validation for this card - return true; - }; - is.chinaUnion = is.chinaUnionPayCard = is.chinaUnionPayCardNumber; - - /** - * Test if card number is a Diner's Club Carte Blance card. - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.dinersClubCarteBlancheCardNumber = function(str) { - if (!is.str(str) || str.length !== 14) - return false; - - var prefix = Math.floor(str.slice(0,3)); - if (prefix < 300 || prefix > 305) - return false; - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - is.dinersClubCB = is.dinersClubCarteBlancheCard = - is.dinersClubCarteBlancheCardNumber; - - /** - * Test if card number is a Diner's Club International card. - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.dinersClubInternationalCardNumber = function(str) { - if (!is.str(str) || str.length !== 14) - return false; - var prefix = Math.floor(str.slice(0,3)); - var prefix2 = Math.floor(str.slice(0,2)); - - // 300-305, 309, 36, 38-39 - if ((prefix < 300 || prefix > 305) && prefix !== 309 && prefix2 !== 36 && - (prefix2 < 38 || prefix2 > 39)) { - return false; - } - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - is.dinersClubInt = is.dinersClubInternationalCard = - is.dinersClubInternationalCardNumber; - - /** - * Test if card number is a Diner's Club USA & CA card. - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.dinersClubUSACanadaCardNumber = function(str) { - if (!is.str(str) || str.length !== 16) - return false; - var prefix = Math.floor(str.slice(0,2)); - - if (prefix !== 54 && prefix !== 55) - return false; - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - is.dinersClub = is.dinersClubUSACanCard = is.dinersClubUSACanadaCardNumber; - - /** - * Test if card number is a Diner's Club USA/CA card. - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.discoverCardNumber = function(str) { - if (!is.str(str) || str.length !== 16) - return false; - - var prefix = Math.floor(str.slice(0,6)); - var prefix2 = Math.floor(str.slice(0,3)); - - if (str.slice(0,4) !== '6011' && (prefix < 622126 || prefix > 622925) && - (prefix2 < 644 || prefix2 > 649) && str.slice(0,2) !== '65') { - return false; - } - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - is.discover = is.discoverCard = is.discoverCardNumber; - - /** - * Test if card number is an InstaPayment card number - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.instaPaymentCardNumber = function(str) { - if (!is.str(str) || str.length !== 16) - return false; - - var prefix = Math.floor(str.slice(0,3)); - if (prefix < 637 || prefix > 639) - return false; - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - is.instaPayment = is.instaPaymentCardNumber; - - /** - * Test if card number is a JCB card number - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.jcbCardNumber = function(str) { - if (!is.str(str) || str.length !== 16) - return false; - - var prefix = Math.floor(str.slice(0,4)); - if (prefix < 3528 || prefix > 3589) - return false; - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - is.jcb = is.jcbCard = is.jcbCardNumber; - - /** - * Test if card number is a Laser card number - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.laserCardNumber = function(str) { - if (!is.str(str) || (str.length < 16 && str.length > 19)) - return false; - - var prefix = Math.floor(str.slice(0,4)); - var valid = [ 6304, 6706, 6771, 6709 ]; - if (valid.indexOf(prefix) === -1) - return false; - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - is.laser = is.laserCard = is.laserCardNumber; - - /** - * Test if card number is a Maestro card number - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.maestroCardNumber = function(str) { - if (!is.str(str) || str.length < 12 || str.length > 19) - return false; - - var prefix = str.slice(0,4); - var valid = [ '5018', '5020', '5038', '5612', '5893', '6304', '6759', - '6761', '6762', '6763', '0604', '6390' ]; - - if (valid.indexOf(prefix) === -1) - return false; - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - is.maestro = is.maestroCard = is.maestroCardNumber; - - /** - * Test if card number is a Dankort card number - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.dankortCardNumber = function(str) { - if (!is.str(str) || str.length !== 16) - return false; - - if (str.slice(0,4) !== '5019') - return false; - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - is.dankort = is.dankortCard = is.dankortCardNumber; - - /** - * Test if card number is a MasterCard card number - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.masterCardCardNumber = function(str) { - if (!is.str(str) || str.length !== 16) - return false; - - var prefix = Math.floor(str.slice(0,2)); - if (prefix < 50 || prefix > 55) - return false; - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - is.masterCard = is.masterCardCard = is.masterCardCardNumber; - - /** - * Test if card number is a Visa card number - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.visaCardNumber = function(str) { - if (!is.str(str) || (str.length !== 13 && str.length !== 16)) - return false; - - if ('4' !== str.slice(0,1)) - return false; - - if (!is.creditCardNumber(str)) - return false; - - return true; - }; - - is.visa = is.visaCard = is.visaCardNumber; - - /** - * Test if card number is a Visa card number - * @param {String} the credit card number string to test. - * @return true if the string is the correct format, false otherwise - */ - is.visaElectronCardNumber = function(str) { - if (!is.str(str) || str.length !== 16) - return false; - - var prefix = Math.floor(str.slice(0,4)); - var valid = [ 4026, 4405, 4508, 4844, 4913, 4917 ]; - if ('417500' !== str.slice(0,6) && valid.indexOf(prefix) === -1) - return false; - - if (!is.creditCardNumber(str)) - return false; - - return false; - }; - - is.visaElectron = is.visaElectronCard = is.visaElectronCardNumber; - - /** - * Test if the input is a valid MongoDB id. - * @param {String|Object} Either a mongodb object id or a string representation. - * @return true if the string is the correct format, false otherwise - * Thanks to Jason Denizac (https://github.com/jden) for pointing this out. - * https://github.com/jden/objectid/blob/master/index.js#L7-L10 - */ - var objIdPattern = /^[0-9a-fA-F]{24}$/; - is.mongoId = is.objectId = is.objId = function(id) { - return (Boolean(id) && !Array.isArray(id) && objIdPattern.test(String(id))); - }; - - /** - * Test is the first argument is structly equal to any of the subsequent args. - * @param Value to test against subsequent arguments. - * @return true if the first value matches any of subsequent values. - */ - is.matching = is.match = is.inArgs = function(val) { - if (arguments.length < 2) - return false; - var result = false; - for (var i=1; i 0) { - return parse(val); - } else if (type === 'number' && isFinite(val)) { - return options.long ? fmtLong(val) : fmtShort(val); - } - throw new Error( - 'val is not a non-empty string or a valid number. val=' + - JSON.stringify(val) - ); - }; - - /** - * Parse the given `str` and return milliseconds. - * - * @param {String} str - * @return {Number} - * @api private - */ - - function parse(str) { - str = String(str); - if (str.length > 100) { - return; - } - var match = /^(-?(?:\d+)?\.?\d+) *(milliseconds?|msecs?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d|weeks?|w|years?|yrs?|y)?$/i.exec( - str - ); - if (!match) { - return; - } - var n = parseFloat(match[1]); - var type = (match[2] || 'ms').toLowerCase(); - switch (type) { - case 'years': - case 'year': - case 'yrs': - case 'yr': - case 'y': - return n * y; - case 'weeks': - case 'week': - case 'w': - return n * w; - case 'days': - case 'day': - case 'd': - return n * d; - case 'hours': - case 'hour': - case 'hrs': - case 'hr': - case 'h': - return n * h; - case 'minutes': - case 'minute': - case 'mins': - case 'min': - case 'm': - return n * m; - case 'seconds': - case 'second': - case 'secs': - case 'sec': - case 's': - return n * s; - case 'milliseconds': - case 'millisecond': - case 'msecs': - case 'msec': - case 'ms': - return n; - default: - return undefined; - } - } - - /** - * Short format for `ms`. - * - * @param {Number} ms - * @return {String} - * @api private - */ - - function fmtShort(ms) { - var msAbs = Math.abs(ms); - if (msAbs >= d) { - return Math.round(ms / d) + 'd'; - } - if (msAbs >= h) { - return Math.round(ms / h) + 'h'; - } - if (msAbs >= m) { - return Math.round(ms / m) + 'm'; - } - if (msAbs >= s) { - return Math.round(ms / s) + 's'; - } - return ms + 'ms'; - } - - /** - * Long format for `ms`. - * - * @param {Number} ms - * @return {String} - * @api private - */ - - function fmtLong(ms) { - var msAbs = Math.abs(ms); - if (msAbs >= d) { - return plural(ms, msAbs, d, 'day'); - } - if (msAbs >= h) { - return plural(ms, msAbs, h, 'hour'); - } - if (msAbs >= m) { - return plural(ms, msAbs, m, 'minute'); - } - if (msAbs >= s) { - return plural(ms, msAbs, s, 'second'); - } - return ms + ' ms'; - } - - /** - * Pluralization helper. - */ - - function plural(ms, msAbs, n, name) { - var isPlural = msAbs >= n * 1.5; - return Math.round(ms / n) + ' ' + name + (isPlural ? 's' : ''); - } - return ms; -} - -var common; -var hasRequiredCommon; - -function requireCommon () { - if (hasRequiredCommon) return common; - hasRequiredCommon = 1; - /** - * This is the common logic for both the Node.js and web browser - * implementations of `debug()`. - */ - - function setup(env) { - createDebug.debug = createDebug; - createDebug.default = createDebug; - createDebug.coerce = coerce; - createDebug.disable = disable; - createDebug.enable = enable; - createDebug.enabled = enabled; - createDebug.humanize = requireMs(); - createDebug.destroy = destroy; - - Object.keys(env).forEach(key => { - createDebug[key] = env[key]; - }); - - /** - * The currently active debug mode names, and names to skip. - */ - - createDebug.names = []; - createDebug.skips = []; - - /** - * Map of special "%n" handling functions, for the debug "format" argument. - * - * Valid key names are a single, lower or upper-case letter, i.e. "n" and "N". - */ - createDebug.formatters = {}; - - /** - * Selects a color for a debug namespace - * @param {String} namespace The namespace string for the for the debug instance to be colored - * @return {Number|String} An ANSI color code for the given namespace - * @api private - */ - function selectColor(namespace) { - let hash = 0; - - for (let i = 0; i < namespace.length; i++) { - hash = ((hash << 5) - hash) + namespace.charCodeAt(i); - hash |= 0; // Convert to 32bit integer - } - - return createDebug.colors[Math.abs(hash) % createDebug.colors.length]; - } - createDebug.selectColor = selectColor; - - /** - * Create a debugger with the given `namespace`. - * - * @param {String} namespace - * @return {Function} - * @api public - */ - function createDebug(namespace) { - let prevTime; - let enableOverride = null; - - function debug(...args) { - // Disabled? - if (!debug.enabled) { - return; - } - - const self = debug; - - // Set `diff` timestamp - const curr = Number(new Date()); - const ms = curr - (prevTime || curr); - self.diff = ms; - self.prev = prevTime; - self.curr = curr; - prevTime = curr; - - args[0] = createDebug.coerce(args[0]); - - if (typeof args[0] !== 'string') { - // Anything else let's inspect with %O - args.unshift('%O'); - } - - // Apply any `formatters` transformations - let index = 0; - args[0] = args[0].replace(/%([a-zA-Z%])/g, (match, format) => { - // If we encounter an escaped % then don't increase the array index - if (match === '%%') { - return '%'; - } - index++; - const formatter = createDebug.formatters[format]; - if (typeof formatter === 'function') { - const val = args[index]; - match = formatter.call(self, val); - - // Now we need to remove `args[index]` since it's inlined in the `format` - args.splice(index, 1); - index--; - } - return match; - }); - - // Apply env-specific formatting (colors, etc.) - createDebug.formatArgs.call(self, args); - - const logFn = self.log || createDebug.log; - logFn.apply(self, args); - } - - debug.namespace = namespace; - debug.useColors = createDebug.useColors(); - debug.color = createDebug.selectColor(namespace); - debug.extend = extend; - debug.destroy = createDebug.destroy; // XXX Temporary. Will be removed in the next major release. - - Object.defineProperty(debug, 'enabled', { - enumerable: true, - configurable: false, - get: () => enableOverride === null ? createDebug.enabled(namespace) : enableOverride, - set: v => { - enableOverride = v; - } - }); - - // Env-specific initialization logic for debug instances - if (typeof createDebug.init === 'function') { - createDebug.init(debug); - } - - return debug; - } - - function extend(namespace, delimiter) { - const newDebug = createDebug(this.namespace + (typeof delimiter === 'undefined' ? ':' : delimiter) + namespace); - newDebug.log = this.log; - return newDebug; - } - - /** - * Enables a debug mode by namespaces. This can include modes - * separated by a colon and wildcards. - * - * @param {String} namespaces - * @api public - */ - function enable(namespaces) { - createDebug.save(namespaces); - - createDebug.names = []; - createDebug.skips = []; - - let i; - const split = (typeof namespaces === 'string' ? namespaces : '').split(/[\s,]+/); - const len = split.length; - - for (i = 0; i < len; i++) { - if (!split[i]) { - // ignore empty strings - continue; - } - - namespaces = split[i].replace(/\*/g, '.*?'); - - if (namespaces[0] === '-') { - createDebug.skips.push(new RegExp('^' + namespaces.substr(1) + '$')); - } else { - createDebug.names.push(new RegExp('^' + namespaces + '$')); - } - } - } - - /** - * Disable debug output. - * - * @return {String} namespaces - * @api public - */ - function disable() { - const namespaces = [ - ...createDebug.names.map(toNamespace), - ...createDebug.skips.map(toNamespace).map(namespace => '-' + namespace) - ].join(','); - createDebug.enable(''); - return namespaces; - } - - /** - * Returns true if the given mode name is enabled, false otherwise. - * - * @param {String} name - * @return {Boolean} - * @api public - */ - function enabled(name) { - if (name[name.length - 1] === '*') { - return true; - } - - let i; - let len; - - for (i = 0, len = createDebug.skips.length; i < len; i++) { - if (createDebug.skips[i].test(name)) { - return false; - } - } - - for (i = 0, len = createDebug.names.length; i < len; i++) { - if (createDebug.names[i].test(name)) { - return true; - } - } - - return false; - } - - /** - * Convert regexp to namespace - * - * @param {RegExp} regxep - * @return {String} namespace - * @api private - */ - function toNamespace(regexp) { - return regexp.toString() - .substring(2, regexp.toString().length - 2) - .replace(/\.\*\?$/, '*'); - } - - /** - * Coerce `val`. - * - * @param {Mixed} val - * @return {Mixed} - * @api private - */ - function coerce(val) { - if (val instanceof Error) { - return val.stack || val.message; - } - return val; - } - - /** - * XXX DO NOT USE. This is a temporary stub function. - * XXX It WILL be removed in the next major release. - */ - function destroy() { - console.warn('Instance method `debug.destroy()` is deprecated and no longer does anything. It will be removed in the next major version of `debug`.'); - } - - createDebug.enable(createDebug.load()); - - return createDebug; - } - - common = setup; - return common; -} - -/* eslint-env browser */ -browser.exports; - -var hasRequiredBrowser; - -function requireBrowser () { - if (hasRequiredBrowser) return browser.exports; - hasRequiredBrowser = 1; - (function (module, exports) { - /** - * This is the web browser implementation of `debug()`. - */ - - exports.formatArgs = formatArgs; - exports.save = save; - exports.load = load; - exports.useColors = useColors; - exports.storage = localstorage(); - exports.destroy = (() => { - let warned = false; - - return () => { - if (!warned) { - warned = true; - console.warn('Instance method `debug.destroy()` is deprecated and no longer does anything. It will be removed in the next major version of `debug`.'); - } - }; - })(); - - /** - * Colors. - */ - - exports.colors = [ - '#0000CC', - '#0000FF', - '#0033CC', - '#0033FF', - '#0066CC', - '#0066FF', - '#0099CC', - '#0099FF', - '#00CC00', - '#00CC33', - '#00CC66', - '#00CC99', - '#00CCCC', - '#00CCFF', - '#3300CC', - '#3300FF', - '#3333CC', - '#3333FF', - '#3366CC', - '#3366FF', - '#3399CC', - '#3399FF', - '#33CC00', - '#33CC33', - '#33CC66', - '#33CC99', - '#33CCCC', - '#33CCFF', - '#6600CC', - '#6600FF', - '#6633CC', - '#6633FF', - '#66CC00', - '#66CC33', - '#9900CC', - '#9900FF', - '#9933CC', - '#9933FF', - '#99CC00', - '#99CC33', - '#CC0000', - '#CC0033', - '#CC0066', - '#CC0099', - '#CC00CC', - '#CC00FF', - '#CC3300', - '#CC3333', - '#CC3366', - '#CC3399', - '#CC33CC', - '#CC33FF', - '#CC6600', - '#CC6633', - '#CC9900', - '#CC9933', - '#CCCC00', - '#CCCC33', - '#FF0000', - '#FF0033', - '#FF0066', - '#FF0099', - '#FF00CC', - '#FF00FF', - '#FF3300', - '#FF3333', - '#FF3366', - '#FF3399', - '#FF33CC', - '#FF33FF', - '#FF6600', - '#FF6633', - '#FF9900', - '#FF9933', - '#FFCC00', - '#FFCC33' - ]; - - /** - * Currently only WebKit-based Web Inspectors, Firefox >= v31, - * and the Firebug extension (any Firefox version) are known - * to support "%c" CSS customizations. - * - * TODO: add a `localStorage` variable to explicitly enable/disable colors - */ - - // eslint-disable-next-line complexity - function useColors() { - // NB: In an Electron preload script, document will be defined but not fully - // initialized. Since we know we're in Chrome, we'll just detect this case - // explicitly - if (typeof window !== 'undefined' && window.process && (window.process.type === 'renderer' || window.process.__nwjs)) { - return true; - } - - // Internet Explorer and Edge do not support colors. - if (typeof navigator !== 'undefined' && navigator.userAgent && navigator.userAgent.toLowerCase().match(/(edge|trident)\/(\d+)/)) { - return false; - } - - // Is webkit? http://stackoverflow.com/a/16459606/376773 - // document is undefined in react-native: https://github.com/facebook/react-native/pull/1632 - return (typeof document !== 'undefined' && document.documentElement && document.documentElement.style && document.documentElement.style.WebkitAppearance) || - // Is firebug? http://stackoverflow.com/a/398120/376773 - (typeof window !== 'undefined' && window.console && (window.console.firebug || (window.console.exception && window.console.table))) || - // Is firefox >= v31? - // https://developer.mozilla.org/en-US/docs/Tools/Web_Console#Styling_messages - (typeof navigator !== 'undefined' && navigator.userAgent && navigator.userAgent.toLowerCase().match(/firefox\/(\d+)/) && parseInt(RegExp.$1, 10) >= 31) || - // Double check webkit in userAgent just in case we are in a worker - (typeof navigator !== 'undefined' && navigator.userAgent && navigator.userAgent.toLowerCase().match(/applewebkit\/(\d+)/)); - } - - /** - * Colorize log arguments if enabled. - * - * @api public - */ - - function formatArgs(args) { - args[0] = (this.useColors ? '%c' : '') + - this.namespace + - (this.useColors ? ' %c' : ' ') + - args[0] + - (this.useColors ? '%c ' : ' ') + - '+' + module.exports.humanize(this.diff); - - if (!this.useColors) { - return; - } - - const c = 'color: ' + this.color; - args.splice(1, 0, c, 'color: inherit'); - - // The final "%c" is somewhat tricky, because there could be other - // arguments passed either before or after the %c, so we need to - // figure out the correct index to insert the CSS into - let index = 0; - let lastC = 0; - args[0].replace(/%[a-zA-Z%]/g, match => { - if (match === '%%') { - return; - } - index++; - if (match === '%c') { - // We only are interested in the *last* %c - // (the user may have provided their own) - lastC = index; - } - }); - - args.splice(lastC, 0, c); - } - - /** - * Invokes `console.debug()` when available. - * No-op when `console.debug` is not a "function". - * If `console.debug` is not available, falls back - * to `console.log`. - * - * @api public - */ - exports.log = console.debug || console.log || (() => {}); - - /** - * Save `namespaces`. - * - * @param {String} namespaces - * @api private - */ - function save(namespaces) { - try { - if (namespaces) { - exports.storage.setItem('debug', namespaces); - } else { - exports.storage.removeItem('debug'); - } - } catch (error) { - // Swallow - // XXX (@Qix-) should we be logging these? - } - } - - /** - * Load `namespaces`. - * - * @return {String} returns the previously persisted debug modes - * @api private - */ - function load() { - let r; - try { - r = exports.storage.getItem('debug'); - } catch (error) { - // Swallow - // XXX (@Qix-) should we be logging these? - } - - // If debug isn't set in LS, and we're in Electron, try to load $DEBUG - if (!r && typeof process !== 'undefined' && 'env' in process) { - r = process.env.DEBUG; - } - - return r; - } - - /** - * Localstorage attempts to return the localstorage. - * - * This is necessary because safari throws - * when a user disables cookies/localstorage - * and you attempt to access it. - * - * @return {LocalStorage} - * @api private - */ - - function localstorage() { - try { - // TVMLKit (Apple TV JS Runtime) does not have a window object, just localStorage in the global context - // The Browser also has localStorage in the global context. - return localStorage; - } catch (error) { - // Swallow - // XXX (@Qix-) should we be logging these? - } - } - - module.exports = requireCommon()(exports); - - const {formatters} = module.exports; - - /** - * Map %j to `JSON.stringify()`, since no Web Inspectors do that by default. - */ - - formatters.j = function (v) { - try { - return JSON.stringify(v); - } catch (error) { - return '[UnexpectedJSONParseError]: ' + error.message; - } - }; - } (browser, browser.exports)); - return browser.exports; -} - -var node = {exports: {}}; - -var hasFlag; -var hasRequiredHasFlag; - -function requireHasFlag () { - if (hasRequiredHasFlag) return hasFlag; - hasRequiredHasFlag = 1; - - hasFlag = (flag, argv = process.argv) => { - const prefix = flag.startsWith('-') ? '' : (flag.length === 1 ? '-' : '--'); - const position = argv.indexOf(prefix + flag); - const terminatorPosition = argv.indexOf('--'); - return position !== -1 && (terminatorPosition === -1 || position < terminatorPosition); - }; - return hasFlag; -} - -var supportsColor_1; -var hasRequiredSupportsColor; - -function requireSupportsColor () { - if (hasRequiredSupportsColor) return supportsColor_1; - hasRequiredSupportsColor = 1; - const os = require$$0__default["default"]; - const tty = require$$1__default["default"]; - const hasFlag = requireHasFlag(); - - const {env} = process; - - let forceColor; - if (hasFlag('no-color') || - hasFlag('no-colors') || - hasFlag('color=false') || - hasFlag('color=never')) { - forceColor = 0; - } else if (hasFlag('color') || - hasFlag('colors') || - hasFlag('color=true') || - hasFlag('color=always')) { - forceColor = 1; - } - - if ('FORCE_COLOR' in env) { - if (env.FORCE_COLOR === 'true') { - forceColor = 1; - } else if (env.FORCE_COLOR === 'false') { - forceColor = 0; - } else { - forceColor = env.FORCE_COLOR.length === 0 ? 1 : Math.min(parseInt(env.FORCE_COLOR, 10), 3); - } - } - - function translateLevel(level) { - if (level === 0) { - return false; - } - - return { - level, - hasBasic: true, - has256: level >= 2, - has16m: level >= 3 - }; - } - - function supportsColor(haveStream, streamIsTTY) { - if (forceColor === 0) { - return 0; - } - - if (hasFlag('color=16m') || - hasFlag('color=full') || - hasFlag('color=truecolor')) { - return 3; - } - - if (hasFlag('color=256')) { - return 2; - } - - if (haveStream && !streamIsTTY && forceColor === undefined) { - return 0; - } - - const min = forceColor || 0; - - if (env.TERM === 'dumb') { - return min; - } - - if (process.platform === 'win32') { - // Windows 10 build 10586 is the first Windows release that supports 256 colors. - // Windows 10 build 14931 is the first release that supports 16m/TrueColor. - const osRelease = os.release().split('.'); - if ( - Number(osRelease[0]) >= 10 && - Number(osRelease[2]) >= 10586 - ) { - return Number(osRelease[2]) >= 14931 ? 3 : 2; - } - - return 1; - } - - if ('CI' in env) { - if (['TRAVIS', 'CIRCLECI', 'APPVEYOR', 'GITLAB_CI', 'GITHUB_ACTIONS', 'BUILDKITE'].some(sign => sign in env) || env.CI_NAME === 'codeship') { - return 1; - } - - return min; - } - - if ('TEAMCITY_VERSION' in env) { - return /^(9\.(0*[1-9]\d*)\.|\d{2,}\.)/.test(env.TEAMCITY_VERSION) ? 1 : 0; - } - - if (env.COLORTERM === 'truecolor') { - return 3; - } - - if ('TERM_PROGRAM' in env) { - const version = parseInt((env.TERM_PROGRAM_VERSION || '').split('.')[0], 10); - - switch (env.TERM_PROGRAM) { - case 'iTerm.app': - return version >= 3 ? 3 : 2; - case 'Apple_Terminal': - return 2; - // No default - } - } - - if (/-256(color)?$/i.test(env.TERM)) { - return 2; - } - - if (/^screen|^xterm|^vt100|^vt220|^rxvt|color|ansi|cygwin|linux/i.test(env.TERM)) { - return 1; - } - - if ('COLORTERM' in env) { - return 1; - } - - return min; - } - - function getSupportLevel(stream) { - const level = supportsColor(stream, stream && stream.isTTY); - return translateLevel(level); - } - - supportsColor_1 = { - supportsColor: getSupportLevel, - stdout: translateLevel(supportsColor(true, tty.isatty(1))), - stderr: translateLevel(supportsColor(true, tty.isatty(2))) - }; - return supportsColor_1; -} - -/** - * Module dependencies. - */ -node.exports; - -var hasRequiredNode; - -function requireNode () { - if (hasRequiredNode) return node.exports; - hasRequiredNode = 1; - (function (module, exports) { - const tty = require$$1__default["default"]; - const util = require$$1__default$1["default"]; - - /** - * This is the Node.js implementation of `debug()`. - */ - - exports.init = init; - exports.log = log; - exports.formatArgs = formatArgs; - exports.save = save; - exports.load = load; - exports.useColors = useColors; - exports.destroy = util.deprecate( - () => {}, - 'Instance method `debug.destroy()` is deprecated and no longer does anything. It will be removed in the next major version of `debug`.' - ); - - /** - * Colors. - */ - - exports.colors = [6, 2, 3, 4, 5, 1]; - - try { - // Optional dependency (as in, doesn't need to be installed, NOT like optionalDependencies in package.json) - // eslint-disable-next-line import/no-extraneous-dependencies - const supportsColor = requireSupportsColor(); - - if (supportsColor && (supportsColor.stderr || supportsColor).level >= 2) { - exports.colors = [ - 20, - 21, - 26, - 27, - 32, - 33, - 38, - 39, - 40, - 41, - 42, - 43, - 44, - 45, - 56, - 57, - 62, - 63, - 68, - 69, - 74, - 75, - 76, - 77, - 78, - 79, - 80, - 81, - 92, - 93, - 98, - 99, - 112, - 113, - 128, - 129, - 134, - 135, - 148, - 149, - 160, - 161, - 162, - 163, - 164, - 165, - 166, - 167, - 168, - 169, - 170, - 171, - 172, - 173, - 178, - 179, - 184, - 185, - 196, - 197, - 198, - 199, - 200, - 201, - 202, - 203, - 204, - 205, - 206, - 207, - 208, - 209, - 214, - 215, - 220, - 221 - ]; - } - } catch (error) { - // Swallow - we only care if `supports-color` is available; it doesn't have to be. - } - - /** - * Build up the default `inspectOpts` object from the environment variables. - * - * $ DEBUG_COLORS=no DEBUG_DEPTH=10 DEBUG_SHOW_HIDDEN=enabled node script.js - */ - - exports.inspectOpts = Object.keys(process.env).filter(key => { - return /^debug_/i.test(key); - }).reduce((obj, key) => { - // Camel-case - const prop = key - .substring(6) - .toLowerCase() - .replace(/_([a-z])/g, (_, k) => { - return k.toUpperCase(); - }); - - // Coerce string value into JS value - let val = process.env[key]; - if (/^(yes|on|true|enabled)$/i.test(val)) { - val = true; - } else if (/^(no|off|false|disabled)$/i.test(val)) { - val = false; - } else if (val === 'null') { - val = null; - } else { - val = Number(val); - } - - obj[prop] = val; - return obj; - }, {}); - - /** - * Is stdout a TTY? Colored output is enabled when `true`. - */ - - function useColors() { - return 'colors' in exports.inspectOpts ? - Boolean(exports.inspectOpts.colors) : - tty.isatty(process.stderr.fd); - } - - /** - * Adds ANSI color escape codes if enabled. - * - * @api public - */ - - function formatArgs(args) { - const {namespace: name, useColors} = this; - - if (useColors) { - const c = this.color; - const colorCode = '\u001B[3' + (c < 8 ? c : '8;5;' + c); - const prefix = ` ${colorCode};1m${name} \u001B[0m`; - - args[0] = prefix + args[0].split('\n').join('\n' + prefix); - args.push(colorCode + 'm+' + module.exports.humanize(this.diff) + '\u001B[0m'); - } else { - args[0] = getDate() + name + ' ' + args[0]; - } - } - - function getDate() { - if (exports.inspectOpts.hideDate) { - return ''; - } - return new Date().toISOString() + ' '; - } - - /** - * Invokes `util.format()` with the specified arguments and writes to stderr. - */ - - function log(...args) { - return process.stderr.write(util.format(...args) + '\n'); - } - - /** - * Save `namespaces`. - * - * @param {String} namespaces - * @api private - */ - function save(namespaces) { - if (namespaces) { - process.env.DEBUG = namespaces; - } else { - // If you set a process.env field to null or undefined, it gets cast to the - // string 'null' or 'undefined'. Just delete instead. - delete process.env.DEBUG; - } - } - - /** - * Load `namespaces`. - * - * @return {String} returns the previously persisted debug modes - * @api private - */ - - function load() { - return process.env.DEBUG; - } - - /** - * Init logic for `debug` instances. - * - * Create a new `inspectOpts` object in case `useColors` is set - * differently for a particular `debug` instance. - */ - - function init(debug) { - debug.inspectOpts = {}; - - const keys = Object.keys(exports.inspectOpts); - for (let i = 0; i < keys.length; i++) { - debug.inspectOpts[keys[i]] = exports.inspectOpts[keys[i]]; - } - } - - module.exports = requireCommon()(exports); - - const {formatters} = module.exports; - - /** - * Map %o to `util.inspect()`, all on a single line. - */ - - formatters.o = function (v) { - this.inspectOpts.colors = this.useColors; - return util.inspect(v, this.inspectOpts) - .split('\n') - .map(str => str.trim()) - .join(' '); - }; - - /** - * Map %O to `util.inspect()`, allowing multiple lines if needed. - */ - - formatters.O = function (v) { - this.inspectOpts.colors = this.useColors; - return util.inspect(v, this.inspectOpts); - }; - } (node, node.exports)); - return node.exports; -} - -/** - * Detect Electron renderer / nwjs process, which is node, but we should - * treat as a browser. - */ - -if (typeof process === 'undefined' || process.type === 'renderer' || process.browser === true || process.__nwjs) { - src.exports = requireBrowser(); -} else { - src.exports = requireNode(); -} - -var srcExports = src.exports; - -/** - * @fileOverview - * A simple promises-based check to see if a TCP port is already in use. - */ - -// define the exports first to avoid cyclic dependencies. -tcpPortUsed.check = check; -tcpPortUsed.waitUntilFreeOnHost = waitUntilFreeOnHost; -tcpPortUsed.waitUntilFree = waitUntilFree; -tcpPortUsed.waitUntilUsedOnHost = waitUntilUsedOnHost; -tcpPortUsed.waitUntilUsed = waitUntilUsed; -tcpPortUsed.waitForStatus = waitForStatus; - -var is = is2; -var net = require$$1__default$2["default"]; -var util = require$$1__default$1["default"]; -var debug = srcExports('tcp-port-used'); - -// Global Values -var TIMEOUT = 2000; -var RETRYTIME = 250; - -function getDeferred() { - var resolve, reject, promise = new Promise(function(res, rej) { - resolve = res; - reject = rej; - }); - - return { - resolve: resolve, - reject: reject, - promise: promise - }; -} - -/** - * Creates an options object from all the possible arguments - * @private - * @param {Number} port a valid TCP port number - * @param {String} host The DNS name or IP address. - * @param {Boolean} status The desired in use status to wait for: false === not in use, true === in use - * @param {Number} retryTimeMs the retry interval in milliseconds - defaultis is 200ms - * @param {Number} timeOutMs the amount of time to wait until port is free default is 1000ms - * @return {Object} An options object with all the above parameters as properties. - */ -function makeOptionsObj(port, host, inUse, retryTimeMs, timeOutMs) { - var opts = {}; - opts.port = port; - opts.host = host; - opts.inUse = inUse; - opts.retryTimeMs = retryTimeMs; - opts.timeOutMs = timeOutMs; - return opts; -} - -/** - * Checks if a TCP port is in use by creating the socket and binding it to the - * target port. Once bound, successfully, it's assume the port is availble. - * After the socket is closed or in error, the promise is resolved. - * Note: you have to be super user to correctly test system ports (0-1023). - * @param {Number|Object} port The port you are curious to see if available. If an object, must have the parameters as properties. - * @param {String} [host] May be a DNS name or IP address. Default '127.0.0.1' - * @return {Object} A deferred Q promise. - * - * Example usage: - * - * var tcpPortUsed = require('tcp-port-used'); - * tcpPortUsed.check(22, '127.0.0.1') - * .then(function(inUse) { - * debug('Port 22 usage: '+inUse); - * }, function(err) { - * console.error('Error on check: '+util.inspect(err)); - * }); - */ -function check(port, host) { - - var deferred = getDeferred(); - var inUse = true; - var client; - - var opts; - if (!is.obj(port)) { - opts = makeOptionsObj(port, host); - } else { - opts = port; - } - - if (!is.port(opts.port)) { - debug('Error invalid port: '+util.inspect(opts.port)); - deferred.reject(new Error('invalid port: '+util.inspect(opts.port))); - return deferred.promise; - } - - if (is.nullOrUndefined(opts.host)) { - debug('set host address to default 127.0.0.1'); - opts.host = '127.0.0.1'; - } - - function cleanUp() { - if (client) { - client.removeAllListeners('connect'); - client.removeAllListeners('error'); - client.end(); - client.destroy(); - client.unref(); - } - //debug('listeners removed from client socket'); - } - - function onConnectCb() { - //debug('check - promise resolved - in use'); - deferred.resolve(inUse); - cleanUp(); - } - - function onErrorCb(err) { - if (err.code !== 'ECONNREFUSED') { - //debug('check - promise rejected, error: '+err.message); - deferred.reject(err); - } else { - //debug('ECONNREFUSED'); - inUse = false; - //debug('check - promise resolved - not in use'); - deferred.resolve(inUse); - } - cleanUp(); - } - - client = new net.Socket(); - client.once('connect', onConnectCb); - client.once('error', onErrorCb); - client.connect({port: opts.port, host: opts.host}, function() {}); - - return deferred.promise; -} - -/** - * Creates a deferred promise and fulfills it only when the socket's usage - * equals status in terms of 'in use' (false === not in use, true === in use). - * Will retry on an interval specified in retryTimeMs. Note: you have to be - * super user to correctly test system ports (0-1023). - * @param {Number|Object} port a valid TCP port number, if an object, has all the parameters described as properties. - * @param {String} host The DNS name or IP address. - * @param {Boolean} status The desired in use status to wait for false === not in use, true === in use - * @param {Number} [retryTimeMs] the retry interval in milliseconds - defaultis is 200ms - * @param {Number} [timeOutMs] the amount of time to wait until port is free default is 1000ms - * @return {Object} A deferred promise from the Q library. - * - * Example usage: - * - * var tcpPortUsed = require('tcp-port-used'); - * tcpPortUsed.waitForStatus(44204, 'some.host.com', true, 500, 4000) - * .then(function() { - * console.log('Port 44204 is now in use.'); - * }, function(err) { - * console.log('Error: ', error.message); - * }); - */ -function waitForStatus(port, host, inUse, retryTimeMs, timeOutMs) { - - var deferred = getDeferred(); - var timeoutId; - var timedout = false; - var retryId; - - // the first arument may be an object, if it is not, make an object - var opts; - if (is.obj(port)) { - opts = port; - } else { - opts = makeOptionsObj(port, host, inUse, retryTimeMs, timeOutMs); - } - - //debug('opts:'+util.inspect(opts); - - if (!is.bool(opts.inUse)) { - deferred.reject(new Error('inUse must be a boolean')); - return deferred.promise; - } - - if (!is.positiveInt(opts.retryTimeMs)) { - opts.retryTimeMs = RETRYTIME; - debug('set retryTime to default '+RETRYTIME+'ms'); - } - - if (!is.positiveInt(opts.timeOutMs)) { - opts.timeOutMs = TIMEOUT; - debug('set timeOutMs to default '+TIMEOUT+'ms'); - } - - function cleanUp() { - if (timeoutId) { - clearTimeout(timeoutId); - } - if (retryId) { - clearTimeout(retryId); - } - } - - function timeoutFunc() { - timedout = true; - cleanUp(); - deferred.reject(new Error('timeout')); - } - timeoutId = setTimeout(timeoutFunc, opts.timeOutMs); - - function doCheck() { - check(opts.port, opts.host) - .then(function(inUse) { - if (timedout) { - return; - } - //debug('doCheck inUse: '+inUse); - //debug('doCheck opts.inUse: '+opts.inUse); - if (inUse === opts.inUse) { - deferred.resolve(); - cleanUp(); - return; - } else { - retryId = setTimeout(function() { doCheck(); }, opts.retryTimeMs); - return; - } - }, function(err) { - if (timedout) { - return; - } - deferred.reject(err); - cleanUp(); - }); - } - - doCheck(); - return deferred.promise; -} - -/** - * Creates a deferred promise and fulfills it only when the socket is free. - * Will retry on an interval specified in retryTimeMs. - * Note: you have to be super user to correctly test system ports (0-1023). - * @param {Number} port a valid TCP port number - * @param {String} [host] The hostname or IP address of where the socket is. - * @param {Number} [retryTimeMs] the retry interval in milliseconds - defaultis is 100ms. - * @param {Number} [timeOutMs] the amount of time to wait until port is free. Default 300ms. - * @return {Object} A deferred promise from the q library. - * - * Example usage: - * - * var tcpPortUsed = require('tcp-port-used'); - * tcpPortUsed.waitUntilFreeOnHost(44203, 'some.host.com', 500, 4000) - * .then(function() { - * console.log('Port 44203 is now free.'); - * }, function(err) { - * console.loh('Error: ', error.message); - * }); - */ -function waitUntilFreeOnHost(port, host, retryTimeMs, timeOutMs) { - - // the first arument may be an object, if it is not, make an object - var opts; - if (is.obj(port)) { - opts = port; - opts.inUse = false; - } else { - opts = makeOptionsObj(port, host, false, retryTimeMs, timeOutMs); - } - - return waitForStatus(opts); -} - -/** - * For compatibility with previous version of the module, that did not provide - * arguements for hostnames. The host is set to the localhost '127.0.0.1'. - * @param {Number|Object} port a valid TCP port number. If an object, must contain all the parameters as properties. - * @param {Number} [retryTimeMs] the retry interval in milliseconds - defaultis is 100ms. - * @param {Number} [timeOutMs] the amount of time to wait until port is free. Default 300ms. - * @return {Object} A deferred promise from the q library. - * - * Example usage: - * - * var tcpPortUsed = require('tcp-port-used'); - * tcpPortUsed.waitUntilFree(44203, 500, 4000) - * .then(function() { - * console.log('Port 44203 is now free.'); - * }, function(err) { - * console.loh('Error: ', error.message); - * }); - */ -function waitUntilFree(port, retryTimeMs, timeOutMs) { - - // the first arument may be an object, if it is not, make an object - var opts; - if (is.obj(port)) { - opts = port; - opts.host = '127.0.0.1'; - opts.inUse = false; - } else { - opts = makeOptionsObj(port, '127.0.0.1', false, retryTimeMs, timeOutMs); - } - - return waitForStatus(opts); -} - -/** - * Creates a deferred promise and fulfills it only when the socket is used. - * Will retry on an interval specified in retryTimeMs. - * Note: you have to be super user to correctly test system ports (0-1023). - * @param {Number|Object} port a valid TCP port number. If an object, must contain all the parameters as properties. - * @param {Number} [retryTimeMs] the retry interval in milliseconds - defaultis is 500ms - * @param {Number} [timeOutMs] the amount of time to wait until port is free - * @return {Object} A deferred promise from the q library. - * - * Example usage: - * - * var tcpPortUsed = require('tcp-port-used'); - * tcpPortUsed.waitUntilUsedOnHost(44204, 'some.host.com', 500, 4000) - * .then(function() { - * console.log('Port 44204 is now in use.'); - * }, function(err) { - * console.log('Error: ', error.message); - * }); - */ -function waitUntilUsedOnHost(port, host, retryTimeMs, timeOutMs) { - - // the first arument may be an object, if it is not, make an object - var opts; - if (is.obj(port)) { - opts = port; - opts.inUse = true; - } else { - opts = makeOptionsObj(port, host, true, retryTimeMs, timeOutMs); - } - - return waitForStatus(opts); -} - -/** - * For compatibility to previous version of module which did not have support - * for host addresses. This function works only for localhost. - * @param {Number} port a valid TCP port number. If an Object, must contain all the parameters as properties. - * @param {Number} [retryTimeMs] the retry interval in milliseconds - defaultis is 500ms - * @param {Number} [timeOutMs] the amount of time to wait until port is free - * @return {Object} A deferred promise from the q library. - * - * Example usage: - * - * var tcpPortUsed = require('tcp-port-used'); - * tcpPortUsed.waitUntilUsed(44204, 500, 4000) - * .then(function() { - * console.log('Port 44204 is now in use.'); - * }, function(err) { - * console.log('Error: ', error.message); - * }); - */ -function waitUntilUsed(port, retryTimeMs, timeOutMs) { - - // the first arument may be an object, if it is not, make an object - var opts; - if (is.obj(port)) { - opts = port; - opts.host = '127.0.0.1'; - opts.inUse = true; - } else { - opts = makeOptionsObj(port, '127.0.0.1', true, retryTimeMs, timeOutMs); - } - - return waitUntilUsedOnHost(opts); -} - -var fetchRetry$1 = function (fetch, defaults) { - defaults = defaults || {}; - if (typeof fetch !== 'function') { - throw new ArgumentError('fetch must be a function'); - } - - if (typeof defaults !== 'object') { - throw new ArgumentError('defaults must be an object'); - } - - if (defaults.retries !== undefined && !isPositiveInteger(defaults.retries)) { - throw new ArgumentError('retries must be a positive integer'); - } - - if (defaults.retryDelay !== undefined && !isPositiveInteger(defaults.retryDelay) && typeof defaults.retryDelay !== 'function') { - throw new ArgumentError('retryDelay must be a positive integer or a function returning a positive integer'); - } - - if (defaults.retryOn !== undefined && !Array.isArray(defaults.retryOn) && typeof defaults.retryOn !== 'function') { - throw new ArgumentError('retryOn property expects an array or function'); - } - - var baseDefaults = { - retries: 3, - retryDelay: 1000, - retryOn: [], - }; - - defaults = Object.assign(baseDefaults, defaults); - - return function fetchRetry(input, init) { - var retries = defaults.retries; - var retryDelay = defaults.retryDelay; - var retryOn = defaults.retryOn; - - if (init && init.retries !== undefined) { - if (isPositiveInteger(init.retries)) { - retries = init.retries; - } else { - throw new ArgumentError('retries must be a positive integer'); - } - } - - if (init && init.retryDelay !== undefined) { - if (isPositiveInteger(init.retryDelay) || (typeof init.retryDelay === 'function')) { - retryDelay = init.retryDelay; - } else { - throw new ArgumentError('retryDelay must be a positive integer or a function returning a positive integer'); - } - } - - if (init && init.retryOn) { - if (Array.isArray(init.retryOn) || (typeof init.retryOn === 'function')) { - retryOn = init.retryOn; - } else { - throw new ArgumentError('retryOn property expects an array or function'); - } - } - - // eslint-disable-next-line no-undef - return new Promise(function (resolve, reject) { - var wrappedFetch = function (attempt) { - // As of node 18, this is no longer needed since node comes with native support for fetch: - /* istanbul ignore next */ - var _input = - typeof Request !== 'undefined' && input instanceof Request - ? input.clone() - : input; - fetch(_input, init) - .then(function (response) { - if (Array.isArray(retryOn) && retryOn.indexOf(response.status) === -1) { - resolve(response); - } else if (typeof retryOn === 'function') { - try { - // eslint-disable-next-line no-undef - return Promise.resolve(retryOn(attempt, null, response)) - .then(function (retryOnResponse) { - if(retryOnResponse) { - retry(attempt, null, response); - } else { - resolve(response); - } - }).catch(reject); - } catch (error) { - reject(error); - } - } else { - if (attempt < retries) { - retry(attempt, null, response); - } else { - resolve(response); - } - } - }) - .catch(function (error) { - if (typeof retryOn === 'function') { - try { - // eslint-disable-next-line no-undef - Promise.resolve(retryOn(attempt, error, null)) - .then(function (retryOnResponse) { - if(retryOnResponse) { - retry(attempt, error, null); - } else { - reject(error); - } - }) - .catch(function(error) { - reject(error); - }); - } catch(error) { - reject(error); - } - } else if (attempt < retries) { - retry(attempt, error, null); - } else { - reject(error); - } - }); - }; - - function retry(attempt, error, response) { - var delay = (typeof retryDelay === 'function') ? - retryDelay(attempt, error, response) : retryDelay; - setTimeout(function () { - wrappedFetch(++attempt); - }, delay); - } - - wrappedFetch(0); - }); - }; -}; - -function isPositiveInteger(value) { - return Number.isInteger(value) && value >= 0; -} - -function ArgumentError(message) { - this.name = 'ArgumentError'; - this.message = message; -} - -var fetchRT = /*@__PURE__*/getDefaultExportFromCjs(fetchRetry$1); - -var osutils = {}; - -var _os = require$$0__default["default"]; - -osutils.platform = function(){ - return process.platform; -}; - -osutils.cpuCount = function(){ - return _os.cpus().length; -}; - -osutils.sysUptime = function(){ - //seconds - return _os.uptime(); -}; - -osutils.processUptime = function(){ - //seconds - return process.uptime(); -}; - - - -// Memory -osutils.freemem = function(){ - return _os.freemem() / ( 1024 * 1024 ); -}; - -osutils.totalmem = function(){ - - return _os.totalmem() / ( 1024 * 1024 ); -}; - -osutils.freememPercentage = function(){ - return _os.freemem() / _os.totalmem(); -}; - -osutils.freeCommand = function(callback){ - - // Only Linux - require$$1__default$3["default"].exec('free -m', function(error, stdout, stderr) { - - var lines = stdout.split("\n"); - - - var str_mem_info = lines[1].replace( /[\s\n\r]+/g,' '); - - var mem_info = str_mem_info.split(' '); - - total_mem = parseFloat(mem_info[1]); - free_mem = parseFloat(mem_info[3]); - buffers_mem = parseFloat(mem_info[5]); - cached_mem = parseFloat(mem_info[6]); - - used_mem = total_mem - (free_mem + buffers_mem + cached_mem); - - callback(used_mem -2); - }); -}; - - -// Hard Disk Drive -osutils.harddrive = function(callback){ - - require$$1__default$3["default"].exec('df -k', function(error, stdout, stderr) { - - var total = 0; - var used = 0; - var free = 0; - - var lines = stdout.split("\n"); - - var str_disk_info = lines[1].replace( /[\s\n\r]+/g,' '); - - var disk_info = str_disk_info.split(' '); - - total = Math.ceil((disk_info[1] * 1024)/ Math.pow(1024,2)); - used = Math.ceil(disk_info[2] * 1024 / Math.pow(1024,2)) ; - free = Math.ceil(disk_info[3] * 1024 / Math.pow(1024,2)) ; - - callback(total, free, used); - }); -}; - - - -// Return process running current -osutils.getProcesses = function(nProcess, callback){ - - // if nprocess is undefined then is function - if(typeof nProcess === 'function'){ - - callback =nProcess; - nProcess = 0; - } - - command = 'ps -eo pcpu,pmem,time,args | sort -k 1 -r | head -n'+10; - //command = 'ps aux | head -n '+ 11 - //command = 'ps aux | head -n '+ (nProcess + 1) - if (nProcess > 0) - command = 'ps -eo pcpu,pmem,time,args | sort -k 1 -r | head -n'+(nProcess + 1); - - require$$1__default$3["default"].exec(command, function(error, stdout, stderr) { - - var lines = stdout.split("\n"); - lines.shift(); - lines.pop(); - - var result = ''; - - - lines.forEach(function(_item,_i){ - - var _str = _item.replace( /[\s\n\r]+/g,' '); - - _str = _str.split(' '); - - // result += _str[10]+" "+_str[9]+" "+_str[2]+" "+_str[3]+"\n"; // process - result += _str[1]+" "+_str[2]+" "+_str[3]+" "+_str[4].substring((_str[4].length - 25))+"\n"; // process - - }); - - callback(result); - }); -}; - - - -/* -* Returns All the load average usage for 1, 5 or 15 minutes. -*/ -osutils.allLoadavg = function(){ - - var loads = _os.loadavg(); - - return loads[0].toFixed(4)+','+loads[1].toFixed(4)+','+loads[2].toFixed(4); -}; - -/* -* Returns the load average usage for 1, 5 or 15 minutes. -*/ -osutils.loadavg = function(_time){ - - if(_time === undefined || (_time !== 5 && _time !== 15) ) _time = 1; - - var loads = _os.loadavg(); - var v = 0; - if(_time == 1) v = loads[0]; - if(_time == 5) v = loads[1]; - if(_time == 15) v = loads[2]; - - return v; -}; - - -osutils.cpuFree = function(callback){ - getCPUUsage(callback, true); -}; - -osutils.cpuUsage = function(callback){ - getCPUUsage(callback, false); -}; - -function getCPUUsage(callback, free){ - - var stats1 = getCPUInfo(); - var startIdle = stats1.idle; - var startTotal = stats1.total; - - setTimeout(function() { - var stats2 = getCPUInfo(); - var endIdle = stats2.idle; - var endTotal = stats2.total; - - var idle = endIdle - startIdle; - var total = endTotal - startTotal; - var perc = idle / total; - - if(free === true) - callback( perc ); - else - callback( (1 - perc) ); - - }, 1000 ); -} - -function getCPUInfo(callback){ - var cpus = _os.cpus(); - - var user = 0; - var nice = 0; - var sys = 0; - var idle = 0; - var irq = 0; - var total = 0; - - for(var cpu in cpus){ - - user += cpus[cpu].times.user; - nice += cpus[cpu].times.nice; - sys += cpus[cpu].times.sys; - irq += cpus[cpu].times.irq; - idle += cpus[cpu].times.idle; - } - - var total = user + nice + sys + idle + irq; - - return { - 'idle': idle, - 'total': total - }; -} - -/** - * Current nitro process - */ -var nitroProcessInfo = undefined; -/** - * This will retrive GPU informations and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported - */ -function updateNvidiaInfo(nvidiaSettings) { - return __awaiter(this, void 0, void 0, function () { - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - if (!(process.platform !== "darwin")) return [3 /*break*/, 2]; - return [4 /*yield*/, Promise.all([ - updateNvidiaDriverInfo(nvidiaSettings), - updateCudaExistence(nvidiaSettings), - updateGpuInfo(nvidiaSettings), - ])]; - case 1: - _a.sent(); - _a.label = 2; - case 2: return [2 /*return*/]; - } - }); - }); -} -/** - * Retrieve current nitro process - */ -var getNitroProcessInfo = function (subprocess) { - nitroProcessInfo = { - isRunning: subprocess != null, - }; - return nitroProcessInfo; -}; -/** - * Validate nvidia and cuda for linux and windows - */ -function updateNvidiaDriverInfo(nvidiaSettings) { - return __awaiter(this, void 0, void 0, function () { - return __generator(this, function (_a) { - node_child_process.exec("nvidia-smi --query-gpu=driver_version --format=csv,noheader", function (error, stdout) { - if (!error) { - var firstLine = stdout.split("\n")[0].trim(); - nvidiaSettings["nvidia_driver"].exist = true; - nvidiaSettings["nvidia_driver"].version = firstLine; - } - else { - nvidiaSettings["nvidia_driver"].exist = false; - } - }); - return [2 /*return*/]; - }); - }); -} -/** - * Check if file exists in paths - */ -function checkFileExistenceInPaths(file, paths) { - return paths.some(function (p) { return fs.existsSync(path__default["default"].join(p, file)); }); -} -/** - * Validate cuda for linux and windows - */ -function updateCudaExistence(nvidiaSettings) { - var filesCuda12; - var filesCuda11; - var paths; - var cudaVersion = ""; - if (process.platform === "win32") { - filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; - filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; - paths = process.env.PATH ? process.env.PATH.split(path__default["default"].delimiter) : []; - } - else { - filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; - filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; - paths = process.env.LD_LIBRARY_PATH - ? process.env.LD_LIBRARY_PATH.split(path__default["default"].delimiter) - : []; - paths.push("/usr/lib/x86_64-linux-gnu/"); - } - var cudaExists = filesCuda12.every(function (file) { return fs.existsSync(file) || checkFileExistenceInPaths(file, paths); }); - if (!cudaExists) { - cudaExists = filesCuda11.every(function (file) { return fs.existsSync(file) || checkFileExistenceInPaths(file, paths); }); - if (cudaExists) { - cudaVersion = "11"; - } - } - else { - cudaVersion = "12"; - } - nvidiaSettings["cuda"].exist = cudaExists; - nvidiaSettings["cuda"].version = cudaVersion; - if (cudaExists) { - nvidiaSettings.run_mode = "gpu"; - } -} -/** - * Get GPU information - */ -function updateGpuInfo(nvidiaSettings) { - return __awaiter(this, void 0, void 0, function () { - return __generator(this, function (_a) { - node_child_process.exec("nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", function (error, stdout) { - if (!error) { - // Get GPU info and gpu has higher memory first - var highestVram_1 = 0; - var highestVramId_1 = "0"; - var gpus = stdout - .trim() - .split("\n") - .map(function (line) { - var _a = line.split(", "), id = _a[0], vram = _a[1]; - vram = vram.replace(/\r/g, ""); - if (parseFloat(vram) > highestVram_1) { - highestVram_1 = parseFloat(vram); - highestVramId_1 = id; - } - return { id: id, vram: vram }; - }); - nvidiaSettings["gpus"] = gpus; - nvidiaSettings["gpu_highest_vram"] = highestVramId_1; - } - else { - nvidiaSettings["gpus"] = []; - } - }); - return [2 /*return*/]; - }); - }); -} - -/** - * Find which executable file to run based on the current platform. - * @returns The name of the executable file to run. - */ -var executableNitroFile = function (nvidiaSettings) { - var binaryFolder = path__default["default"].join(__dirname, "..", "bin"); // Current directory by default - var cudaVisibleDevices = ""; - var binaryName = "nitro"; - /** - * The binary folder is different for each platform. - */ - if (process.platform === "win32") { - /** - * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 - */ - if (nvidiaSettings["run_mode"] === "cpu") { - binaryFolder = path__default["default"].join(binaryFolder, "win-cpu"); - } - else { - if (nvidiaSettings["cuda"].version === "12") { - binaryFolder = path__default["default"].join(binaryFolder, "win-cuda-12-0"); - } - else { - binaryFolder = path__default["default"].join(binaryFolder, "win-cuda-11-7"); - } - cudaVisibleDevices = nvidiaSettings["gpu_highest_vram"]; - } - binaryName = "nitro.exe"; - } - else if (process.platform === "darwin") { - /** - * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) - */ - if (process.arch === "arm64") { - binaryFolder = path__default["default"].join(binaryFolder, "mac-arm64"); - } - else { - binaryFolder = path__default["default"].join(binaryFolder, "mac-x64"); - } - } - else { - if (nvidiaSettings["run_mode"] === "cpu") { - binaryFolder = path__default["default"].join(binaryFolder, "linux-cpu"); - } - else { - if (nvidiaSettings["cuda"].version === "12") { - binaryFolder = path__default["default"].join(binaryFolder, "linux-cuda-12-0"); - } - else { - binaryFolder = path__default["default"].join(binaryFolder, "linux-cuda-11-7"); - } - cudaVisibleDevices = nvidiaSettings["gpu_highest_vram"]; - } - } - return { - executablePath: path__default["default"].join(binaryFolder, binaryName), - cudaVisibleDevices: cudaVisibleDevices, - }; -}; - -// Polyfill fetch with retry -var fetchRetry = fetchRT(fetch); -// The PORT to use for the Nitro subprocess -var PORT = 3928; -// The HOST address to use for the Nitro subprocess -var LOCAL_HOST = "127.0.0.1"; -// The URL for the Nitro subprocess -var NITRO_HTTP_SERVER_URL = "http://".concat(LOCAL_HOST, ":").concat(PORT); -// The URL for the Nitro subprocess to load a model -var NITRO_HTTP_LOAD_MODEL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/loadmodel"); -// The URL for the Nitro subprocess to validate a model -var NITRO_HTTP_VALIDATE_MODEL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/modelstatus"); -// The URL for the Nitro subprocess to kill itself -var NITRO_HTTP_KILL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/processmanager/destroy"); -// The URL for the Nitro subprocess to run chat completion -var NITRO_HTTP_CHAT_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/chat_completion"); -// The default config for using Nvidia GPU -var NVIDIA_DEFAULT_CONFIG = { - notify: true, - run_mode: "cpu", - nvidia_driver: { - exist: false, - version: "", - }, - cuda: { - exist: false, - version: "", - }, - gpus: [], - gpu_highest_vram: "", -}; -// The supported model format -// TODO: Should be an array to support more models -var SUPPORTED_MODEL_FORMATS = [".gguf"]; -// The subprocess instance for Nitro -var subprocess = undefined; -// The current model file url -var currentModelFile = ""; -// The current model settings -var currentSettings = undefined; -// The Nvidia info file for checking for CUDA support on the system -var nvidiaConfig = NVIDIA_DEFAULT_CONFIG; -// The logger to use, default to stdout -var log = function (message) { - return process.stdout.write(message + os__default["default"].EOL); -}; -/** - * Get current Nvidia config - * @returns {NitroNvidiaConfig} A copy of the config object - * The returned object should be used for reading only - * Writing to config should be via the function {@setNvidiaConfig} - */ -function getNvidiaConfig() { - return Object.assign({}, nvidiaConfig); -} -/** - * Set custom Nvidia config for running inference over GPU - * @param {NitroNvidiaConfig} config The new config to apply - */ -function setNvidiaConfig(config) { - nvidiaConfig = config; -} -/** - * Set logger before running nitro - * @param {NitroLogger} logger The logger to use - */ -function setLogger(logger) { - log = logger; -} -/** - * Stops a Nitro subprocess. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -function stopModel() { - return killSubprocess(); -} -/** - * Initializes a Nitro subprocess to load a machine learning model. - * @param modelFullPath - The absolute full path to model directory. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package - */ -function runModel(_a) { - var modelFullPath = _a.modelFullPath, promptTemplate = _a.promptTemplate; - return __awaiter(this, void 0, void 0, function () { - var files, ggufBinFile, nitroResourceProbe, prompt; - return __generator(this, function (_b) { - switch (_b.label) { - case 0: - files = fs__default["default"].readdirSync(modelFullPath); - ggufBinFile = files.find(function (file) { - return file === path__default["default"].basename(modelFullPath) || - SUPPORTED_MODEL_FORMATS.some(function (ext) { return file.toLowerCase().endsWith(ext); }); - }); - if (!ggufBinFile) - return [2 /*return*/, Promise.reject("No GGUF model file found")]; - currentModelFile = path__default["default"].join(modelFullPath, ggufBinFile); - return [4 /*yield*/, getResourcesInfo()]; - case 1: - nitroResourceProbe = _b.sent(); - prompt = {}; - if (promptTemplate) { - try { - Object.assign(prompt, promptTemplateConverter(promptTemplate)); - } - catch (e) { - return [2 /*return*/, Promise.reject(e)]; - } - } - currentSettings = __assign(__assign({}, prompt), { llama_model_path: currentModelFile, - // This is critical and requires real system information - cpu_threads: Math.max(1, Math.round(nitroResourceProbe.numCpuPhysicalCore / 2)) }); - return [2 /*return*/, runNitroAndLoadModel()]; - } - }); - }); -} -/** - * 1. Spawn Nitro process - * 2. Load model into Nitro subprocess - * 3. Validate model status - * @returns - */ -function runNitroAndLoadModel() { - return __awaiter(this, void 0, void 0, function () { - return __generator(this, function (_a) { - // Gather system information for CPU physical cores and memory - return [2 /*return*/, killSubprocess() - .then(function () { return tcpPortUsed.waitUntilFree(PORT, 300, 5000); }) - .then(function () { - /** - * There is a problem with Windows process manager - * Should wait for awhile to make sure the port is free and subprocess is killed - * The tested threshold is 500ms - **/ - if (process.platform === "win32") { - return new Promise(function (resolve) { return setTimeout(function () { return resolve({}); }, 500); }); - } - else { - return Promise.resolve({}); - } - }) - .then(spawnNitroProcess) - .then(function () { return loadLLMModel(currentSettings); }) - .then(validateModelStatus) - .catch(function (err) { - // TODO: Broadcast error so app could display proper error message - log("[NITRO]::Error: ".concat(err)); - return { error: err }; - })]; - }); - }); -} -/** - * Parse prompt template into agrs settings - * @param {string} promptTemplate Template as string - * @returns {(NitroPromptSetting | never)} parsed prompt setting - * @throws {Error} if cannot split promptTemplate - */ -function promptTemplateConverter(promptTemplate) { - // Split the string using the markers - var systemMarker = "{system_message}"; - var promptMarker = "{prompt}"; - if (promptTemplate.includes(systemMarker) && - promptTemplate.includes(promptMarker)) { - // Find the indices of the markers - var systemIndex = promptTemplate.indexOf(systemMarker); - var promptIndex = promptTemplate.indexOf(promptMarker); - // Extract the parts of the string - var system_prompt = promptTemplate.substring(0, systemIndex); - var user_prompt = promptTemplate.substring(systemIndex + systemMarker.length, promptIndex); - var ai_prompt = promptTemplate.substring(promptIndex + promptMarker.length); - // Return the split parts - return { system_prompt: system_prompt, user_prompt: user_prompt, ai_prompt: ai_prompt }; - } - else if (promptTemplate.includes(promptMarker)) { - // Extract the parts of the string for the case where only promptMarker is present - var promptIndex = promptTemplate.indexOf(promptMarker); - var user_prompt = promptTemplate.substring(0, promptIndex); - var ai_prompt = promptTemplate.substring(promptIndex + promptMarker.length); - // Return the split parts - return { user_prompt: user_prompt, ai_prompt: ai_prompt }; - } - // Throw error if none of the conditions are met - throw Error("Cannot split prompt template"); -} -/** - * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - */ -function loadLLMModel(settings) { - return __awaiter(this, void 0, void 0, function () { - var res, err_1; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - log("[NITRO]::Debug: Loading model with params ".concat(JSON.stringify(settings))); - _a.label = 1; - case 1: - _a.trys.push([1, 4, , 6]); - return [4 /*yield*/, fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(settings), - retries: 3, - retryDelay: 500, - })]; - case 2: - res = _a.sent(); - // FIXME: Actually check response, as the model directory might not exist - log("[NITRO]::Debug: Load model success with response ".concat(JSON.stringify(res))); - return [4 /*yield*/, Promise.resolve(res)]; - case 3: return [2 /*return*/, _a.sent()]; - case 4: - err_1 = _a.sent(); - log("[NITRO]::Error: Load model failed with error ".concat(err_1)); - return [4 /*yield*/, Promise.reject()]; - case 5: return [2 /*return*/, _a.sent()]; - case 6: return [2 /*return*/]; - } - }); - }); -} -/** - * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified - * @param {any} request The request that is then sent to nitro - * @param {WritableStream} outStream Optional stream that consume the response body - * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. - * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data - */ -function chatCompletion(request, outStream) { - return __awaiter(this, void 0, void 0, function () { - var _this = this; - return __generator(this, function (_a) { - if (outStream) { - // Add stream option if there is an outStream specified when calling this function - Object.assign(request, { - stream: true, - }); - } - log("[NITRO]::Debug: Running chat completion with request ".concat(JSON.stringify(request))); - return [2 /*return*/, fetchRetry(NITRO_HTTP_CHAT_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "text/event-stream", - "Access-Control-Allow-Origin": "*", - }, - body: JSON.stringify(request), - retries: 3, - retryDelay: 500, - }) - .then(function (response) { return __awaiter(_this, void 0, void 0, function () { - var outPipe; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - if (!outStream) return [3 /*break*/, 2]; - if (!response.body) { - throw new Error("Error running chat completion"); - } - outPipe = response.body - .pipeThrough(new TextDecoderStream()) - .pipeTo(outStream); - // Wait for all the streams to complete before returning from async function - return [4 /*yield*/, outPipe]; - case 1: - // Wait for all the streams to complete before returning from async function - _a.sent(); - _a.label = 2; - case 2: - log("[NITRO]::Debug: Chat completion success"); - return [2 /*return*/, response]; - } - }); - }); }) - .catch(function (err) { - log("[NITRO]::Error: Chat completion failed with error ".concat(err)); - throw err; - })]; - }); - }); -} -/** - * Validates the status of a model. - * @returns {Promise} A promise that resolves to an object. - * If the model is loaded successfully, the object is empty. - * If the model is not loaded successfully, the object contains an error message. - */ -function validateModelStatus() { - return __awaiter(this, void 0, void 0, function () { - var _this = this; - return __generator(this, function (_a) { - // Send a GET request to the validation URL. - // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. - return [2 /*return*/, fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { - method: "GET", - headers: { - "Content-Type": "application/json", - }, - retries: 5, - retryDelay: 500, - }).then(function (res) { return __awaiter(_this, void 0, void 0, function () { - var body; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - log("[NITRO]::Debug: Validate model state success with response ".concat(JSON.stringify(res))); - if (!res.ok) return [3 /*break*/, 2]; - return [4 /*yield*/, res.json()]; - case 1: - body = _a.sent(); - // If the model is loaded, return an empty object. - // Otherwise, return an object with an error message. - if (body.model_loaded) { - return [2 /*return*/, Promise.resolve({})]; - } - _a.label = 2; - case 2: return [2 /*return*/, Promise.resolve({ error: "Validate model status failed" })]; - } - }); - }); })]; - }); - }); -} -/** - * Terminates the Nitro subprocess. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -function killSubprocess() { - return __awaiter(this, void 0, void 0, function () { - var controller; - return __generator(this, function (_a) { - controller = new AbortController(); - setTimeout(function () { return controller.abort(); }, 5000); - log("[NITRO]::Debug: Request to kill Nitro"); - return [2 /*return*/, fetch(NITRO_HTTP_KILL_URL, { - method: "DELETE", - signal: controller.signal, - }) - .then(function () { - subprocess === null || subprocess === void 0 ? void 0 : subprocess.kill(); - subprocess = undefined; - }) - .catch(function (err) { return ({ error: err }); }) - .then(function () { return tcpPortUsed.waitUntilFree(PORT, 300, 5000); }) - .then(function () { return log("[NITRO]::Debug: Nitro process is terminated"); }) - .then(function () { return Promise.resolve({}); })]; - }); - }); -} -/** - * Spawns a Nitro subprocess. - * @returns A promise that resolves when the Nitro subprocess is started. - */ -function spawnNitroProcess() { - var _this = this; - log("[NITRO]::Debug: Spawning Nitro subprocess..."); - return new Promise(function (resolve, reject) { return __awaiter(_this, void 0, void 0, function () { - var binaryFolder, executableOptions, args; - return __generator(this, function (_a) { - binaryFolder = path__default["default"].join(__dirname, "..", "bin"); - executableOptions = executableNitroFile(nvidiaConfig); - args = ["1", LOCAL_HOST, PORT.toString()]; - // Execute the binary - log("[NITRO]::Debug: Spawn nitro at path: ".concat(executableOptions.executablePath, ", and args: ").concat(args)); - subprocess = node_child_process.spawn(executableOptions.executablePath, ["1", LOCAL_HOST, PORT.toString()], { - cwd: binaryFolder, - env: __assign(__assign({}, process.env), { CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices }), - }); - // Handle subprocess output - subprocess.stdout.on("data", function (data) { - log("[NITRO]::Debug: ".concat(data)); - }); - subprocess.stderr.on("data", function (data) { - log("[NITRO]::Error: ".concat(data)); - }); - subprocess.on("close", function (code) { - log("[NITRO]::Debug: Nitro exited with code: ".concat(code)); - subprocess = undefined; - reject("child process exited with code ".concat(code)); - }); - tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(function () { - log("[NITRO]::Debug: Nitro is ready"); - resolve({}); - }); - return [2 /*return*/]; - }); - }); }); -} -/** - * Get the system resources information - */ -function getResourcesInfo() { - var _this = this; - return new Promise(function (resolve) { return __awaiter(_this, void 0, void 0, function () { - var cpu, response; - return __generator(this, function (_a) { - cpu = osutils.cpuCount(); - log("[NITRO]::CPU informations - ".concat(cpu)); - response = { - numCpuPhysicalCore: cpu, - memAvailable: 0, - }; - resolve(response); - return [2 /*return*/]; - }); - }); }); -} -var index = { - getNvidiaConfig: getNvidiaConfig, - setNvidiaConfig: setNvidiaConfig, - setLogger: setLogger, - runModel: runModel, - stopModel: stopModel, - loadLLMModel: loadLLMModel, - validateModelStatus: validateModelStatus, - chatCompletion: chatCompletion, - killSubprocess: killSubprocess, - updateNvidiaInfo: function () { return __awaiter(void 0, void 0, void 0, function () { return __generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, updateNvidiaInfo(nvidiaConfig)]; - case 1: return [2 /*return*/, _a.sent()]; - } - }); }); }, - getCurrentNitroProcessInfo: function () { return getNitroProcessInfo(subprocess); }, -}; - -module.exports = index; -//# sourceMappingURL=index.cjs.js.map diff --git a/nitro-node/dist/index.js b/nitro-node/dist/index.js deleted file mode 100644 index 9d3eff8a9..000000000 --- a/nitro-node/dist/index.js +++ /dev/null @@ -1,448 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -var tslib_1 = require("tslib"); -var node_os_1 = tslib_1.__importDefault(require("node:os")); -var node_fs_1 = tslib_1.__importDefault(require("node:fs")); -var node_path_1 = tslib_1.__importDefault(require("node:path")); -var node_child_process_1 = require("node:child_process"); -var tcp_port_used_1 = tslib_1.__importDefault(require("tcp-port-used")); -var fetch_retry_1 = tslib_1.__importDefault(require("fetch-retry")); -var os_utils_1 = tslib_1.__importDefault(require("os-utils")); -var nvidia_1 = require("./nvidia"); -var execute_1 = require("./execute"); -// Polyfill fetch with retry -var fetchRetry = (0, fetch_retry_1.default)(fetch); -// The PORT to use for the Nitro subprocess -var PORT = 3928; -// The HOST address to use for the Nitro subprocess -var LOCAL_HOST = "127.0.0.1"; -// The URL for the Nitro subprocess -var NITRO_HTTP_SERVER_URL = "http://".concat(LOCAL_HOST, ":").concat(PORT); -// The URL for the Nitro subprocess to load a model -var NITRO_HTTP_LOAD_MODEL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/loadmodel"); -// The URL for the Nitro subprocess to validate a model -var NITRO_HTTP_VALIDATE_MODEL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/modelstatus"); -// The URL for the Nitro subprocess to kill itself -var NITRO_HTTP_KILL_URL = "".concat(NITRO_HTTP_SERVER_URL, "/processmanager/destroy"); -// The URL for the Nitro subprocess to run chat completion -var NITRO_HTTP_CHAT_URL = "".concat(NITRO_HTTP_SERVER_URL, "/inferences/llamacpp/chat_completion"); -// The default config for using Nvidia GPU -var NVIDIA_DEFAULT_CONFIG = { - notify: true, - run_mode: "cpu", - nvidia_driver: { - exist: false, - version: "", - }, - cuda: { - exist: false, - version: "", - }, - gpus: [], - gpu_highest_vram: "", -}; -// The supported model format -// TODO: Should be an array to support more models -var SUPPORTED_MODEL_FORMATS = [".gguf"]; -// The subprocess instance for Nitro -var subprocess = undefined; -// The current model file url -var currentModelFile = ""; -// The current model settings -var currentSettings = undefined; -// The Nvidia info file for checking for CUDA support on the system -var nvidiaConfig = NVIDIA_DEFAULT_CONFIG; -// The logger to use, default to stdout -var log = function (message) { - var _ = []; - for (var _i = 1; _i < arguments.length; _i++) { - _[_i - 1] = arguments[_i]; - } - return process.stdout.write(message + node_os_1.default.EOL); -}; -/** - * Get current Nvidia config - * @returns {NitroNvidiaConfig} A copy of the config object - * The returned object should be used for reading only - * Writing to config should be via the function {@setNvidiaConfig} - */ -function getNvidiaConfig() { - return Object.assign({}, nvidiaConfig); -} -/** - * Set custom Nvidia config for running inference over GPU - * @param {NitroNvidiaConfig} config The new config to apply - */ -function setNvidiaConfig(config) { - nvidiaConfig = config; -} -/** - * Set logger before running nitro - * @param {NitroLogger} logger The logger to use - */ -function setLogger(logger) { - log = logger; -} -/** - * Stops a Nitro subprocess. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -function stopModel() { - return killSubprocess(); -} -/** - * Initializes a Nitro subprocess to load a machine learning model. - * @param modelFullPath - The absolute full path to model directory. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package - */ -function runModel(_a) { - var modelFullPath = _a.modelFullPath, promptTemplate = _a.promptTemplate; - return tslib_1.__awaiter(this, void 0, void 0, function () { - var files, ggufBinFile, nitroResourceProbe, prompt; - return tslib_1.__generator(this, function (_b) { - switch (_b.label) { - case 0: - files = node_fs_1.default.readdirSync(modelFullPath); - ggufBinFile = files.find(function (file) { - return file === node_path_1.default.basename(modelFullPath) || - SUPPORTED_MODEL_FORMATS.some(function (ext) { return file.toLowerCase().endsWith(ext); }); - }); - if (!ggufBinFile) - return [2 /*return*/, Promise.reject("No GGUF model file found")]; - currentModelFile = node_path_1.default.join(modelFullPath, ggufBinFile); - return [4 /*yield*/, getResourcesInfo()]; - case 1: - nitroResourceProbe = _b.sent(); - prompt = {}; - if (promptTemplate) { - try { - Object.assign(prompt, promptTemplateConverter(promptTemplate)); - } - catch (e) { - return [2 /*return*/, Promise.reject(e)]; - } - } - currentSettings = tslib_1.__assign(tslib_1.__assign({}, prompt), { llama_model_path: currentModelFile, - // This is critical and requires real system information - cpu_threads: Math.max(1, Math.round(nitroResourceProbe.numCpuPhysicalCore / 2)) }); - return [2 /*return*/, runNitroAndLoadModel()]; - } - }); - }); -} -/** - * 1. Spawn Nitro process - * 2. Load model into Nitro subprocess - * 3. Validate model status - * @returns - */ -function runNitroAndLoadModel() { - return tslib_1.__awaiter(this, void 0, void 0, function () { - return tslib_1.__generator(this, function (_a) { - // Gather system information for CPU physical cores and memory - return [2 /*return*/, killSubprocess() - .then(function () { return tcp_port_used_1.default.waitUntilFree(PORT, 300, 5000); }) - .then(function () { - /** - * There is a problem with Windows process manager - * Should wait for awhile to make sure the port is free and subprocess is killed - * The tested threshold is 500ms - **/ - if (process.platform === "win32") { - return new Promise(function (resolve) { return setTimeout(function () { return resolve({}); }, 500); }); - } - else { - return Promise.resolve({}); - } - }) - .then(spawnNitroProcess) - .then(function () { return loadLLMModel(currentSettings); }) - .then(validateModelStatus) - .catch(function (err) { - // TODO: Broadcast error so app could display proper error message - log("[NITRO]::Error: ".concat(err)); - return { error: err }; - })]; - }); - }); -} -/** - * Parse prompt template into agrs settings - * @param {string} promptTemplate Template as string - * @returns {(NitroPromptSetting | never)} parsed prompt setting - * @throws {Error} if cannot split promptTemplate - */ -function promptTemplateConverter(promptTemplate) { - // Split the string using the markers - var systemMarker = "{system_message}"; - var promptMarker = "{prompt}"; - if (promptTemplate.includes(systemMarker) && - promptTemplate.includes(promptMarker)) { - // Find the indices of the markers - var systemIndex = promptTemplate.indexOf(systemMarker); - var promptIndex = promptTemplate.indexOf(promptMarker); - // Extract the parts of the string - var system_prompt = promptTemplate.substring(0, systemIndex); - var user_prompt = promptTemplate.substring(systemIndex + systemMarker.length, promptIndex); - var ai_prompt = promptTemplate.substring(promptIndex + promptMarker.length); - // Return the split parts - return { system_prompt: system_prompt, user_prompt: user_prompt, ai_prompt: ai_prompt }; - } - else if (promptTemplate.includes(promptMarker)) { - // Extract the parts of the string for the case where only promptMarker is present - var promptIndex = promptTemplate.indexOf(promptMarker); - var user_prompt = promptTemplate.substring(0, promptIndex); - var ai_prompt = promptTemplate.substring(promptIndex + promptMarker.length); - // Return the split parts - return { user_prompt: user_prompt, ai_prompt: ai_prompt }; - } - // Throw error if none of the conditions are met - throw Error("Cannot split prompt template"); -} -/** - * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - */ -function loadLLMModel(settings) { - return tslib_1.__awaiter(this, void 0, void 0, function () { - var res, err_1; - return tslib_1.__generator(this, function (_a) { - switch (_a.label) { - case 0: - log("[NITRO]::Debug: Loading model with params ".concat(JSON.stringify(settings))); - _a.label = 1; - case 1: - _a.trys.push([1, 4, , 6]); - return [4 /*yield*/, fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(settings), - retries: 3, - retryDelay: 500, - })]; - case 2: - res = _a.sent(); - // FIXME: Actually check response, as the model directory might not exist - log("[NITRO]::Debug: Load model success with response ".concat(JSON.stringify(res))); - return [4 /*yield*/, Promise.resolve(res)]; - case 3: return [2 /*return*/, _a.sent()]; - case 4: - err_1 = _a.sent(); - log("[NITRO]::Error: Load model failed with error ".concat(err_1)); - return [4 /*yield*/, Promise.reject()]; - case 5: return [2 /*return*/, _a.sent()]; - case 6: return [2 /*return*/]; - } - }); - }); -} -/** - * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified - * @param {any} request The request that is then sent to nitro - * @param {WritableStream} outStream Optional stream that consume the response body - * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. - * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data - */ -function chatCompletion(request, outStream) { - return tslib_1.__awaiter(this, void 0, void 0, function () { - var _this = this; - return tslib_1.__generator(this, function (_a) { - if (outStream) { - // Add stream option if there is an outStream specified when calling this function - Object.assign(request, { - stream: true, - }); - } - log("[NITRO]::Debug: Running chat completion with request ".concat(JSON.stringify(request))); - return [2 /*return*/, fetchRetry(NITRO_HTTP_CHAT_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "text/event-stream", - "Access-Control-Allow-Origin": "*", - }, - body: JSON.stringify(request), - retries: 3, - retryDelay: 500, - }) - .then(function (response) { return tslib_1.__awaiter(_this, void 0, void 0, function () { - var outPipe; - return tslib_1.__generator(this, function (_a) { - switch (_a.label) { - case 0: - if (!outStream) return [3 /*break*/, 2]; - if (!response.body) { - throw new Error("Error running chat completion"); - } - outPipe = response.body - .pipeThrough(new TextDecoderStream()) - .pipeTo(outStream); - // Wait for all the streams to complete before returning from async function - return [4 /*yield*/, outPipe]; - case 1: - // Wait for all the streams to complete before returning from async function - _a.sent(); - _a.label = 2; - case 2: - log("[NITRO]::Debug: Chat completion success"); - return [2 /*return*/, response]; - } - }); - }); }) - .catch(function (err) { - log("[NITRO]::Error: Chat completion failed with error ".concat(err)); - throw err; - })]; - }); - }); -} -/** - * Validates the status of a model. - * @returns {Promise} A promise that resolves to an object. - * If the model is loaded successfully, the object is empty. - * If the model is not loaded successfully, the object contains an error message. - */ -function validateModelStatus() { - return tslib_1.__awaiter(this, void 0, void 0, function () { - var _this = this; - return tslib_1.__generator(this, function (_a) { - // Send a GET request to the validation URL. - // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. - return [2 /*return*/, fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { - method: "GET", - headers: { - "Content-Type": "application/json", - }, - retries: 5, - retryDelay: 500, - }).then(function (res) { return tslib_1.__awaiter(_this, void 0, void 0, function () { - var body; - return tslib_1.__generator(this, function (_a) { - switch (_a.label) { - case 0: - log("[NITRO]::Debug: Validate model state success with response ".concat(JSON.stringify(res))); - if (!res.ok) return [3 /*break*/, 2]; - return [4 /*yield*/, res.json()]; - case 1: - body = _a.sent(); - // If the model is loaded, return an empty object. - // Otherwise, return an object with an error message. - if (body.model_loaded) { - return [2 /*return*/, Promise.resolve({})]; - } - _a.label = 2; - case 2: return [2 /*return*/, Promise.resolve({ error: "Validate model status failed" })]; - } - }); - }); })]; - }); - }); -} -/** - * Terminates the Nitro subprocess. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -function killSubprocess() { - return tslib_1.__awaiter(this, void 0, void 0, function () { - var controller; - return tslib_1.__generator(this, function (_a) { - controller = new AbortController(); - setTimeout(function () { return controller.abort(); }, 5000); - log("[NITRO]::Debug: Request to kill Nitro"); - return [2 /*return*/, fetch(NITRO_HTTP_KILL_URL, { - method: "DELETE", - signal: controller.signal, - }) - .then(function () { - subprocess === null || subprocess === void 0 ? void 0 : subprocess.kill(); - subprocess = undefined; - }) - .catch(function (err) { return ({ error: err }); }) - .then(function () { return tcp_port_used_1.default.waitUntilFree(PORT, 300, 5000); }) - .then(function () { return log("[NITRO]::Debug: Nitro process is terminated"); }) - .then(function () { return Promise.resolve({}); })]; - }); - }); -} -/** - * Spawns a Nitro subprocess. - * @returns A promise that resolves when the Nitro subprocess is started. - */ -function spawnNitroProcess() { - var _this = this; - log("[NITRO]::Debug: Spawning Nitro subprocess..."); - return new Promise(function (resolve, reject) { return tslib_1.__awaiter(_this, void 0, void 0, function () { - var binaryFolder, executableOptions, args; - return tslib_1.__generator(this, function (_a) { - binaryFolder = node_path_1.default.join(__dirname, "..", "bin"); - executableOptions = (0, execute_1.executableNitroFile)(nvidiaConfig); - args = ["1", LOCAL_HOST, PORT.toString()]; - // Execute the binary - log("[NITRO]::Debug: Spawn nitro at path: ".concat(executableOptions.executablePath, ", and args: ").concat(args)); - subprocess = (0, node_child_process_1.spawn)(executableOptions.executablePath, ["1", LOCAL_HOST, PORT.toString()], { - cwd: binaryFolder, - env: tslib_1.__assign(tslib_1.__assign({}, process.env), { CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices }), - }); - // Handle subprocess output - subprocess.stdout.on("data", function (data) { - log("[NITRO]::Debug: ".concat(data)); - }); - subprocess.stderr.on("data", function (data) { - log("[NITRO]::Error: ".concat(data)); - }); - subprocess.on("close", function (code) { - log("[NITRO]::Debug: Nitro exited with code: ".concat(code)); - subprocess = undefined; - reject("child process exited with code ".concat(code)); - }); - tcp_port_used_1.default.waitUntilUsed(PORT, 300, 30000).then(function () { - log("[NITRO]::Debug: Nitro is ready"); - resolve({}); - }); - return [2 /*return*/]; - }); - }); }); -} -/** - * Get the system resources information - */ -function getResourcesInfo() { - var _this = this; - return new Promise(function (resolve) { return tslib_1.__awaiter(_this, void 0, void 0, function () { - var cpu, response; - return tslib_1.__generator(this, function (_a) { - cpu = os_utils_1.default.cpuCount(); - log("[NITRO]::CPU informations - ".concat(cpu)); - response = { - numCpuPhysicalCore: cpu, - memAvailable: 0, - }; - resolve(response); - return [2 /*return*/]; - }); - }); }); -} -exports.default = { - getNvidiaConfig: getNvidiaConfig, - setNvidiaConfig: setNvidiaConfig, - setLogger: setLogger, - runModel: runModel, - stopModel: stopModel, - loadLLMModel: loadLLMModel, - validateModelStatus: validateModelStatus, - chatCompletion: chatCompletion, - killSubprocess: killSubprocess, - updateNvidiaInfo: function () { return tslib_1.__awaiter(void 0, void 0, void 0, function () { return tslib_1.__generator(this, function (_a) { - switch (_a.label) { - case 0: return [4 /*yield*/, (0, nvidia_1.updateNvidiaInfo)(nvidiaConfig)]; - case 1: return [2 /*return*/, _a.sent()]; - } - }); }); }, - getCurrentNitroProcessInfo: function () { return (0, nvidia_1.getNitroProcessInfo)(subprocess); }, -}; -//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/nitro-node/dist/nvidia.js b/nitro-node/dist/nvidia.js deleted file mode 100644 index 7157c08c1..000000000 --- a/nitro-node/dist/nvidia.js +++ /dev/null @@ -1,147 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.updateGpuInfo = exports.updateCudaExistence = exports.checkFileExistenceInPaths = exports.updateNvidiaDriverInfo = exports.getNitroProcessInfo = exports.updateNvidiaInfo = void 0; -var tslib_1 = require("tslib"); -var node_fs_1 = require("node:fs"); -var node_child_process_1 = require("node:child_process"); -var node_path_1 = tslib_1.__importDefault(require("node:path")); -/** - * Current nitro process - */ -var nitroProcessInfo = undefined; -/** - * This will retrive GPU informations and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported - */ -function updateNvidiaInfo(nvidiaSettings) { - return tslib_1.__awaiter(this, void 0, void 0, function () { - return tslib_1.__generator(this, function (_a) { - switch (_a.label) { - case 0: - if (!(process.platform !== "darwin")) return [3 /*break*/, 2]; - return [4 /*yield*/, Promise.all([ - updateNvidiaDriverInfo(nvidiaSettings), - updateCudaExistence(nvidiaSettings), - updateGpuInfo(nvidiaSettings), - ])]; - case 1: - _a.sent(); - _a.label = 2; - case 2: return [2 /*return*/]; - } - }); - }); -} -exports.updateNvidiaInfo = updateNvidiaInfo; -/** - * Retrieve current nitro process - */ -var getNitroProcessInfo = function (subprocess) { - nitroProcessInfo = { - isRunning: subprocess != null, - }; - return nitroProcessInfo; -}; -exports.getNitroProcessInfo = getNitroProcessInfo; -/** - * Validate nvidia and cuda for linux and windows - */ -function updateNvidiaDriverInfo(nvidiaSettings) { - return tslib_1.__awaiter(this, void 0, void 0, function () { - return tslib_1.__generator(this, function (_a) { - (0, node_child_process_1.exec)("nvidia-smi --query-gpu=driver_version --format=csv,noheader", function (error, stdout) { - if (!error) { - var firstLine = stdout.split("\n")[0].trim(); - nvidiaSettings["nvidia_driver"].exist = true; - nvidiaSettings["nvidia_driver"].version = firstLine; - } - else { - nvidiaSettings["nvidia_driver"].exist = false; - } - }); - return [2 /*return*/]; - }); - }); -} -exports.updateNvidiaDriverInfo = updateNvidiaDriverInfo; -/** - * Check if file exists in paths - */ -function checkFileExistenceInPaths(file, paths) { - return paths.some(function (p) { return (0, node_fs_1.existsSync)(node_path_1.default.join(p, file)); }); -} -exports.checkFileExistenceInPaths = checkFileExistenceInPaths; -/** - * Validate cuda for linux and windows - */ -function updateCudaExistence(nvidiaSettings) { - var filesCuda12; - var filesCuda11; - var paths; - var cudaVersion = ""; - if (process.platform === "win32") { - filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; - filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; - paths = process.env.PATH ? process.env.PATH.split(node_path_1.default.delimiter) : []; - } - else { - filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; - filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; - paths = process.env.LD_LIBRARY_PATH - ? process.env.LD_LIBRARY_PATH.split(node_path_1.default.delimiter) - : []; - paths.push("/usr/lib/x86_64-linux-gnu/"); - } - var cudaExists = filesCuda12.every(function (file) { return (0, node_fs_1.existsSync)(file) || checkFileExistenceInPaths(file, paths); }); - if (!cudaExists) { - cudaExists = filesCuda11.every(function (file) { return (0, node_fs_1.existsSync)(file) || checkFileExistenceInPaths(file, paths); }); - if (cudaExists) { - cudaVersion = "11"; - } - } - else { - cudaVersion = "12"; - } - nvidiaSettings["cuda"].exist = cudaExists; - nvidiaSettings["cuda"].version = cudaVersion; - if (cudaExists) { - nvidiaSettings.run_mode = "gpu"; - } -} -exports.updateCudaExistence = updateCudaExistence; -/** - * Get GPU information - */ -function updateGpuInfo(nvidiaSettings) { - return tslib_1.__awaiter(this, void 0, void 0, function () { - return tslib_1.__generator(this, function (_a) { - (0, node_child_process_1.exec)("nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", function (error, stdout) { - if (!error) { - // Get GPU info and gpu has higher memory first - var highestVram_1 = 0; - var highestVramId_1 = "0"; - var gpus = stdout - .trim() - .split("\n") - .map(function (line) { - var _a = line.split(", "), id = _a[0], vram = _a[1]; - vram = vram.replace(/\r/g, ""); - if (parseFloat(vram) > highestVram_1) { - highestVram_1 = parseFloat(vram); - highestVramId_1 = id; - } - return { id: id, vram: vram }; - }); - nvidiaSettings["gpus"] = gpus; - nvidiaSettings["gpu_highest_vram"] = highestVramId_1; - } - else { - nvidiaSettings["gpus"] = []; - } - }); - return [2 /*return*/]; - }); - }); -} -exports.updateGpuInfo = updateGpuInfo; -//# sourceMappingURL=nvidia.js.map \ No newline at end of file diff --git a/nitro-node/dist/types/execute.d.ts b/nitro-node/dist/types/execute.d.ts deleted file mode 100644 index 753e7a739..000000000 --- a/nitro-node/dist/types/execute.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -export interface NitroExecutableOptions { - executablePath: string; - cudaVisibleDevices: string; -} -/** - * Find which executable file to run based on the current platform. - * @returns The name of the executable file to run. - */ -export declare const executableNitroFile: (nvidiaSettings: NitroNvidiaConfig) => NitroExecutableOptions; diff --git a/nitro-node/dist/types/index.d.ts b/nitro-node/dist/types/index.d.ts deleted file mode 100644 index cb5d1d859..000000000 --- a/nitro-node/dist/types/index.d.ts +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Get current Nvidia config - * @returns {NitroNvidiaConfig} A copy of the config object - * The returned object should be used for reading only - * Writing to config should be via the function {@setNvidiaConfig} - */ -declare function getNvidiaConfig(): NitroNvidiaConfig; -/** - * Set custom Nvidia config for running inference over GPU - * @param {NitroNvidiaConfig} config The new config to apply - */ -declare function setNvidiaConfig(config: NitroNvidiaConfig): void; -/** - * Set logger before running nitro - * @param {NitroLogger} logger The logger to use - */ -declare function setLogger(logger: NitroLogger): void; -/** - * Stops a Nitro subprocess. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -declare function stopModel(): Promise; -/** - * Initializes a Nitro subprocess to load a machine learning model. - * @param modelFullPath - The absolute full path to model directory. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package - */ -declare function runModel({ modelFullPath, promptTemplate, }: NitroModelInitOptions): Promise; -/** - * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - */ -declare function loadLLMModel(settings: any): Promise; -/** - * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified - * @param {any} request The request that is then sent to nitro - * @param {WritableStream} outStream Optional stream that consume the response body - * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. - * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data - */ -declare function chatCompletion(request: any, outStream?: WritableStream): Promise; -/** - * Validates the status of a model. - * @returns {Promise} A promise that resolves to an object. - * If the model is loaded successfully, the object is empty. - * If the model is not loaded successfully, the object contains an error message. - */ -declare function validateModelStatus(): Promise; -/** - * Terminates the Nitro subprocess. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -declare function killSubprocess(): Promise; -declare const _default: { - getNvidiaConfig: typeof getNvidiaConfig; - setNvidiaConfig: typeof setNvidiaConfig; - setLogger: typeof setLogger; - runModel: typeof runModel; - stopModel: typeof stopModel; - loadLLMModel: typeof loadLLMModel; - validateModelStatus: typeof validateModelStatus; - chatCompletion: typeof chatCompletion; - killSubprocess: typeof killSubprocess; - updateNvidiaInfo: () => Promise; - getCurrentNitroProcessInfo: () => import("./nvidia").NitroProcessInfo; -}; -export default _default; diff --git a/nitro-node/dist/types/nvidia.d.ts b/nitro-node/dist/types/nvidia.d.ts deleted file mode 100644 index 45ce2c507..000000000 --- a/nitro-node/dist/types/nvidia.d.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Nitro process info - */ -export interface NitroProcessInfo { - isRunning: boolean; -} -/** - * This will retrive GPU informations and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported - */ -export declare function updateNvidiaInfo(nvidiaSettings: NitroNvidiaConfig): Promise; -/** - * Retrieve current nitro process - */ -export declare const getNitroProcessInfo: (subprocess: any) => NitroProcessInfo; -/** - * Validate nvidia and cuda for linux and windows - */ -export declare function updateNvidiaDriverInfo(nvidiaSettings: NitroNvidiaConfig): Promise; -/** - * Check if file exists in paths - */ -export declare function checkFileExistenceInPaths(file: string, paths: string[]): boolean; -/** - * Validate cuda for linux and windows - */ -export declare function updateCudaExistence(nvidiaSettings: NitroNvidiaConfig): void; -/** - * Get GPU information - */ -export declare function updateGpuInfo(nvidiaSettings: NitroNvidiaConfig): Promise; diff --git a/nitro-node/dist/types/scripts/download-nitro.d.ts b/nitro-node/dist/types/scripts/download-nitro.d.ts deleted file mode 100644 index 862e9b9e8..000000000 --- a/nitro-node/dist/types/scripts/download-nitro.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare const downloadNitro: () => void; -export default downloadNitro; diff --git a/nitro-node/dist/types/src/execute.d.ts b/nitro-node/dist/types/src/execute.d.ts deleted file mode 100644 index 753e7a739..000000000 --- a/nitro-node/dist/types/src/execute.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -export interface NitroExecutableOptions { - executablePath: string; - cudaVisibleDevices: string; -} -/** - * Find which executable file to run based on the current platform. - * @returns The name of the executable file to run. - */ -export declare const executableNitroFile: (nvidiaSettings: NitroNvidiaConfig) => NitroExecutableOptions; diff --git a/nitro-node/dist/types/src/index.d.ts b/nitro-node/dist/types/src/index.d.ts deleted file mode 100644 index cb5d1d859..000000000 --- a/nitro-node/dist/types/src/index.d.ts +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Get current Nvidia config - * @returns {NitroNvidiaConfig} A copy of the config object - * The returned object should be used for reading only - * Writing to config should be via the function {@setNvidiaConfig} - */ -declare function getNvidiaConfig(): NitroNvidiaConfig; -/** - * Set custom Nvidia config for running inference over GPU - * @param {NitroNvidiaConfig} config The new config to apply - */ -declare function setNvidiaConfig(config: NitroNvidiaConfig): void; -/** - * Set logger before running nitro - * @param {NitroLogger} logger The logger to use - */ -declare function setLogger(logger: NitroLogger): void; -/** - * Stops a Nitro subprocess. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -declare function stopModel(): Promise; -/** - * Initializes a Nitro subprocess to load a machine learning model. - * @param modelFullPath - The absolute full path to model directory. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package - */ -declare function runModel({ modelFullPath, promptTemplate, }: NitroModelInitOptions): Promise; -/** - * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - */ -declare function loadLLMModel(settings: any): Promise; -/** - * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified - * @param {any} request The request that is then sent to nitro - * @param {WritableStream} outStream Optional stream that consume the response body - * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. - * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data - */ -declare function chatCompletion(request: any, outStream?: WritableStream): Promise; -/** - * Validates the status of a model. - * @returns {Promise} A promise that resolves to an object. - * If the model is loaded successfully, the object is empty. - * If the model is not loaded successfully, the object contains an error message. - */ -declare function validateModelStatus(): Promise; -/** - * Terminates the Nitro subprocess. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -declare function killSubprocess(): Promise; -declare const _default: { - getNvidiaConfig: typeof getNvidiaConfig; - setNvidiaConfig: typeof setNvidiaConfig; - setLogger: typeof setLogger; - runModel: typeof runModel; - stopModel: typeof stopModel; - loadLLMModel: typeof loadLLMModel; - validateModelStatus: typeof validateModelStatus; - chatCompletion: typeof chatCompletion; - killSubprocess: typeof killSubprocess; - updateNvidiaInfo: () => Promise; - getCurrentNitroProcessInfo: () => import("./nvidia").NitroProcessInfo; -}; -export default _default; diff --git a/nitro-node/dist/types/src/nvidia.d.ts b/nitro-node/dist/types/src/nvidia.d.ts deleted file mode 100644 index 45ce2c507..000000000 --- a/nitro-node/dist/types/src/nvidia.d.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Nitro process info - */ -export interface NitroProcessInfo { - isRunning: boolean; -} -/** - * This will retrive GPU informations and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported - */ -export declare function updateNvidiaInfo(nvidiaSettings: NitroNvidiaConfig): Promise; -/** - * Retrieve current nitro process - */ -export declare const getNitroProcessInfo: (subprocess: any) => NitroProcessInfo; -/** - * Validate nvidia and cuda for linux and windows - */ -export declare function updateNvidiaDriverInfo(nvidiaSettings: NitroNvidiaConfig): Promise; -/** - * Check if file exists in paths - */ -export declare function checkFileExistenceInPaths(file: string, paths: string[]): boolean; -/** - * Validate cuda for linux and windows - */ -export declare function updateCudaExistence(nvidiaSettings: NitroNvidiaConfig): void; -/** - * Get GPU information - */ -export declare function updateGpuInfo(nvidiaSettings: NitroNvidiaConfig): Promise; From 869470b3866dd01eaca99f0b8055cdbee74c6b3c Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sat, 27 Jan 2024 16:53:13 +0700 Subject: [PATCH 21/49] fix(nitro-node): several fixes - Fix MacOS build GitHub workflow for nitro using wrong command to get number of cpu cores - Fix JSDoc for runModel function - Fix download-nitro script to set executable mode after downloading the binaries --- .github/workflows/build.yml | 2 +- nitro-node/scripts/download-nitro.ts | 23 ++++++++++++++++++----- nitro-node/src/index.ts | 9 ++++----- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 43532a27a..c3056e9af 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -272,7 +272,7 @@ jobs: ./install_deps.sh mkdir build && cd build cmake -DWHISPER_COREML=1 -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }} .. - CC=gcc-8 make -j $(sysctl -n hw.ncp) + CC=gcc-8 make -j $(sysctl -n hw.ncpu) ls -la - name: Package diff --git a/nitro-node/scripts/download-nitro.ts b/nitro-node/scripts/download-nitro.ts index ed0f42b29..db28cc455 100644 --- a/nitro-node/scripts/download-nitro.ts +++ b/nitro-node/scripts/download-nitro.ts @@ -1,5 +1,7 @@ +import fs from 'node:fs'; import path from "node:path"; import download from "download"; +import { Duplex } from "node:stream"; // Define nitro version to download in env variable const NITRO_VERSION = process.env.NITRO_VERSION || "0.2.11"; @@ -51,7 +53,7 @@ const getTarUrl = (version: string, suffix: string) => `https://github.com/janhq/nitro/releases/download/v${version}/nitro-${version}-${suffix}.tar.gz`; // Report download progress -const createProgressReporter = (variant: string) => (stream: any) => +const createProgressReporter = (variant: string) => (stream: Promise & Duplex) => stream .on( "downloadProgress", @@ -69,16 +71,27 @@ const createProgressReporter = (variant: string) => (stream: any) => }); // Download single binary -const downloadBinary = (version: string, suffix: string, filePath: string) => { +const downloadBinary = (version: string, suffix: string, destDirPath: string) => { const tarUrl = getTarUrl(version, suffix); - console.log(`Downloading ${tarUrl} to ${filePath}`); + console.log(`Downloading ${tarUrl} to ${destDirPath}`); const progressReporter = createProgressReporter(suffix); return progressReporter( - download(tarUrl, filePath, { + download(tarUrl, destDirPath, { strip: 1, extract: true, }), - ); + ).then(() => { + // Set mode of downloaded binaries to executable + (fs + .readdirSync(destDirPath, { recursive: true }) as string[]) + .filter((fname) => fs.lstatSync(path.join(destDirPath, fname)).isFile()) + .filter((fname) => fname.includes('nitro')) + .forEach( + (nitroBinary) => { + const absPath = path.join(destDirPath, nitroBinary) + fs.chmodSync(absPath, fs.constants.S_IRWXU | fs.constants.S_IRWXG | fs.constants.S_IRWXO) + }) + }); }; // Download the binaries diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index afe16f550..088886a50 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -95,9 +95,8 @@ function stopModel(): Promise { /** * Initializes a Nitro subprocess to load a machine learning model. * @param modelFullPath - The absolute full path to model directory. - * @param wrapper - The model wrapper. + * @param promptTemplate - The template to use for generating prompts. * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package */ async function runModel({ modelFullPath, @@ -273,8 +272,8 @@ async function chatCompletion( method: "POST", headers: { "Content-Type": "application/json", - Accept: "text/event-stream", "Access-Control-Allow-Origin": "*", + Accept: Boolean(outStream) ? "text/event-stream" : "application/json", }, body: JSON.stringify(request), retries: 3, @@ -352,10 +351,10 @@ async function killSubprocess(): Promise { subprocess?.kill(); subprocess = undefined; }) - .catch((err) => ({ error: err })) .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) - .then(() => Promise.resolve({})); + .then(() => Promise.resolve({})) + .catch((err) => ({ error: err })); } /** From fb7cb5c87505715ebccf9ccc5565e146f943cd96 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sat, 27 Jan 2024 18:23:34 +0700 Subject: [PATCH 22/49] feat(nitro-node): only download binaries if they are not exist - Do not re-download binaries if they are already exist - Check and download binaries if they are not exist during runtime --- nitro-node/package.json | 1 - nitro-node/postinstall.js | 2 +- nitro-node/rollup.config.ts | 2 +- nitro-node/scripts/download-nitro.ts | 94 ++++++++++++++++++---------- nitro-node/src/index.ts | 3 + 5 files changed, 65 insertions(+), 37 deletions(-) diff --git a/nitro-node/package.json b/nitro-node/package.json index 59cb02ebc..0a823ed86 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -8,7 +8,6 @@ "license": "AGPL-3.0", "scripts": { "test": "jest --verbose --detectOpenHandles", - "pretest": "ts-node scripts/download-nitro.ts", "build": "tsc --module commonjs && rollup -c rollup.config.ts", "downloadnitro": "ts-node scripts/download-nitro.ts", "build:publish": "npm pack", diff --git a/nitro-node/postinstall.js b/nitro-node/postinstall.js index 18f6acbe1..f3b7ca271 100644 --- a/nitro-node/postinstall.js +++ b/nitro-node/postinstall.js @@ -1,5 +1,5 @@ // Only run if this package is installed as dependency if (process.env.INIT_CWD === process.cwd()) process.exit(); -const downloadNitro = require("./dist/download-nitro.cjs"); +const downloadNitro = require(`${__dirname}/scripts/download-nitro`); downloadNitro(); diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts index cbfa184d2..0ca6ddad0 100644 --- a/nitro-node/rollup.config.ts +++ b/nitro-node/rollup.config.ts @@ -34,7 +34,7 @@ export default [ { input: `scripts/download-nitro.ts`, output: [ - { file: "dist/download-nitro.cjs.js", format: "cjs", sourcemap: true }, + { file: "dist/scripts/download-nitro.js", format: "cjs", sourcemap: true }, ], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], diff --git a/nitro-node/scripts/download-nitro.ts b/nitro-node/scripts/download-nitro.ts index db28cc455..8bc459975 100644 --- a/nitro-node/scripts/download-nitro.ts +++ b/nitro-node/scripts/download-nitro.ts @@ -1,4 +1,4 @@ -import fs from 'node:fs'; +import fs from "node:fs"; import path from "node:path"; import download from "download"; import { Duplex } from "node:stream"; @@ -53,45 +53,48 @@ const getTarUrl = (version: string, suffix: string) => `https://github.com/janhq/nitro/releases/download/v${version}/nitro-${version}-${suffix}.tar.gz`; // Report download progress -const createProgressReporter = (variant: string) => (stream: Promise & Duplex) => - stream - .on( - "downloadProgress", - (progress: { transferred: any; total: any; percent: number }) => { - // Print and update progress on a single line of terminal - process.stdout.write( - `\r\x1b[K[${variant}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`, - ); - }, - ) - .on("end", () => { - // Jump to new line to log next message - console.log(); - console.log(`[${variant}] Finished downloading!`); - }); +const createProgressReporter = + (variant: string) => (stream: Promise & Duplex) => + stream + .on( + "downloadProgress", + (progress: { transferred: any; total: any; percent: number }) => { + // Print and update progress on a single line of terminal + process.stdout.write( + `\r\x1b[K[${variant}] ${progress.transferred}/${progress.total} ${Math.floor(progress.percent * 100)}%...`, + ); + }, + ) + .on("end", () => { + // Jump to new line to log next message + console.log(); + console.log(`[${variant}] Finished downloading!`); + }); // Download single binary -const downloadBinary = (version: string, suffix: string, destDirPath: string) => { +const downloadBinary = async ( + version: string, + suffix: string, + destDirPath: string, +) => { const tarUrl = getTarUrl(version, suffix); console.log(`Downloading ${tarUrl} to ${destDirPath}`); const progressReporter = createProgressReporter(suffix); - return progressReporter( + await progressReporter( download(tarUrl, destDirPath, { strip: 1, extract: true, - }), - ).then(() => { - // Set mode of downloaded binaries to executable - (fs - .readdirSync(destDirPath, { recursive: true }) as string[]) - .filter((fname) => fs.lstatSync(path.join(destDirPath, fname)).isFile()) - .filter((fname) => fname.includes('nitro')) - .forEach( - (nitroBinary) => { - const absPath = path.join(destDirPath, nitroBinary) - fs.chmodSync(absPath, fs.constants.S_IRWXU | fs.constants.S_IRWXG | fs.constants.S_IRWXO) - }) - }); + })); + // Set mode of downloaded binaries to executable + (fs.readdirSync(destDirPath, { recursive: true }) as string[]) + .filter((fname) => fs.lstatSync(path.join(destDirPath, fname)).isFile()) + .filter((fname_1) => fname_1.includes("nitro")) + .forEach((nitroBinary) => { + const absPath = path.join(destDirPath, nitroBinary); + fs.chmodSync( + absPath, + fs.constants.S_IRWXU | fs.constants.S_IRWXG | fs.constants.S_IRWXO); + }); }; // Download the binaries @@ -102,9 +105,32 @@ const downloadBinaries = (version: string, config: Record) => { ); }; +// Check for a files with nitro in name in the corresponding directory +const verifyDownloadedBinaries = () => { + // Check all the paths in variantConfig for a file with nitro in its name + return Object.values(variantConfig).every((binDirVariant: string) => { + try { + const pathToCheck = path.join(binDirVariant, "nitro"); + return ( + fs.readdirSync(binDirVariant, { recursive: true }) as string[] + ).some((fname) => { + const fullPath = path.join(binDirVariant, fname); + return fullPath.startsWith(pathToCheck); + }); + } catch (_e: any) { + return false; + } + }); +}; + // Call the download function with version and config -const downloadNitro = () => { - downloadBinaries(NITRO_VERSION, variantConfig); +const downloadNitro = async () => { + // Return early without downloading if nitro binaries are already downloaded + if (verifyDownloadedBinaries()) { + //console.log("Nitro binaries are already downloaded!"); + return; + } + return await downloadBinaries(NITRO_VERSION, variantConfig); }; export default downloadNitro; diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index 088886a50..715a8ad52 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -7,6 +7,7 @@ import fetchRT from "fetch-retry"; import osUtils from "os-utils"; import { getNitroProcessInfo, updateNvidiaInfo } from "./nvidia"; import { executableNitroFile } from "./execute"; +import downloadNitro from '../scripts/download-nitro'; // Polyfill fetch with retry const fetchRetry = fetchRT(fetch); @@ -102,6 +103,8 @@ async function runModel({ modelFullPath, promptTemplate, }: NitroModelInitOptions): Promise { + // Download nitro binaries if it's not already downloaded + await downloadNitro(); const files: string[] = fs.readdirSync(modelFullPath); // Look for model file with supported format From c60e4f47702117973dbd20530ae6bd6a74087983 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sat, 27 Jan 2024 20:00:02 +0700 Subject: [PATCH 23/49] feat(nitro-node): allow setting custom bin directory - Caller can set their custom bin directory before invoking `runModel` - Export download utility to let caller download binaries to the desired path - There is some regression magic for the postinstall script to make the installation process orchestrate with local development. Make sure the e2e-installation-test passed for any change of the logic inside. --- nitro-node/Makefile | 8 ++++ nitro-node/package.json | 1 + nitro-node/postinstall.js | 2 +- nitro-node/rollup.config.ts | 8 +++- nitro-node/scripts/download-nitro.ts | 71 +++++++++++++++------------- nitro-node/src/execute.ts | 2 +- nitro-node/src/index.ts | 37 +++++++++++++-- 7 files changed, 87 insertions(+), 42 deletions(-) diff --git a/nitro-node/Makefile b/nitro-node/Makefile index 8639f21f5..bfcf4faac 100644 --- a/nitro-node/Makefile +++ b/nitro-node/Makefile @@ -33,6 +33,14 @@ test: install pack: build yarn run build:publish +# Test that installation will also download nitro binaries +test-e2e-installation: pack +ifeq ($(OS),Windows_NT) + $env:NITRO_NODE_PKG=(Resolve-Path -Path janhq-nitro-node-1.0.0.tgz) node ..\.github\scripts\e2e-test-install-nitro-node.js +else + NITRO_NODE_PKG=$(realpath janhq-nitro-node-1.0.0.tgz) node ../.github/scripts/e2e-test-install-nitro-node.js +endif + clean: ifeq ($(OS),Windows_NT) powershell -Command "Remove-Item -Recurse -Force -Path *.tgz, .yarn, yarn.lock, package-lock.json, bin, dist" diff --git a/nitro-node/package.json b/nitro-node/package.json index 0a823ed86..d6cdf78cb 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -16,6 +16,7 @@ "exports": { ".": "./dist/index.js", "./main": "./dist/index.cjs.js", + "./scripts/download-nitro": "./dist/scripts/download-nitro.js", "./postinstall": "./postinstall.js" }, "devDependencies": { diff --git a/nitro-node/postinstall.js b/nitro-node/postinstall.js index f3b7ca271..2c5ad054b 100644 --- a/nitro-node/postinstall.js +++ b/nitro-node/postinstall.js @@ -1,5 +1,5 @@ // Only run if this package is installed as dependency if (process.env.INIT_CWD === process.cwd()) process.exit(); -const downloadNitro = require(`${__dirname}/scripts/download-nitro`); +const downloadNitro = require("./dist/scripts/download-nitro"); downloadNitro(); diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts index 0ca6ddad0..506c7ca2a 100644 --- a/nitro-node/rollup.config.ts +++ b/nitro-node/rollup.config.ts @@ -7,7 +7,7 @@ import json from "@rollup/plugin-json"; export default [ { input: `src/index.ts`, - output: [{ file: "dist/index.cjs.js", format: "cjs", sourcemap: true }], + output: [{ file: "dist/index.js", format: "cjs", sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { @@ -34,7 +34,11 @@ export default [ { input: `scripts/download-nitro.ts`, output: [ - { file: "dist/scripts/download-nitro.js", format: "cjs", sourcemap: true }, + { + file: "dist/scripts/download-nitro.js", + format: "cjs", + sourcemap: true, + }, ], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], diff --git a/nitro-node/scripts/download-nitro.ts b/nitro-node/scripts/download-nitro.ts index 8bc459975..b352d527a 100644 --- a/nitro-node/scripts/download-nitro.ts +++ b/nitro-node/scripts/download-nitro.ts @@ -11,28 +11,20 @@ const PLATFORM = process.env.npm_config_platform || process.platform; //const ARCH = process.env.npm_config_arch || process.arch; const linuxVariants = { - "linux-amd64": path.normalize(path.join(__dirname, "..", "bin", "linux-cpu")), - "linux-amd64-cuda-12-0": path.normalize( - path.join(__dirname, "..", "bin", "linux-cuda-12-0"), - ), - "linux-amd64-cuda-11-7": path.normalize( - path.join(__dirname, "..", "bin", "linux-cuda-11-7"), - ), + "linux-amd64": "linux-cpu", + "linux-amd64-cuda-12-0": "linux-cuda-12-0", + "linux-amd64-cuda-11-7": "linux-cuda-11-7", }; const darwinVariants = { - "mac-arm64": path.normalize(path.join(__dirname, "..", "bin", "mac-arm64")), - "mac-amd64": path.normalize(path.join(__dirname, "..", "bin", "mac-x64")), + "mac-arm64": "mac-arm64", + "mac-amd64": "mac-x64", }; const win32Variants = { - "win-amd64-cuda-12-0": path.normalize( - path.join(__dirname, "..", "bin", "win-cuda-12-0"), - ), - "win-amd64-cuda-11-7": path.normalize( - path.join(__dirname, "..", "bin", "win-cuda-11-7"), - ), - "win-amd64": path.normalize(path.join(__dirname, "..", "bin", "win-cpu")), + "win-amd64-cuda-12-0": "win-cuda-12-0", + "win-amd64-cuda-11-7": "win-cuda-11-7", + "win-amd64": "win-cpu", }; // Mapping to installation variants @@ -84,39 +76,50 @@ const downloadBinary = async ( download(tarUrl, destDirPath, { strip: 1, extract: true, - })); + }), + ); // Set mode of downloaded binaries to executable (fs.readdirSync(destDirPath, { recursive: true }) as string[]) - .filter((fname) => fs.lstatSync(path.join(destDirPath, fname)).isFile()) - .filter((fname_1) => fname_1.includes("nitro")) + .filter( + (fname) => + fname.includes("nitro") && + fs.lstatSync(path.join(destDirPath, fname)).isFile(), + ) .forEach((nitroBinary) => { const absPath = path.join(destDirPath, nitroBinary); fs.chmodSync( absPath, - fs.constants.S_IRWXU | fs.constants.S_IRWXG | fs.constants.S_IRWXO); + fs.constants.S_IRWXU | fs.constants.S_IRWXG | fs.constants.S_IRWXO, + ); }); }; // Download the binaries -const downloadBinaries = (version: string, config: Record) => { +const downloadBinaries = ( + version: string, + config: Record, + absBinDirPath: string, +) => { return Object.entries(config).reduce( - (p: Promise, [k, v]) => p.then(() => downloadBinary(version, k, v)), + (p: Promise, [k, v]) => + p.then(() => downloadBinary(version, k, path.join(absBinDirPath, v))), Promise.resolve(), ); }; // Check for a files with nitro in name in the corresponding directory -const verifyDownloadedBinaries = () => { +const verifyDownloadedBinaries = (absBinDirPath: string) => { // Check all the paths in variantConfig for a file with nitro in its name return Object.values(variantConfig).every((binDirVariant: string) => { try { - const pathToCheck = path.join(binDirVariant, "nitro"); - return ( - fs.readdirSync(binDirVariant, { recursive: true }) as string[] - ).some((fname) => { - const fullPath = path.join(binDirVariant, fname); - return fullPath.startsWith(pathToCheck); - }); + const dirToCheck = path.join(absBinDirPath, binDirVariant); + const pathToCheck = path.join(dirToCheck, "nitro"); + return (fs.readdirSync(dirToCheck, { recursive: true }) as string[]).some( + (fname) => { + const fullPath = path.join(dirToCheck, fname); + return fullPath.startsWith(pathToCheck); + }, + ); } catch (_e: any) { return false; } @@ -124,13 +127,15 @@ const verifyDownloadedBinaries = () => { }; // Call the download function with version and config -const downloadNitro = async () => { +const downloadNitro = async ( + absBinDirPath: string = path.join(__dirname, "..", '..', "bin"), +) => { // Return early without downloading if nitro binaries are already downloaded - if (verifyDownloadedBinaries()) { + if (verifyDownloadedBinaries(absBinDirPath)) { //console.log("Nitro binaries are already downloaded!"); return; } - return await downloadBinaries(NITRO_VERSION, variantConfig); + return await downloadBinaries(NITRO_VERSION, variantConfig, absBinDirPath); }; export default downloadNitro; diff --git a/nitro-node/src/execute.ts b/nitro-node/src/execute.ts index 6f09d538c..ef7b671b1 100644 --- a/nitro-node/src/execute.ts +++ b/nitro-node/src/execute.ts @@ -11,8 +11,8 @@ export interface NitroExecutableOptions { */ export const executableNitroFile = ( nvidiaSettings: NitroNvidiaConfig, + binaryFolder: string, ): NitroExecutableOptions => { - let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default let cudaVisibleDevices = ""; let binaryName = "nitro"; /** diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index 715a8ad52..e7695b303 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -7,7 +7,7 @@ import fetchRT from "fetch-retry"; import osUtils from "os-utils"; import { getNitroProcessInfo, updateNvidiaInfo } from "./nvidia"; import { executableNitroFile } from "./execute"; -import downloadNitro from '../scripts/download-nitro'; +import downloadNitro from "../scripts/download-nitro"; // Polyfill fetch with retry const fetchRetry = fetchRT(fetch); @@ -57,6 +57,32 @@ let nvidiaConfig: NitroNvidiaConfig = NVIDIA_DEFAULT_CONFIG; // The logger to use, default to stdout let log: NitroLogger = (message, ..._) => process.stdout.write(message + os.EOL); +// The absolute path to bin directory +let binPath: string = path.join(__dirname, "..", "bin"); + +/** + * Get current bin path + * @returns {string} The bin path + */ +function getBinPath(): string { + return binPath; +} +/** + * Set custom bin path + */ +function setBinPath(customBinPath: string): void | never { + // Check if the path is a directory + if ( + fs.existsSync(customBinPath) && + fs.statSync(customBinPath).isDirectory() + ) { + // If a valid directory, resolve to absolute path and set to binPath + const resolvedPath = path.resolve(customBinPath); + binPath = resolvedPath; + } else { + throw new Error(`${customBinPath} is not a valid directory!`); + } +} /** * Get current Nvidia config @@ -104,7 +130,7 @@ async function runModel({ promptTemplate, }: NitroModelInitOptions): Promise { // Download nitro binaries if it's not already downloaded - await downloadNitro(); + await downloadNitro(binPath); const files: string[] = fs.readdirSync(modelFullPath); // Look for model file with supported format @@ -368,8 +394,7 @@ function spawnNitroProcess(): Promise { log(`[NITRO]::Debug: Spawning Nitro subprocess...`); return new Promise(async (resolve, reject) => { - const binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default - const executableOptions = executableNitroFile(nvidiaConfig); + const executableOptions = executableNitroFile(nvidiaConfig, binPath); const args: string[] = ["1", LOCAL_HOST, PORT.toString()]; // Execute the binary @@ -380,7 +405,7 @@ function spawnNitroProcess(): Promise { executableOptions.executablePath, ["1", LOCAL_HOST, PORT.toString()], { - cwd: binaryFolder, + cwd: binPath, env: { ...process.env, CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices, @@ -426,6 +451,8 @@ function getResourcesInfo(): Promise { } export default { + getBinPath, + setBinPath, getNvidiaConfig, setNvidiaConfig, setLogger, From 6a42350dec3d352ba5d5ba9099452124223cb52a Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sun, 28 Jan 2024 03:18:03 +0700 Subject: [PATCH 24/49] fix(nitro-node): Fix exporting types --- nitro-node/Makefile | 4 +- nitro-node/package.json | 39 +- nitro-node/postinstall.js | 5 +- nitro-node/rollup.config.ts | 59 ++- nitro-node/src/execute.ts | 1 + nitro-node/src/index.ts | 469 +----------------- nitro-node/src/nitro.ts | 468 +++++++++++++++++ nitro-node/src/nvidia.ts | 1 + .../{ => src}/scripts/download-nitro.ts | 10 +- nitro-node/src/scripts/index.ts | 1 + .../{@types/global.d.ts => types/index.ts} | 34 +- nitro-node/test/nitro-process.test.ts | 5 +- nitro-node/tsconfig.json | 13 +- 13 files changed, 576 insertions(+), 533 deletions(-) create mode 100644 nitro-node/src/nitro.ts rename nitro-node/{ => src}/scripts/download-nitro.ts (95%) create mode 100644 nitro-node/src/scripts/index.ts rename nitro-node/src/{@types/global.d.ts => types/index.ts} (59%) diff --git a/nitro-node/Makefile b/nitro-node/Makefile index bfcf4faac..6f67435ee 100644 --- a/nitro-node/Makefile +++ b/nitro-node/Makefile @@ -36,9 +36,9 @@ pack: build # Test that installation will also download nitro binaries test-e2e-installation: pack ifeq ($(OS),Windows_NT) - $env:NITRO_NODE_PKG=(Resolve-Path -Path janhq-nitro-node-1.0.0.tgz) node ..\.github\scripts\e2e-test-install-nitro-node.js + $$env:NITRO_NODE_VERSION=(npm version --json | jq '.["@janhq/nitro-node"]' | foreach {$$_.replace('"','')}) $$env:NITRO_NODE_PKG=(Resolve-Path -Path "janhq-nitro-node$$NITRO_NODE_VERSION.tgz") node ..\.github\scripts\e2e-test-install-nitro-node.js else - NITRO_NODE_PKG=$(realpath janhq-nitro-node-1.0.0.tgz) node ../.github/scripts/e2e-test-install-nitro-node.js + NITRO_NODE_VERSION=$$(npm version --json | jq '.["@janhq/nitro-node"]' | tr -d '"') NITRO_NODE_PKG=$$(realpath "janhq-nitro-node-$${NITRO_NODE_VERSION}.tgz") node ../.github/scripts/e2e-test-install-nitro-node.js endif clean: diff --git a/nitro-node/package.json b/nitro-node/package.json index d6cdf78cb..c697639c0 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -2,23 +2,45 @@ "name": "@janhq/nitro-node", "version": "1.0.0", "description": "This NodeJS library is a wrapper for Nitro, a lightweight (3mb) inference engine written in C++. See nitro.jan.ai", - "main": "dist/index.js", - "node": "dist/index.cjs.js", + "main": "./dist/index", + "module": "./dist/index.esm.js", + "types": "./dist/types/index", "author": "Jan ", "license": "AGPL-3.0", "scripts": { "test": "jest --verbose --detectOpenHandles", "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "downloadnitro": "ts-node scripts/download-nitro.ts", + "downloadnitro": "tsx src/scripts/download-nitro.ts", "build:publish": "npm pack", "postinstall": "node -r @janhq/nitro-node/postinstall" }, "exports": { - ".": "./dist/index.js", - "./main": "./dist/index.cjs.js", - "./scripts/download-nitro": "./dist/scripts/download-nitro.js", + ".": { + "import": "./dist/index.esm.js", + "require": "./dist/index.cjs", + "types": "./dist/types/index.d.ts" + }, + "./scripts": { + "import": "./dist/scripts/index.esm.js", + "require": "./dist/scripts/index.cjs", + "types": "./dist/types/scripts/index.d.ts" + }, "./postinstall": "./postinstall.js" }, + "typesVersions": { + "*": { + ".": [ + "./dist/index.esm.js", + "./dist/index.esm.js.map", + "./dist/types/index.d.ts" + ], + "./scripts": [ + "./dist/scripts/index.esm.js", + "./dist/scripts/index.esm.js.map", + "./dist/types/scripts/index.d.ts" + ] + } + }, "devDependencies": { "@rollup/plugin-commonjs": "^25.0.7", "@rollup/plugin-json": "^6.1.0", @@ -33,7 +55,7 @@ "rollup-plugin-sourcemaps": "^0.6.3", "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^29.1.2", - "ts-node": "^10.9.2", + "tsx": "^4.7.0", "typescript": "^5.3.3" }, "dependencies": { @@ -47,8 +69,9 @@ }, "files": [ "postinstall.js", - "dist/*", + "dist", "package.json", + "tsconfig.json", "README.md" ] } diff --git a/nitro-node/postinstall.js b/nitro-node/postinstall.js index 2c5ad054b..281a678e3 100644 --- a/nitro-node/postinstall.js +++ b/nitro-node/postinstall.js @@ -1,5 +1,6 @@ // Only run if this package is installed as dependency if (process.env.INIT_CWD === process.cwd()) process.exit(); -const downloadNitro = require("./dist/scripts/download-nitro"); -downloadNitro(); +const path = require("node:path"); +const { downloadNitro } = require("@janhq/nitro-node/scripts"); +downloadNitro(path.join(__dirname, "bin")); diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts index 506c7ca2a..0dfd92a66 100644 --- a/nitro-node/rollup.config.ts +++ b/nitro-node/rollup.config.ts @@ -7,24 +7,38 @@ import json from "@rollup/plugin-json"; export default [ { input: `src/index.ts`, - output: [{ file: "dist/index.js", format: "cjs", sourcemap: true }], + output: [ + { file: "dist/index.cjs", format: "cjs", sourcemap: true }, + { file: "dist/index.esm.js", format: "es", sourcemap: true }, + ], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: [], + external: [ + "electron", + "node:fs", + "node:child_process", + "node:os", + "node:path", + ], watch: { include: "src/**", }, plugins: [ // Allow json resolution json(), - // Compile TypeScript files - typescript({ useTsconfigDeclarationDir: true }), - // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) - commonjs(), // Allow node_modules resolution, so you can use 'external' to control // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ extensions: [".ts", ".js", ".json"], + preferBuiltins: false, + }), + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + // This should be after resolve() plugin + commonjs(), + // Compile TypeScript files + typescript({ + useTsconfigDeclarationDir: true, + tsconfig: "tsconfig.json", }), // Resolve source maps to the original source @@ -32,33 +46,42 @@ export default [ ], }, { - input: `scripts/download-nitro.ts`, + input: `src/scripts/index.ts`, output: [ { - file: "dist/scripts/download-nitro.js", + file: "dist/scripts/index.cjs", format: "cjs", sourcemap: true, }, + { + file: "dist/scripts/index.esm.js", + format: "es", + sourcemap: true, + }, ], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: [], + external: ["electron", "node:fs", "node:path"], watch: { - include: "scripts/**", + include: "src/scripts/**", }, plugins: [ // Allow json resolution json(), - // Compile TypeScript files - typescript({ useTsconfigDeclarationDir: true }), - // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) - commonjs(), // Allow node_modules resolution, so you can use 'external' to control // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage - //resolve({ - // extensions: [".ts", ".js", ".json"], - // preferBuiltins: false, - //}), + resolve({ + extensions: [".ts", ".js", ".json"], + preferBuiltins: false, + }), + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) + // This should be after resolve() plugin + commonjs(), + // Compile TypeScript files + typescript({ + useTsconfigDeclarationDir: true, + tsconfig: "tsconfig.json", + }), // Resolve source maps to the original source sourceMaps(), diff --git a/nitro-node/src/execute.ts b/nitro-node/src/execute.ts index ef7b671b1..b713ddb1f 100644 --- a/nitro-node/src/execute.ts +++ b/nitro-node/src/execute.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import { NitroNvidiaConfig } from "./types"; export interface NitroExecutableOptions { executablePath: string; diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index e7695b303..c44d2e13c 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -1,467 +1,2 @@ -import os from "node:os"; -import fs from "node:fs"; -import path from "node:path"; -import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; -import tcpPortUsed from "tcp-port-used"; -import fetchRT from "fetch-retry"; -import osUtils from "os-utils"; -import { getNitroProcessInfo, updateNvidiaInfo } from "./nvidia"; -import { executableNitroFile } from "./execute"; -import downloadNitro from "../scripts/download-nitro"; -// Polyfill fetch with retry -const fetchRetry = fetchRT(fetch); - -// The PORT to use for the Nitro subprocess -const PORT = 3928; -// The HOST address to use for the Nitro subprocess -const LOCAL_HOST = "127.0.0.1"; -// The URL for the Nitro subprocess -const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`; -// The URL for the Nitro subprocess to load a model -const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`; -// The URL for the Nitro subprocess to validate a model -const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; -// The URL for the Nitro subprocess to kill itself -const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; -// The URL for the Nitro subprocess to run chat completion -const NITRO_HTTP_CHAT_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/chat_completion`; - -// The default config for using Nvidia GPU -const NVIDIA_DEFAULT_CONFIG: NitroNvidiaConfig = { - notify: true, - run_mode: "cpu", - nvidia_driver: { - exist: false, - version: "", - }, - cuda: { - exist: false, - version: "", - }, - gpus: [], - gpu_highest_vram: "", -}; - -// The supported model format -// TODO: Should be an array to support more models -const SUPPORTED_MODEL_FORMATS = [".gguf"]; - -// The subprocess instance for Nitro -let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; -// The current model file url -let currentModelFile: string = ""; -// The current model settings -let currentSettings: NitroModelSetting | undefined = undefined; -// The Nvidia info file for checking for CUDA support on the system -let nvidiaConfig: NitroNvidiaConfig = NVIDIA_DEFAULT_CONFIG; -// The logger to use, default to stdout -let log: NitroLogger = (message, ..._) => - process.stdout.write(message + os.EOL); -// The absolute path to bin directory -let binPath: string = path.join(__dirname, "..", "bin"); - -/** - * Get current bin path - * @returns {string} The bin path - */ -function getBinPath(): string { - return binPath; -} -/** - * Set custom bin path - */ -function setBinPath(customBinPath: string): void | never { - // Check if the path is a directory - if ( - fs.existsSync(customBinPath) && - fs.statSync(customBinPath).isDirectory() - ) { - // If a valid directory, resolve to absolute path and set to binPath - const resolvedPath = path.resolve(customBinPath); - binPath = resolvedPath; - } else { - throw new Error(`${customBinPath} is not a valid directory!`); - } -} - -/** - * Get current Nvidia config - * @returns {NitroNvidiaConfig} A copy of the config object - * The returned object should be used for reading only - * Writing to config should be via the function {@setNvidiaConfig} - */ -function getNvidiaConfig(): NitroNvidiaConfig { - return Object.assign({}, nvidiaConfig); -} - -/** - * Set custom Nvidia config for running inference over GPU - * @param {NitroNvidiaConfig} config The new config to apply - */ -function setNvidiaConfig(config: NitroNvidiaConfig) { - nvidiaConfig = config; -} - -/** - * Set logger before running nitro - * @param {NitroLogger} logger The logger to use - */ -function setLogger(logger: NitroLogger) { - log = logger; -} - -/** - * Stops a Nitro subprocess. - * @param wrapper - The model wrapper. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -function stopModel(): Promise { - return killSubprocess(); -} - -/** - * Initializes a Nitro subprocess to load a machine learning model. - * @param modelFullPath - The absolute full path to model directory. - * @param promptTemplate - The template to use for generating prompts. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - */ -async function runModel({ - modelFullPath, - promptTemplate, -}: NitroModelInitOptions): Promise { - // Download nitro binaries if it's not already downloaded - await downloadNitro(binPath); - const files: string[] = fs.readdirSync(modelFullPath); - - // Look for model file with supported format - const ggufBinFile = files.find( - (file) => - file === path.basename(modelFullPath) || - SUPPORTED_MODEL_FORMATS.some((ext) => file.toLowerCase().endsWith(ext)), - ); - - if (!ggufBinFile) return Promise.reject("No GGUF model file found"); - - currentModelFile = path.join(modelFullPath, ggufBinFile); - - const nitroResourceProbe = await getResourcesInfo(); - // Convert promptTemplate to system_prompt, user_prompt, ai_prompt - const prompt: NitroPromptSetting = {}; - if (promptTemplate) { - try { - Object.assign(prompt, promptTemplateConverter(promptTemplate)); - } catch (e: any) { - return Promise.reject(e); - } - } - - currentSettings = { - ...prompt, - llama_model_path: currentModelFile, - // This is critical and requires real system information - cpu_threads: Math.max( - 1, - Math.round(nitroResourceProbe.numCpuPhysicalCore / 2), - ), - }; - return runNitroAndLoadModel(); -} - -/** - * 1. Spawn Nitro process - * 2. Load model into Nitro subprocess - * 3. Validate model status - * @returns - */ -async function runNitroAndLoadModel(): Promise< - NitroModelOperationResponse | { error: any } -> { - // Gather system information for CPU physical cores and memory - return killSubprocess() - .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => { - /** - * There is a problem with Windows process manager - * Should wait for awhile to make sure the port is free and subprocess is killed - * The tested threshold is 500ms - **/ - if (process.platform === "win32") { - return new Promise((resolve) => setTimeout(() => resolve({}), 500)); - } else { - return Promise.resolve({}); - } - }) - .then(spawnNitroProcess) - .then(() => loadLLMModel(currentSettings)) - .then(validateModelStatus) - .catch((err) => { - // TODO: Broadcast error so app could display proper error message - log(`[NITRO]::Error: ${err}`); - return { error: err }; - }); -} - -/** - * Parse prompt template into agrs settings - * @param {string} promptTemplate Template as string - * @returns {(NitroPromptSetting | never)} parsed prompt setting - * @throws {Error} if cannot split promptTemplate - */ -function promptTemplateConverter( - promptTemplate: string, -): NitroPromptSetting | never { - // Split the string using the markers - const systemMarker = "{system_message}"; - const promptMarker = "{prompt}"; - - if ( - promptTemplate.includes(systemMarker) && - promptTemplate.includes(promptMarker) - ) { - // Find the indices of the markers - const systemIndex = promptTemplate.indexOf(systemMarker); - const promptIndex = promptTemplate.indexOf(promptMarker); - - // Extract the parts of the string - const system_prompt = promptTemplate.substring(0, systemIndex); - const user_prompt = promptTemplate.substring( - systemIndex + systemMarker.length, - promptIndex, - ); - const ai_prompt = promptTemplate.substring( - promptIndex + promptMarker.length, - ); - - // Return the split parts - return { system_prompt, user_prompt, ai_prompt }; - } else if (promptTemplate.includes(promptMarker)) { - // Extract the parts of the string for the case where only promptMarker is present - const promptIndex = promptTemplate.indexOf(promptMarker); - const user_prompt = promptTemplate.substring(0, promptIndex); - const ai_prompt = promptTemplate.substring( - promptIndex + promptMarker.length, - ); - - // Return the split parts - return { user_prompt, ai_prompt }; - } - - // Throw error if none of the conditions are met - throw Error("Cannot split prompt template"); -} - -/** - * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. - * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. - */ -async function loadLLMModel(settings: any): Promise { - log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`); - try { - const res = await fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(settings), - retries: 3, - retryDelay: 500, - }); - // FIXME: Actually check response, as the model directory might not exist - log( - `[NITRO]::Debug: Load model success with response ${JSON.stringify(res)}`, - ); - return await Promise.resolve(res); - } catch (err) { - log(`[NITRO]::Error: Load model failed with error ${err}`); - return await Promise.reject(); - } -} - -/** - * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified - * @param {any} request The request that is then sent to nitro - * @param {WritableStream} outStream Optional stream that consume the response body - * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. - * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data - */ -async function chatCompletion( - request: any, - outStream?: WritableStream, -): Promise { - if (outStream) { - // Add stream option if there is an outStream specified when calling this function - Object.assign(request, { - stream: true, - }); - } - log( - `[NITRO]::Debug: Running chat completion with request ${JSON.stringify(request)}`, - ); - return fetchRetry(NITRO_HTTP_CHAT_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - Accept: Boolean(outStream) ? "text/event-stream" : "application/json", - }, - body: JSON.stringify(request), - retries: 3, - retryDelay: 500, - }) - .then(async (response) => { - if (outStream) { - if (!response.body) { - throw new Error("Error running chat completion"); - } - const outPipe = response.body - .pipeThrough(new TextDecoderStream()) - .pipeTo(outStream); - // Wait for all the streams to complete before returning from async function - await outPipe; - } - log(`[NITRO]::Debug: Chat completion success`); - return response; - }) - .catch((err) => { - log(`[NITRO]::Error: Chat completion failed with error ${err}`); - throw err; - }); -} - -/** - * Validates the status of a model. - * @returns {Promise} A promise that resolves to an object. - * If the model is loaded successfully, the object is empty. - * If the model is not loaded successfully, the object contains an error message. - */ -async function validateModelStatus(): Promise { - // Send a GET request to the validation URL. - // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. - return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { - method: "GET", - headers: { - "Content-Type": "application/json", - }, - retries: 5, - retryDelay: 500, - }).then(async (res: Response) => { - log( - `[NITRO]::Debug: Validate model state success with response ${JSON.stringify( - res, - )}`, - ); - // If the response is OK, check model_loaded status. - if (res.ok) { - const body = await res.json(); - // If the model is loaded, return an empty object. - // Otherwise, return an object with an error message. - if (body.model_loaded) { - return Promise.resolve({}); - } - } - return Promise.resolve({ error: "Validate model status failed" }); - }); -} - -/** - * Terminates the Nitro subprocess. - * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. - */ -async function killSubprocess(): Promise { - const controller = new AbortController(); - setTimeout(() => controller.abort(), 5000); - log(`[NITRO]::Debug: Request to kill Nitro`); - - return fetch(NITRO_HTTP_KILL_URL, { - method: "DELETE", - signal: controller.signal, - }) - .then(() => { - subprocess?.kill(); - subprocess = undefined; - }) - .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) - .then(() => Promise.resolve({})) - .catch((err) => ({ error: err })); -} - -/** - * Spawns a Nitro subprocess. - * @returns A promise that resolves when the Nitro subprocess is started. - */ -function spawnNitroProcess(): Promise { - log(`[NITRO]::Debug: Spawning Nitro subprocess...`); - - return new Promise(async (resolve, reject) => { - const executableOptions = executableNitroFile(nvidiaConfig, binPath); - - const args: string[] = ["1", LOCAL_HOST, PORT.toString()]; - // Execute the binary - log( - `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}`, - ); - subprocess = spawn( - executableOptions.executablePath, - ["1", LOCAL_HOST, PORT.toString()], - { - cwd: binPath, - env: { - ...process.env, - CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices, - }, - }, - ); - - // Handle subprocess output - subprocess.stdout.on("data", (data: any) => { - log(`[NITRO]::Debug: ${data}`); - }); - - subprocess.stderr.on("data", (data: any) => { - log(`[NITRO]::Error: ${data}`); - }); - - subprocess.on("close", (code: any) => { - log(`[NITRO]::Debug: Nitro exited with code: ${code}`); - subprocess = undefined; - reject(`child process exited with code ${code}`); - }); - - tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { - log(`[NITRO]::Debug: Nitro is ready`); - resolve({}); - }); - }); -} - -/** - * Get the system resources information - */ -function getResourcesInfo(): Promise { - return new Promise(async (resolve) => { - const cpu = osUtils.cpuCount(); - log(`[NITRO]::CPU informations - ${cpu}`); - const response: ResourcesInfo = { - numCpuPhysicalCore: cpu, - memAvailable: 0, - }; - resolve(response); - }); -} - -export default { - getBinPath, - setBinPath, - getNvidiaConfig, - setNvidiaConfig, - setLogger, - runModel, - stopModel, - loadLLMModel, - validateModelStatus, - chatCompletion, - killSubprocess, - updateNvidiaInfo: async () => await updateNvidiaInfo(nvidiaConfig), - getCurrentNitroProcessInfo: () => getNitroProcessInfo(subprocess), -}; +export * from "./types"; +export * from "./nitro"; diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts new file mode 100644 index 000000000..46a80c716 --- /dev/null +++ b/nitro-node/src/nitro.ts @@ -0,0 +1,468 @@ +import os from "node:os"; +import fs from "node:fs"; +import path from "node:path"; +import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import tcpPortUsed from "tcp-port-used"; +import fetchRT from "fetch-retry"; +import osUtils from "os-utils"; +import { + getNitroProcessInfo, + updateNvidiaInfo as _updateNvidiaInfo, +} from "./nvidia"; +import { executableNitroFile } from "./execute"; +import { + NitroNvidiaConfig, + NitroModelSetting, + NitroPromptSetting, + NitroLogger, + NitroModelOperationResponse, + NitroModelInitOptions, + ResourcesInfo, +} from "./types"; +import { downloadNitro } from "./scripts"; +// Polyfill fetch with retry +console.log(typeof fetchRT); +const fetchRetry = fetchRT(fetch); + +// The PORT to use for the Nitro subprocess +const PORT = 3928; +// The HOST address to use for the Nitro subprocess +const LOCAL_HOST = "127.0.0.1"; +// The URL for the Nitro subprocess +const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`; +// The URL for the Nitro subprocess to load a model +const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`; +// The URL for the Nitro subprocess to validate a model +const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; +// The URL for the Nitro subprocess to kill itself +const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; +// The URL for the Nitro subprocess to run chat completion +const NITRO_HTTP_CHAT_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/chat_completion`; + +// The default config for using Nvidia GPU +const NVIDIA_DEFAULT_CONFIG: NitroNvidiaConfig = { + notify: true, + run_mode: "cpu", + nvidia_driver: { + exist: false, + version: "", + }, + cuda: { + exist: false, + version: "", + }, + gpus: [], + gpu_highest_vram: "", +}; + +// The supported model format +// TODO: Should be an array to support more models +const SUPPORTED_MODEL_FORMATS = [".gguf"]; + +// The subprocess instance for Nitro +let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; +// The current model file url +let currentModelFile: string = ""; +// The current model settings +let currentSettings: NitroModelSetting | undefined = undefined; +// The Nvidia info file for checking for CUDA support on the system +let nvidiaConfig: NitroNvidiaConfig = NVIDIA_DEFAULT_CONFIG; +// The logger to use, default to stdout +let log: NitroLogger = (message, ..._) => + process.stdout.write(message + os.EOL); +// The absolute path to bin directory +let binPath: string = path.join(__dirname, "..", "bin"); + +/** + * Get current bin path + * @returns {string} The bin path + */ +export function getBinPath(): string { + return binPath; +} +/** + * Set custom bin path + */ +export function setBinPath(customBinPath: string): void | never { + // Check if the path is a directory + if ( + fs.existsSync(customBinPath) && + fs.statSync(customBinPath).isDirectory() + ) { + // If a valid directory, resolve to absolute path and set to binPath + const resolvedPath = path.resolve(customBinPath); + binPath = resolvedPath; + } else { + throw new Error(`${customBinPath} is not a valid directory!`); + } +} + +/** + * Get current Nvidia config + * @returns {NitroNvidiaConfig} A copy of the config object + * The returned object should be used for reading only + * Writing to config should be via the function {@setNvidiaConfig} + */ +export function getNvidiaConfig(): NitroNvidiaConfig { + return Object.assign({}, nvidiaConfig); +} + +/** + * Set custom Nvidia config for running inference over GPU + * @param {NitroNvidiaConfig} config The new config to apply + */ +export function setNvidiaConfig(config: NitroNvidiaConfig) { + nvidiaConfig = config; +} + +/** + * Set logger before running nitro + * @param {NitroLogger} logger The logger to use + */ +export function setLogger(logger: NitroLogger) { + log = logger; +} + +/** + * Stops a Nitro subprocess. + * @param wrapper - The model wrapper. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +export function stopModel(): Promise { + return killSubprocess(); +} + +/** + * Initializes a Nitro subprocess to load a machine learning model. + * @param modelFullPath - The absolute full path to model directory. + * @param promptTemplate - The template to use for generating prompts. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + */ +export async function runModel({ + modelFullPath, + promptTemplate, +}: NitroModelInitOptions): Promise { + // Download nitro binaries if it's not already downloaded + await downloadNitro(binPath); + const files: string[] = fs.readdirSync(modelFullPath); + + // Look for model file with supported format + const ggufBinFile = files.find( + (file) => + file === path.basename(modelFullPath) || + SUPPORTED_MODEL_FORMATS.some((ext) => file.toLowerCase().endsWith(ext)), + ); + + if (!ggufBinFile) return Promise.reject("No GGUF model file found"); + + currentModelFile = path.join(modelFullPath, ggufBinFile); + + const nitroResourceProbe = await getResourcesInfo(); + // Convert promptTemplate to system_prompt, user_prompt, ai_prompt + const prompt: NitroPromptSetting = {}; + if (promptTemplate) { + try { + Object.assign(prompt, promptTemplateConverter(promptTemplate)); + } catch (e: any) { + return Promise.reject(e); + } + } + + currentSettings = { + ...prompt, + llama_model_path: currentModelFile, + // This is critical and requires real system information + cpu_threads: Math.max( + 1, + Math.round(nitroResourceProbe.numCpuPhysicalCore / 2), + ), + }; + return runNitroAndLoadModel(); +} + +/** + * 1. Spawn Nitro process + * 2. Load model into Nitro subprocess + * 3. Validate model status + * @returns + */ +export async function runNitroAndLoadModel(): Promise< + NitroModelOperationResponse | { error: any } +> { + // Gather system information for CPU physical cores and memory + return killSubprocess() + .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) + .then(() => { + /** + * There is a problem with Windows process manager + * Should wait for awhile to make sure the port is free and subprocess is killed + * The tested threshold is 500ms + **/ + if (process.platform === "win32") { + return new Promise((resolve) => setTimeout(() => resolve({}), 500)); + } else { + return Promise.resolve({}); + } + }) + .then(spawnNitroProcess) + .then(() => loadLLMModel(currentSettings)) + .then(validateModelStatus) + .catch((err) => { + // TODO: Broadcast error so app could display proper error message + log(`[NITRO]::Error: ${err}`); + return { error: err }; + }); +} + +/** + * Parse prompt template into agrs settings + * @param {string} promptTemplate Template as string + * @returns {(NitroPromptSetting | never)} parsed prompt setting + * @throws {Error} if cannot split promptTemplate + */ +export function promptTemplateConverter( + promptTemplate: string, +): NitroPromptSetting | never { + // Split the string using the markers + const systemMarker = "{system_message}"; + const promptMarker = "{prompt}"; + + if ( + promptTemplate.includes(systemMarker) && + promptTemplate.includes(promptMarker) + ) { + // Find the indices of the markers + const systemIndex = promptTemplate.indexOf(systemMarker); + const promptIndex = promptTemplate.indexOf(promptMarker); + + // Extract the parts of the string + const system_prompt = promptTemplate.substring(0, systemIndex); + const user_prompt = promptTemplate.substring( + systemIndex + systemMarker.length, + promptIndex, + ); + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length, + ); + + // Return the split parts + return { system_prompt, user_prompt, ai_prompt }; + } else if (promptTemplate.includes(promptMarker)) { + // Extract the parts of the string for the case where only promptMarker is present + const promptIndex = promptTemplate.indexOf(promptMarker); + const user_prompt = promptTemplate.substring(0, promptIndex); + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length, + ); + + // Return the split parts + return { user_prompt, ai_prompt }; + } + + // Throw error if none of the conditions are met + throw Error("Cannot split prompt template"); +} + +/** + * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. + * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. + */ +export async function loadLLMModel(settings: any): Promise { + log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`); + try { + const res = await fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(settings), + retries: 3, + retryDelay: 500, + }); + // FIXME: Actually check response, as the model directory might not exist + log( + `[NITRO]::Debug: Load model success with response ${JSON.stringify(res)}`, + ); + return await Promise.resolve(res); + } catch (err) { + log(`[NITRO]::Error: Load model failed with error ${err}`); + return await Promise.reject(); + } +} + +/** + * Run chat completion by sending a HTTP POST request and stream the response if outStream is specified + * @param {any} request The request that is then sent to nitro + * @param {WritableStream} outStream Optional stream that consume the response body + * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. + * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data + */ +export async function chatCompletion( + request: any, + outStream?: WritableStream, +): Promise { + if (outStream) { + // Add stream option if there is an outStream specified when calling this function + Object.assign(request, { + stream: true, + }); + } + log( + `[NITRO]::Debug: Running chat completion with request ${JSON.stringify(request)}`, + ); + return fetchRetry(NITRO_HTTP_CHAT_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + Accept: Boolean(outStream) ? "text/event-stream" : "application/json", + }, + body: JSON.stringify(request), + retries: 3, + retryDelay: 500, + }) + .then(async (response) => { + if (outStream) { + if (!response.body) { + throw new Error("Error running chat completion"); + } + const outPipe = response.body + .pipeThrough(new TextDecoderStream()) + .pipeTo(outStream); + // Wait for all the streams to complete before returning from async function + await outPipe; + } + log(`[NITRO]::Debug: Chat completion success`); + return response; + }) + .catch((err) => { + log(`[NITRO]::Error: Chat completion failed with error ${err}`); + throw err; + }); +} + +/** + * Validates the status of a model. + * @returns {Promise} A promise that resolves to an object. + * If the model is loaded successfully, the object is empty. + * If the model is not loaded successfully, the object contains an error message. + */ +export async function validateModelStatus(): Promise { + // Send a GET request to the validation URL. + // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. + return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + retries: 5, + retryDelay: 500, + }).then(async (res: Response) => { + log( + `[NITRO]::Debug: Validate model state success with response ${JSON.stringify( + res, + )}`, + ); + // If the response is OK, check model_loaded status. + if (res.ok) { + const body = await res.json(); + // If the model is loaded, return an empty object. + // Otherwise, return an object with an error message. + if (body.model_loaded) { + return Promise.resolve({}); + } + } + return Promise.resolve({ error: "Validate model status failed" }); + }); +} + +/** + * Terminates the Nitro subprocess. + * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. + */ +export async function killSubprocess(): Promise { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 5000); + log(`[NITRO]::Debug: Request to kill Nitro`); + + return fetch(NITRO_HTTP_KILL_URL, { + method: "DELETE", + signal: controller.signal, + }) + .then(() => { + subprocess?.kill(); + subprocess = undefined; + }) + .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) + .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) + .then(() => Promise.resolve({})) + .catch((err) => ({ error: err })); +} + +/** + * Spawns a Nitro subprocess. + * @returns A promise that resolves when the Nitro subprocess is started. + */ +export function spawnNitroProcess(): Promise { + log(`[NITRO]::Debug: Spawning Nitro subprocess...`); + + return new Promise(async (resolve, reject) => { + const executableOptions = executableNitroFile(nvidiaConfig, binPath); + + const args: string[] = ["1", LOCAL_HOST, PORT.toString()]; + // Execute the binary + log( + `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}`, + ); + subprocess = spawn( + executableOptions.executablePath, + ["1", LOCAL_HOST, PORT.toString()], + { + cwd: binPath, + env: { + ...process.env, + CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices, + }, + }, + ); + + // Handle subprocess output + subprocess.stdout.on("data", (data: any) => { + log(`[NITRO]::Debug: ${data}`); + }); + + subprocess.stderr.on("data", (data: any) => { + log(`[NITRO]::Error: ${data}`); + }); + + subprocess.on("close", (code: any) => { + log(`[NITRO]::Debug: Nitro exited with code: ${code}`); + subprocess = undefined; + reject(`child process exited with code ${code}`); + }); + + tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { + log(`[NITRO]::Debug: Nitro is ready`); + resolve({}); + }); + }); +} + +/** + * Get the system resources information + */ +export function getResourcesInfo(): Promise { + return new Promise(async (resolve) => { + const cpu = osUtils.cpuCount(); + log(`[NITRO]::CPU informations - ${cpu}`); + const response: ResourcesInfo = { + numCpuPhysicalCore: cpu, + memAvailable: 0, + }; + resolve(response); + }); +} + +export const updateNvidiaInfo = async () => + await _updateNvidiaInfo(nvidiaConfig); +export const getCurrentNitroProcessInfo = () => getNitroProcessInfo(subprocess); diff --git a/nitro-node/src/nvidia.ts b/nitro-node/src/nvidia.ts index c73173e73..8ae6d03c9 100644 --- a/nitro-node/src/nvidia.ts +++ b/nitro-node/src/nvidia.ts @@ -1,5 +1,6 @@ import { existsSync } from "node:fs"; import { exec } from "node:child_process"; +import { NitroNvidiaConfig } from "./types"; import path from "node:path"; /** diff --git a/nitro-node/scripts/download-nitro.ts b/nitro-node/src/scripts/download-nitro.ts similarity index 95% rename from nitro-node/scripts/download-nitro.ts rename to nitro-node/src/scripts/download-nitro.ts index b352d527a..70aa7b333 100644 --- a/nitro-node/scripts/download-nitro.ts +++ b/nitro-node/src/scripts/download-nitro.ts @@ -127,9 +127,7 @@ const verifyDownloadedBinaries = (absBinDirPath: string) => { }; // Call the download function with version and config -const downloadNitro = async ( - absBinDirPath: string = path.join(__dirname, "..", '..', "bin"), -) => { +export const downloadNitro = async (absBinDirPath: string) => { // Return early without downloading if nitro binaries are already downloaded if (verifyDownloadedBinaries(absBinDirPath)) { //console.log("Nitro binaries are already downloaded!"); @@ -138,9 +136,9 @@ const downloadNitro = async ( return await downloadBinaries(NITRO_VERSION, variantConfig, absBinDirPath); }; -export default downloadNitro; - // Run script if called directly instead of import as module if (require.main === module) { - downloadNitro(); + // Assume calling the source typescript files + // bin path will be relative to the source code root + downloadNitro(path.join(__dirname, '..', "..", "bin")); } diff --git a/nitro-node/src/scripts/index.ts b/nitro-node/src/scripts/index.ts new file mode 100644 index 000000000..3b6630f8e --- /dev/null +++ b/nitro-node/src/scripts/index.ts @@ -0,0 +1 @@ +export * from "./download-nitro"; diff --git a/nitro-node/src/@types/global.d.ts b/nitro-node/src/types/index.ts similarity index 59% rename from nitro-node/src/@types/global.d.ts rename to nitro-node/src/types/index.ts index bbf8a7f6c..0c9e5c749 100644 --- a/nitro-node/src/@types/global.d.ts +++ b/nitro-node/src/types/index.ts @@ -2,12 +2,12 @@ * The response from the initModel function. * @property error - An error message if the model fails to load. */ -interface NitroModelOperationResponse { +export interface NitroModelOperationResponse { error?: any; modelFile?: string; } -interface ResourcesInfo { +export interface ResourcesInfo { numCpuPhysicalCore: number; memAvailable: number; } @@ -15,7 +15,7 @@ interface ResourcesInfo { /** * Setting for prompts when inferencing with Nitro */ -interface NitroPromptSetting { +export interface NitroPromptSetting { system_prompt?: string; ai_prompt?: string; user_prompt?: string; @@ -24,7 +24,7 @@ interface NitroPromptSetting { /** * The available model settings */ -interface NitroModelSetting extends NitroPromptSetting { +export interface NitroModelSetting extends NitroPromptSetting { llama_model_path: string; cpu_threads: number; } @@ -32,7 +32,7 @@ interface NitroModelSetting extends NitroPromptSetting { /** * The response object for model init operation. */ -interface NitroModelInitOptions { +export interface NitroModelInitOptions { modelFullPath: string; promptTemplate?: string; } @@ -40,24 +40,24 @@ interface NitroModelInitOptions { /** * Logging interface for passing custom logger to nitro-node */ -interface NitroLogger { +export interface NitroLogger { (message: string, fileName?: string): void; } /** * Nvidia settings */ -interface NitroNvidiaConfig { - notify: boolean, - run_mode: "cpu" | "gpu", +export interface NitroNvidiaConfig { + notify: boolean; + run_mode: "cpu" | "gpu"; nvidia_driver: { - exist: boolean, - version: string, - }, + exist: boolean; + version: string; + }; cuda: { - exist: boolean, - version: string, - }, - gpus: { id: string, vram: string }[], - gpu_highest_vram: string, + exist: boolean; + version: string; + }; + gpus: { id: string; vram: string }[]; + gpu_highest_vram: string; } \ No newline at end of file diff --git a/nitro-node/test/nitro-process.test.ts b/nitro-node/test/nitro-process.test.ts index 778d71c2b..97f6cb490 100644 --- a/nitro-node/test/nitro-process.test.ts +++ b/nitro-node/test/nitro-process.test.ts @@ -7,14 +7,13 @@ import path from "node:path"; import download from "download"; import { Duplex } from "node:stream"; -import { default as nitro } from "../src/index"; -const { +import { stopModel, runModel, loadLLMModel, validateModelStatus, chatCompletion, -} = nitro; +} from "../src"; // FIXME: Shorthand only possible for es6 targets and up //import * as model from './model.json' assert {type: 'json'} diff --git a/nitro-node/tsconfig.json b/nitro-node/tsconfig.json index a055f5325..943393d7a 100644 --- a/nitro-node/tsconfig.json +++ b/nitro-node/tsconfig.json @@ -6,30 +6,23 @@ "lib": [ "es2015", "es2016", - "es2017" + "es2017", + "dom" ], "strict": true, "sourceMap": true, "declaration": true, "allowSyntheticDefaultImports": true, + "esModuleInterop": true, "experimentalDecorators": true, "emitDecoratorMetadata": true, "declarationDir": "dist/types", "outDir": "dist", "importHelpers": true, - "esModuleInterop": true, "typeRoots": [ "node_modules/@types" ] }, - "ts-node": { - "compilerOptions": { - "module": "commonjs" - }, - "include": [ - "scripts" - ] - }, "include": [ "src" ], From 36973354df603951e71832c393d73a35a36271d4 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sun, 28 Jan 2024 17:57:17 +0700 Subject: [PATCH 25/49] refactor(nitro-node): convert throwable functions to async Make throwable functions to async to be friendly with IPC calls as catching error synchronously is not an option for some use cases. --- nitro-node/package.json | 1 + nitro-node/rollup.config.ts | 8 +- nitro-node/src/nitro.ts | 210 ++++++++++++++++++------------------ 3 files changed, 109 insertions(+), 110 deletions(-) diff --git a/nitro-node/package.json b/nitro-node/package.json index c697639c0..34e22f89e 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -55,6 +55,7 @@ "rollup-plugin-sourcemaps": "^0.6.3", "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^29.1.2", + "ts-node": "^10.9.2", "tsx": "^4.7.0", "typescript": "^5.3.3" }, diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts index 0dfd92a66..99f3828d4 100644 --- a/nitro-node/rollup.config.ts +++ b/nitro-node/rollup.config.ts @@ -70,10 +70,10 @@ export default [ // Allow node_modules resolution, so you can use 'external' to control // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage - resolve({ - extensions: [".ts", ".js", ".json"], - preferBuiltins: false, - }), + //resolve({ + // extensions: [".ts", ".js", ".json"], + // preferBuiltins: false, + //}), // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) // This should be after resolve() plugin commonjs(), diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index 46a80c716..c0013ed7c 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -83,7 +83,7 @@ export function getBinPath(): string { /** * Set custom bin path */ -export function setBinPath(customBinPath: string): void | never { +export async function setBinPath(customBinPath: string): Promise { // Check if the path is a directory if ( fs.existsSync(customBinPath) && @@ -111,7 +111,9 @@ export function getNvidiaConfig(): NitroNvidiaConfig { * Set custom Nvidia config for running inference over GPU * @param {NitroNvidiaConfig} config The new config to apply */ -export function setNvidiaConfig(config: NitroNvidiaConfig) { +export async function setNvidiaConfig( + config: NitroNvidiaConfig, +): Promise { nvidiaConfig = config; } @@ -119,7 +121,7 @@ export function setNvidiaConfig(config: NitroNvidiaConfig) { * Set logger before running nitro * @param {NitroLogger} logger The logger to use */ -export function setLogger(logger: NitroLogger) { +export async function setLogger(logger: NitroLogger): Promise { log = logger; } @@ -141,7 +143,7 @@ export function stopModel(): Promise { export async function runModel({ modelFullPath, promptTemplate, -}: NitroModelInitOptions): Promise { +}: NitroModelInitOptions): Promise { // Download nitro binaries if it's not already downloaded await downloadNitro(binPath); const files: string[] = fs.readdirSync(modelFullPath); @@ -153,7 +155,7 @@ export async function runModel({ SUPPORTED_MODEL_FORMATS.some((ext) => file.toLowerCase().endsWith(ext)), ); - if (!ggufBinFile) return Promise.reject("No GGUF model file found"); + if (!ggufBinFile) throw new Error("No GGUF model file found"); currentModelFile = path.join(modelFullPath, ggufBinFile); @@ -161,11 +163,7 @@ export async function runModel({ // Convert promptTemplate to system_prompt, user_prompt, ai_prompt const prompt: NitroPromptSetting = {}; if (promptTemplate) { - try { - Object.assign(prompt, promptTemplateConverter(promptTemplate)); - } catch (e: any) { - return Promise.reject(e); - } + Object.assign(prompt, promptTemplateConverter(promptTemplate)); } currentSettings = { @@ -186,32 +184,35 @@ export async function runModel({ * 3. Validate model status * @returns */ -export async function runNitroAndLoadModel(): Promise< - NitroModelOperationResponse | { error: any } -> { - // Gather system information for CPU physical cores and memory - return killSubprocess() - .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => { - /** - * There is a problem with Windows process manager - * Should wait for awhile to make sure the port is free and subprocess is killed - * The tested threshold is 500ms - **/ - if (process.platform === "win32") { - return new Promise((resolve) => setTimeout(() => resolve({}), 500)); - } else { - return Promise.resolve({}); - } - }) - .then(spawnNitroProcess) - .then(() => loadLLMModel(currentSettings)) - .then(validateModelStatus) - .catch((err) => { - // TODO: Broadcast error so app could display proper error message - log(`[NITRO]::Error: ${err}`); - return { error: err }; - }); +export async function runNitroAndLoadModel(): Promise { + try { + // Gather system information for CPU physical cores and memory + await killSubprocess(); + await tcpPortUsed.waitUntilFree(PORT, 300, 5000); + /** + * There is a problem with Windows process manager + * Should wait for awhile to make sure the port is free and subprocess is killed + * The tested threshold is 500ms + **/ + if (process.platform === "win32") { + return await new Promise((resolve) => setTimeout(() => resolve({}), 500)); + } + const spawnResult = await spawnNitroProcess(); + if (spawnResult.error) { + return spawnResult; + } + // TODO: Use this response? + const _loadModelResponse = await loadLLMModel(currentSettings); + const validationResult = await validateModelStatus(); + if (validationResult.error) { + return validationResult; + } + return {}; + } catch (err: any) { + // TODO: Broadcast error so app could display proper error message + log(`[NITRO]::Error: ${err}`); + return { error: err }; + } } /** @@ -220,7 +221,7 @@ export async function runNitroAndLoadModel(): Promise< * @returns {(NitroPromptSetting | never)} parsed prompt setting * @throws {Error} if cannot split promptTemplate */ -export function promptTemplateConverter( +function promptTemplateConverter( promptTemplate: string, ): NitroPromptSetting | never { // Split the string using the markers @@ -283,10 +284,10 @@ export async function loadLLMModel(settings: any): Promise { log( `[NITRO]::Debug: Load model success with response ${JSON.stringify(res)}`, ); - return await Promise.resolve(res); + return res; } catch (err) { log(`[NITRO]::Error: Load model failed with error ${err}`); - return await Promise.reject(); + throw err; } } @@ -310,35 +311,34 @@ export async function chatCompletion( log( `[NITRO]::Debug: Running chat completion with request ${JSON.stringify(request)}`, ); - return fetchRetry(NITRO_HTTP_CHAT_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - Accept: Boolean(outStream) ? "text/event-stream" : "application/json", - }, - body: JSON.stringify(request), - retries: 3, - retryDelay: 500, - }) - .then(async (response) => { - if (outStream) { - if (!response.body) { - throw new Error("Error running chat completion"); - } - const outPipe = response.body - .pipeThrough(new TextDecoderStream()) - .pipeTo(outStream); - // Wait for all the streams to complete before returning from async function - await outPipe; - } - log(`[NITRO]::Debug: Chat completion success`); - return response; - }) - .catch((err) => { - log(`[NITRO]::Error: Chat completion failed with error ${err}`); - throw err; + try { + const response = await fetchRetry(NITRO_HTTP_CHAT_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + Accept: Boolean(outStream) ? "text/event-stream" : "application/json", + }, + body: JSON.stringify(request), + retries: 3, + retryDelay: 500, }); + if (outStream) { + if (!response.body) { + throw new Error("Error running chat completion"); + } + const outPipe = response.body + .pipeThrough(new TextDecoderStream()) + .pipeTo(outStream); + // Wait for all the streams to complete before returning from async function + await outPipe; + } + log(`[NITRO]::Debug: Chat completion success`); + return response; + } catch (err) { + log(`[NITRO]::Error: Chat completion failed with error ${err}`); + throw err; + } } /** @@ -350,30 +350,29 @@ export async function chatCompletion( export async function validateModelStatus(): Promise { // Send a GET request to the validation URL. // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. - return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { + const response = await fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { method: "GET", headers: { "Content-Type": "application/json", }, retries: 5, retryDelay: 500, - }).then(async (res: Response) => { - log( - `[NITRO]::Debug: Validate model state success with response ${JSON.stringify( - res, - )}`, - ); - // If the response is OK, check model_loaded status. - if (res.ok) { - const body = await res.json(); - // If the model is loaded, return an empty object. - // Otherwise, return an object with an error message. - if (body.model_loaded) { - return Promise.resolve({}); - } - } - return Promise.resolve({ error: "Validate model status failed" }); }); + log( + `[NITRO]::Debug: Validate model state success with response ${JSON.stringify( + response, + )}`, + ); + // If the response is OK, check model_loaded status. + if (response.ok) { + const body = await response.json(); + // If the model is loaded, return an empty object. + // Otherwise, return an object with an error message. + if (body.model_loaded) { + return {}; + } + } + return { error: "Validate model status failed" }; } /** @@ -385,18 +384,19 @@ export async function killSubprocess(): Promise { setTimeout(() => controller.abort(), 5000); log(`[NITRO]::Debug: Request to kill Nitro`); - return fetch(NITRO_HTTP_KILL_URL, { - method: "DELETE", - signal: controller.signal, - }) - .then(() => { - subprocess?.kill(); - subprocess = undefined; - }) - .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) - .then(() => Promise.resolve({})) - .catch((err) => ({ error: err })); + try { + const _response = await fetch(NITRO_HTTP_KILL_URL, { + method: "DELETE", + signal: controller.signal, + }); + subprocess?.kill(); + subprocess = undefined; + await tcpPortUsed.waitUntilFree(PORT, 300, 5000); + log(`[NITRO]::Debug: Nitro process is terminated`); + return {}; + } catch (err) { + return { error: err }; + } } /** @@ -451,16 +451,14 @@ export function spawnNitroProcess(): Promise { /** * Get the system resources information */ -export function getResourcesInfo(): Promise { - return new Promise(async (resolve) => { - const cpu = osUtils.cpuCount(); - log(`[NITRO]::CPU informations - ${cpu}`); - const response: ResourcesInfo = { - numCpuPhysicalCore: cpu, - memAvailable: 0, - }; - resolve(response); - }); +export async function getResourcesInfo(): Promise { + const cpu = osUtils.cpuCount(); + log(`[NITRO]::CPU informations - ${cpu}`); + const response: ResourcesInfo = { + numCpuPhysicalCore: cpu, + memAvailable: 0, + }; + return response; } export const updateNvidiaInfo = async () => From 9761294e6fd7ad258973f673c74cd962c25373b2 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Mon, 29 Jan 2024 18:07:27 +0700 Subject: [PATCH 26/49] fix(nitro-node/typings): fix export types for submodule scripts --- nitro-node/package.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nitro-node/package.json b/nitro-node/package.json index 34e22f89e..c21c87341 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -30,12 +30,10 @@ "typesVersions": { "*": { ".": [ - "./dist/index.esm.js", "./dist/index.esm.js.map", "./dist/types/index.d.ts" ], - "./scripts": [ - "./dist/scripts/index.esm.js", + "scripts": [ "./dist/scripts/index.esm.js.map", "./dist/types/scripts/index.d.ts" ] From f026da5988816e4fb7bdb5447cb055afd923b9f9 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Mon, 29 Jan 2024 21:29:56 +0700 Subject: [PATCH 27/49] feat(nitro-node/magic-number): Find matching model file by magic number - Not relying on the extension of the file name anymore - Caller can now just pass the path to directory containing models --- nitro-node/src/nitro.ts | 40 +++++++++++++++++++++++++-- nitro-node/test/nitro-process.test.ts | 37 ++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index c0013ed7c..499a1990d 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -56,9 +56,11 @@ const NVIDIA_DEFAULT_CONFIG: NitroNvidiaConfig = { }; // The supported model format -// TODO: Should be an array to support more models const SUPPORTED_MODEL_FORMATS = [".gguf"]; +// The supported model magic number +const SUPPORTED_MODEL_MAGIC_NUMBERS = ["GGUF"]; + // The subprocess instance for Nitro let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; // The current model file url @@ -134,6 +136,27 @@ export function stopModel(): Promise { return killSubprocess(); } +/** + * Read the magic bytes from a file and check if they match the provided magic bytes + */ +export async function checkMagicBytes( + filePath: string, + magicBytes: string, +): Promise { + const desired = Buffer.from(magicBytes); + const nBytes = desired.byteLength; + const chunks = []; + for await (let chunk of fs.createReadStream(filePath, { + start: 0, + end: nBytes - 1, + })) { + chunks.push(chunk); + } + const actual = Buffer.concat(chunks); + log(`Comparing file's magic bytes <${actual.toString()}> and desired <${desired.toString()}>`); + return Buffer.compare(actual, desired) === 0; +} + /** * Initializes a Nitro subprocess to load a machine learning model. * @param modelFullPath - The absolute full path to model directory. @@ -149,12 +172,25 @@ export async function runModel({ const files: string[] = fs.readdirSync(modelFullPath); // Look for model file with supported format - const ggufBinFile = files.find( + let ggufBinFile = files.find( (file) => file === path.basename(modelFullPath) || SUPPORTED_MODEL_FORMATS.some((ext) => file.toLowerCase().endsWith(ext)), ); + // If not found from path and extension, try from magic number + if (!ggufBinFile) { + for (const f of files) { + for (const magicNum of SUPPORTED_MODEL_MAGIC_NUMBERS) { + if (await checkMagicBytes(path.join(modelFullPath, f), magicNum)) { + ggufBinFile = f; + break; + } + } + if (ggufBinFile) break; + } + } + if (!ggufBinFile) throw new Error("No GGUF model file found"); currentModelFile = path.join(modelFullPath, ggufBinFile); diff --git a/nitro-node/test/nitro-process.test.ts b/nitro-node/test/nitro-process.test.ts index 97f6cb490..9195eaec2 100644 --- a/nitro-node/test/nitro-process.test.ts +++ b/nitro-node/test/nitro-process.test.ts @@ -13,6 +13,7 @@ import { loadLLMModel, validateModelStatus, chatCompletion, + checkMagicBytes, } from "../src"; // FIXME: Shorthand only possible for es6 targets and up @@ -104,7 +105,7 @@ describe("Manage nitro process", () => { const modelFullPath = fs.mkdtempSync( path.join(os.tmpdir(), "nitro-node-test"), ); - let modelCfg: any = {}; + let modelCfg: Record = {}; // Setup steps before running the suite const setupHooks = [ @@ -225,5 +226,39 @@ describe("Manage nitro process", () => { // Set timeout to 1 minutes 1 * 60 * 1000, ); + describe("search model file by magic number", () => { + // Rename model file before test + beforeEach(async () => { + const fileName = modelCfg.source_url.split("/")?.pop() ?? "model.gguf"; + // Rename the extension of model file + fs.renameSync( + path.join(modelFullPath, fileName), + path.join(modelFullPath, `${fileName.replace(/\.gguf$/gi, ".bak")}`), + ); + }); + afterEach(async () => { + const fileName = modelCfg.source_url.split("/")?.pop() ?? "model.gguf"; + // Restore the extension of model file + fs.renameSync( + path.join(modelFullPath, `${fileName.replace(/\.gguf$/gi, ".bak")}`), + path.join(modelFullPath, fileName), + ); + }); + test( + "should be able to detect model file by magic number", + async () => { + const files = fs.readdirSync(modelFullPath) as string[]; + // Test checking magic bytes + const res = await Promise.all( + files.map((f) => + checkMagicBytes(path.join(modelFullPath, f), "GGUF"), + ), + ); + expect(res).toContain(true); + }, + // Set timeout to 2 seconds + 2 * 1000, + ); + }); /// END TESTS }); From 113375989a7126e8cc7c10e6d7ae3141970d7a58 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 30 Jan 2024 06:48:45 +0700 Subject: [PATCH 28/49] chore(nitro-node/ci): remove feature branch from newly added workflows --- .github/workflows/build-nitro-node.yml | 3 +-- .github/workflows/test-install-nitro-node.yml | 2 +- nitro-node/rollup.config.ts | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-nitro-node.yml b/.github/workflows/build-nitro-node.yml index 015b7a138..ab5ebd7f0 100644 --- a/.github/workflows/build-nitro-node.yml +++ b/.github/workflows/build-nitro-node.yml @@ -6,8 +6,7 @@ on: push: branches: - main - - feat/1635/decouple-nitro-inference-engine-into-a-library - #tags: ["v[0-9]+.[0-9]+.[0-9]+"] + tags: ["v[0-9]+.[0-9]+.[0-9]+"] paths: [".github/workflows/build-nitro-node.yml", "nitro-node"] pull_request: types: [opened, synchronize, reopened] diff --git a/.github/workflows/test-install-nitro-node.yml b/.github/workflows/test-install-nitro-node.yml index 5c0a010ea..260d040e4 100644 --- a/.github/workflows/test-install-nitro-node.yml +++ b/.github/workflows/test-install-nitro-node.yml @@ -7,7 +7,7 @@ on: branches: - main - feat/1635/decouple-nitro-inference-engine-into-a-library - #tags: ["v[0-9]+.[0-9]+.[0-9]+"] + tags: ["v[0-9]+.[0-9]+.[0-9]+"] paths: - ".github/scripts/e2e-test-install-nitro-node.js" - ".github/workflows/test-install-nitro-node.yml" diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts index 99f3828d4..cf86f3c98 100644 --- a/nitro-node/rollup.config.ts +++ b/nitro-node/rollup.config.ts @@ -67,6 +67,7 @@ export default [ plugins: [ // Allow json resolution json(), + // Allow node_modules resolution, so you can use 'external' to control // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage @@ -74,6 +75,7 @@ export default [ // extensions: [".ts", ".js", ".json"], // preferBuiltins: false, //}), + // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) // This should be after resolve() plugin commonjs(), From 506d2778f0d01b2754d81661311f0c9844f839a8 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 30 Jan 2024 15:24:05 +0700 Subject: [PATCH 29/49] feat(nitr-node/postinstall): download latest release for nitro by default Can also override the version by setting NITRO_VERSION environment variable --- nitro-node/src/scripts/download-nitro.ts | 167 ++++++++++++++++------- 1 file changed, 121 insertions(+), 46 deletions(-) diff --git a/nitro-node/src/scripts/download-nitro.ts b/nitro-node/src/scripts/download-nitro.ts index 70aa7b333..75588cebd 100644 --- a/nitro-node/src/scripts/download-nitro.ts +++ b/nitro-node/src/scripts/download-nitro.ts @@ -3,47 +3,86 @@ import path from "node:path"; import download from "download"; import { Duplex } from "node:stream"; -// Define nitro version to download in env variable -const NITRO_VERSION = process.env.NITRO_VERSION || "0.2.11"; +// Define nitro version to download in env variable ("latest" or tag "v1.2.3") +const NITRO_VERSION = process.env.NITRO_VERSION || "latest"; // The platform OS to download nitro for const PLATFORM = process.env.npm_config_platform || process.platform; // The platform architecture //const ARCH = process.env.npm_config_arch || process.arch; -const linuxVariants = { - "linux-amd64": "linux-cpu", - "linux-amd64-cuda-12-0": "linux-cuda-12-0", - "linux-amd64-cuda-11-7": "linux-cuda-11-7", +const releaseUrlPrefixVariants = { + latest: "https://api.github.com/repos/janhq/nitro/releases/", + tag: "https://api.github.com/repos/janhq/nitro/releases/tags/", }; -const darwinVariants = { - "mac-arm64": "mac-arm64", - "mac-amd64": "mac-x64", -}; - -const win32Variants = { - "win-amd64-cuda-12-0": "win-cuda-12-0", - "win-amd64-cuda-11-7": "win-cuda-11-7", - "win-amd64": "win-cpu", +const getReleaseInfo = async (taggedVersion: string): Promise => { + const releaseUrlPrefix = + taggedVersion === "latest" + ? releaseUrlPrefixVariants.latest + : taggedVersion.match(/^\d+\.\d+\.\d+$/gi) + ? releaseUrlPrefixVariants.tag + : undefined; + if (!releaseUrlPrefix) throw Error(`Invalid version: ${taggedVersion}`); + const url = `${releaseUrlPrefix}${taggedVersion}`; + console.log(`[Getting release info] ${url}`); + const response = await fetch(url, { + headers: { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + }); + if (!response.ok) + throw Error(`Failed to fetch release info: ${response.status}`); + return await response.json(); }; -// Mapping to installation variants -const variantMapping: Record> = { - darwin: darwinVariants, - linux: linuxVariants, - win32: win32Variants, +const extractDownloadInfo = async ( + /* releaseInfo */ + { + html_url, + tag_name, + name, + prerelease, + assets, + }: { + html_url: string; + tag_name: string; + name: string; + prerelease: boolean; + assets: { + name: string; + size: number; + browser_download_url: string; + }[]; + }, + /* variants for filter for in format {suffix: dir} */ + suffixVariants: Record, +): Promise> => { + console.log(`[Release URL][prerelease=${prerelease}] ${html_url}`); + const assetNames = assets.map(({ name }: { name: string }) => name); + const assetsToDownloads = Object.entries(suffixVariants).reduce( + ( + dict: Record, + [suffix, dir]: [string, string], + ) => { + // Skip if suffix is not in asset names + if (!assetNames.includes(`nitro-${name}-${suffix}.tar.gz`)) return dict; + // Else add the download url + dict[suffix] = { + url: `https://github.com/janhq/nitro/releases/download/${tag_name}/nitro-${name}-${suffix}.tar.gz`, + dir, + }; + return dict; + }, + {}, + ); + // If empty then no assets were found + if (!Object.keys(assetsToDownloads).length) + throw Error("Failed to find any asset to download"); + // Return the dict of download info + return assetsToDownloads; }; -if (!(PLATFORM in variantMapping)) { - throw Error(`Invalid platform: ${PLATFORM}`); -} -// Get download config for this platform -const variantConfig: Record = variantMapping[PLATFORM]; - -// Generate download link for each tarball -const getTarUrl = (version: string, suffix: string) => - `https://github.com/janhq/nitro/releases/download/v${version}/nitro-${version}-${suffix}.tar.gz`; - // Report download progress const createProgressReporter = (variant: string) => (stream: Promise & Duplex) => @@ -65,28 +104,26 @@ const createProgressReporter = // Download single binary const downloadBinary = async ( - version: string, suffix: string, - destDirPath: string, + { url, dir }: { url: string; dir: string }, ) => { - const tarUrl = getTarUrl(version, suffix); - console.log(`Downloading ${tarUrl} to ${destDirPath}`); + console.log(`Downloading ${url} to ${dir}...`); const progressReporter = createProgressReporter(suffix); await progressReporter( - download(tarUrl, destDirPath, { + download(url, dir, { strip: 1, extract: true, }), ); // Set mode of downloaded binaries to executable - (fs.readdirSync(destDirPath, { recursive: true }) as string[]) + (fs.readdirSync(dir, { recursive: true }) as string[]) .filter( (fname) => - fname.includes("nitro") && - fs.lstatSync(path.join(destDirPath, fname)).isFile(), + fname.includes("nitro") && fs.lstatSync(path.join(dir, fname)).isFile(), ) .forEach((nitroBinary) => { - const absPath = path.join(destDirPath, nitroBinary); + const absPath = path.join(dir, nitroBinary); + // Set mode executable for nitro binary fs.chmodSync( absPath, fs.constants.S_IRWXU | fs.constants.S_IRWXG | fs.constants.S_IRWXO, @@ -96,13 +133,19 @@ const downloadBinary = async ( // Download the binaries const downloadBinaries = ( - version: string, - config: Record, + /* downloadInfo */ + downloadInfo: Record, + /* The absolute path to the directory where binaries will be downloaded */ absBinDirPath: string, ) => { - return Object.entries(config).reduce( - (p: Promise, [k, v]) => - p.then(() => downloadBinary(version, k, path.join(absBinDirPath, v))), + return Object.entries(downloadInfo).reduce( + ( + p: Promise, + [suffix, { url, dir }]: [string, { url: string; dir: string }], + ) => + p.then(() => + downloadBinary(suffix, { url, dir: path.join(absBinDirPath, dir) }), + ), Promise.resolve(), ); }; @@ -126,6 +169,36 @@ const verifyDownloadedBinaries = (absBinDirPath: string) => { }); }; +const linuxVariants = { + "linux-amd64": "linux-cpu", + "linux-amd64-cuda-12-0": "linux-cuda-12-0", + "linux-amd64-cuda-11-7": "linux-cuda-11-7", +}; + +const darwinVariants = { + "mac-arm64": "mac-arm64", + "mac-amd64": "mac-x64", +}; + +const win32Variants = { + "win-amd64-cuda-12-0": "win-cuda-12-0", + "win-amd64-cuda-11-7": "win-cuda-11-7", + "win-amd64": "win-cpu", +}; + +// Mapping to installation variants +const variantMapping: Record> = { + darwin: darwinVariants, + linux: linuxVariants, + win32: win32Variants, +}; + +if (!(PLATFORM in variantMapping)) { + throw Error(`Invalid platform: ${PLATFORM}`); +} + +// Get download config for this platform +const variantConfig: Record = variantMapping[PLATFORM]; // Call the download function with version and config export const downloadNitro = async (absBinDirPath: string) => { // Return early without downloading if nitro binaries are already downloaded @@ -133,12 +206,14 @@ export const downloadNitro = async (absBinDirPath: string) => { //console.log("Nitro binaries are already downloaded!"); return; } - return await downloadBinaries(NITRO_VERSION, variantConfig, absBinDirPath); + const releaseInfo = await getReleaseInfo(NITRO_VERSION); + const downloadInfo = await extractDownloadInfo(releaseInfo, variantConfig); + return await downloadBinaries(downloadInfo, absBinDirPath); }; // Run script if called directly instead of import as module if (require.main === module) { // Assume calling the source typescript files // bin path will be relative to the source code root - downloadNitro(path.join(__dirname, '..', "..", "bin")); + downloadNitro(path.join(__dirname, "..", "..", "bin")); } From b4aa567dc6fbaf7c2d702be5b7bc8d3974e941b3 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 30 Jan 2024 16:28:13 +0700 Subject: [PATCH 30/49] fix(nitro-node/downloader): accept version tag with/without 'v' prefix --- llama.cpp | 2 +- nitro-node/src/scripts/download-nitro.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 5f1925a8c..862f5e41a 160000 --- a/llama.cpp +++ b/llama.cpp @@ -1 +1 @@ -Subproject commit 5f1925a8cef81eb9b372faaae34b0dd76d5361d4 +Subproject commit 862f5e41ab1fdf12d6f59455aad3f5dd8258f805 diff --git a/nitro-node/src/scripts/download-nitro.ts b/nitro-node/src/scripts/download-nitro.ts index 75588cebd..547bba3d2 100644 --- a/nitro-node/src/scripts/download-nitro.ts +++ b/nitro-node/src/scripts/download-nitro.ts @@ -19,7 +19,7 @@ const getReleaseInfo = async (taggedVersion: string): Promise => { const releaseUrlPrefix = taggedVersion === "latest" ? releaseUrlPrefixVariants.latest - : taggedVersion.match(/^\d+\.\d+\.\d+$/gi) + : taggedVersion.match(/^v?\d+\.\d+\.\d+$/gi) ? releaseUrlPrefixVariants.tag : undefined; if (!releaseUrlPrefix) throw Error(`Invalid version: ${taggedVersion}`); From ae1d4dd142c65fd936c4ee1fc834c533dee0a5ac Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 30 Jan 2024 16:35:26 +0700 Subject: [PATCH 31/49] fix(nitro-node): remove excessive console log in nitro.ts --- nitro-node/src/nitro.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index 499a1990d..0d06f665d 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -21,7 +21,6 @@ import { } from "./types"; import { downloadNitro } from "./scripts"; // Polyfill fetch with retry -console.log(typeof fetchRT); const fetchRetry = fetchRT(fetch); // The PORT to use for the Nitro subprocess From 80dcc55c65bec6777632c24afbc5bf8375749c1d Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 30 Jan 2024 16:45:36 +0700 Subject: [PATCH 32/49] feat(nitro-node): trap terminate signal in order to kill nitro process As nitro process is spawned in daemon mode, the library should be able to trap SIGTERM signal in order to kill nitro process. --- nitro-node/src/nitro.ts | 12 +++++++++++- nitro-node/src/scripts/download-nitro.ts | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index 0d06f665d..43dabf162 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -152,7 +152,9 @@ export async function checkMagicBytes( chunks.push(chunk); } const actual = Buffer.concat(chunks); - log(`Comparing file's magic bytes <${actual.toString()}> and desired <${desired.toString()}>`); + log( + `Comparing file's magic bytes <${actual.toString()}> and desired <${desired.toString()}>`, + ); return Buffer.compare(actual, desired) === 0; } @@ -499,3 +501,11 @@ export async function getResourcesInfo(): Promise { export const updateNvidiaInfo = async () => await _updateNvidiaInfo(nvidiaConfig); export const getCurrentNitroProcessInfo = () => getNitroProcessInfo(subprocess); + +/** + * Trap for system signal so we can stop nitro process on exit + */ +process.on("SIGTERM", async () => { + log(`[NITRO]::Debug: Received SIGTERM signal`); + await killSubprocess(); +}); diff --git a/nitro-node/src/scripts/download-nitro.ts b/nitro-node/src/scripts/download-nitro.ts index 547bba3d2..b83f575c6 100644 --- a/nitro-node/src/scripts/download-nitro.ts +++ b/nitro-node/src/scripts/download-nitro.ts @@ -150,7 +150,7 @@ const downloadBinaries = ( ); }; -// Check for a files with nitro in name in the corresponding directory +// Check for files with nitro in name in the corresponding directory const verifyDownloadedBinaries = (absBinDirPath: string) => { // Check all the paths in variantConfig for a file with nitro in its name return Object.values(variantConfig).every((binDirVariant: string) => { From 051644003043dc75a661b2806f632659ab54a4f8 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Wed, 31 Jan 2024 17:59:12 +0700 Subject: [PATCH 33/49] chore(nitro-node/ci): enable CI build on MacOS M1 --- .github/workflows/build-nitro-node.yml | 76 +++++++++++++------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/workflows/build-nitro-node.yml b/.github/workflows/build-nitro-node.yml index ab5ebd7f0..cce5bc310 100644 --- a/.github/workflows/build-nitro-node.yml +++ b/.github/workflows/build-nitro-node.yml @@ -150,49 +150,49 @@ jobs: # cd nitro-node # make clean test-ci - #macOS-M-build: - # runs-on: mac-silicon - # steps: - # - name: Clone - # id: checkout - # uses: actions/checkout@v4 - # with: - # submodules: recursive + macOS-M-build: + runs-on: macos-14 + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive - # - uses: actions/setup-node@v4 - # with: - # node-version: 18 + - uses: actions/setup-node@v4 + with: + node-version: 18 - # - name: Restore cached model file - # id: cache-model-restore - # uses: actions/cache/restore@v4 - # with: - # path: | - # nitro-node/test/test_assets/*.gguf - # key: ${{ runner.os }}-model-gguf + - name: Restore cached model file + id: cache-model-restore + uses: actions/cache/restore@v4 + with: + path: | + nitro-node/test/test_assets/*.gguf + key: ${{ runner.os }}-model-gguf - # - uses: suisei-cn/actions-download-file@v1.4.0 - # id: download-model-file - # name: Download model file - # with: - # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: nitro-node/test/test_assets/ - # auto-match: true - # retry-times: 3 + - uses: suisei-cn/actions-download-file@v1.4.0 + id: download-model-file + name: Download model file + with: + url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + target: nitro-node/test/test_assets/ + auto-match: true + retry-times: 3 - # - name: Save downloaded model file to cache - # id: cache-model-save - # uses: actions/cache/save@v4 - # with: - # path: | - # nitro-node/test/test_assets/*.gguf - # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + - name: Save downloaded model file to cache + id: cache-model-save + uses: actions/cache/save@v4 + with: + path: | + nitro-node/test/test_assets/*.gguf + key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - # - name: Run tests - # id: test_nitro_node - # run: | - # cd nitro-node - # make clean test-ci + - name: Run tests + id: test_nitro_node + run: | + cd nitro-node + make clean test-ci macOS-Intel-build: runs-on: macos-latest From e427816ed3dcfbce7cd94532d663c8610d42614f Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Thu, 1 Feb 2024 03:23:52 +0700 Subject: [PATCH 34/49] fix(nitro-node/build): electron got included as dependency When bundling with rollup, `electron` got included as dependency due to a transitive dependency, `download@8.0.0` requires `got@^8.3.1`, which in turn requires `electron` conditionally. --- nitro-node/rollup.config.ts | 28 ++++++++++++++-------------- nitro-node/tsconfig.json | 3 +-- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts index cf86f3c98..a55f126ef 100644 --- a/nitro-node/rollup.config.ts +++ b/nitro-node/rollup.config.ts @@ -13,11 +13,10 @@ export default [ ], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [ - "electron", - "node:fs", - "node:child_process", - "node:os", - "node:path", + // `download@8.0.0` requires `got@^8.3.1` which then optionally requires `electron`, result in wrong dependency + // Ref: https://github.com/kubernetes-client/javascript/issues/350#issue-500860208 + // Ref: https://github.com/kubernetes-client/javascript/issues/350#issuecomment-553644659 + "got", ], watch: { include: "src/**", @@ -30,7 +29,6 @@ export default [ // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ extensions: [".ts", ".js", ".json"], - preferBuiltins: false, }), // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) // This should be after resolve() plugin @@ -38,7 +36,6 @@ export default [ // Compile TypeScript files typescript({ useTsconfigDeclarationDir: true, - tsconfig: "tsconfig.json", }), // Resolve source maps to the original source @@ -60,7 +57,12 @@ export default [ }, ], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: ["electron", "node:fs", "node:path"], + external: [ + // `download@8.0.0` requires `got@^8.3.1` which then optionally requires `electron`, result in wrong dependency + // Ref: https://github.com/kubernetes-client/javascript/issues/350#issue-500860208 + // Ref: https://github.com/kubernetes-client/javascript/issues/350#issuecomment-553644659 + "got", + ], watch: { include: "src/scripts/**", }, @@ -71,18 +73,16 @@ export default [ // Allow node_modules resolution, so you can use 'external' to control // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage - //resolve({ - // extensions: [".ts", ".js", ".json"], - // preferBuiltins: false, - //}), + resolve({ + extensions: [".ts", ".js", ".json"], + }), // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) // This should be after resolve() plugin commonjs(), // Compile TypeScript files typescript({ - useTsconfigDeclarationDir: true, - tsconfig: "tsconfig.json", + useTsconfigDeclarationDir: true }), // Resolve source maps to the original source diff --git a/nitro-node/tsconfig.json b/nitro-node/tsconfig.json index 943393d7a..799e9eec6 100644 --- a/nitro-node/tsconfig.json +++ b/nitro-node/tsconfig.json @@ -6,8 +6,7 @@ "lib": [ "es2015", "es2016", - "es2017", - "dom" + "es2017" ], "strict": true, "sourceMap": true, From 7b2c84644cc286a2b257135c1defa203fdbcebd2 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Thu, 1 Feb 2024 04:02:32 +0700 Subject: [PATCH 35/49] test(nitro-node): update chat completion test with more meaningful use case --- nitro-node/test/nitro-process.test.ts | 38 ++++++++++++++++----------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/nitro-node/test/nitro-process.test.ts b/nitro-node/test/nitro-process.test.ts index 9195eaec2..72fb7be6a 100644 --- a/nitro-node/test/nitro-process.test.ts +++ b/nitro-node/test/nitro-process.test.ts @@ -7,6 +7,7 @@ import path from "node:path"; import download from "download"; import { Duplex } from "node:stream"; +import { WritableStream } from "node:stream/web"; import { stopModel, runModel, @@ -175,43 +176,50 @@ describe("Manage nitro process", () => { // Validate model status await validateModelStatus(); // Arrays of all the chunked response - let streamedContent: string[] = []; + let streamedContent: Record[] = []; // Run chat completion with stream const response = await chatCompletion( { messages: [ - { content: "Hello there", role: "assistant" }, - { content: "Write a long and sad story for me", role: "user" }, + { + content: + "You are a good productivity assistant. You help user with what they are asking in Markdown format . For responses that contain code, you must use ``` with the appropriate coding language to help display the code to user correctly.", + role: "assistant", + }, + { + content: "Please give me a hello world code in cpp", + role: "user", + }, ], model: "gpt-3.5-turbo", - max_tokens: 50, - stop: ["hello"], + max_tokens: 2048, + stop: [], frequency_penalty: 0, presence_penalty: 0, - temperature: 0.1, + temperature: 0.7, + top_p: 0.95, + context_length: 4096, }, new WritableStream({ write(chunk: string) { - if (chunk.trim() == "data: [DONE]") { + const data = chunk.replace(/^\s*data:\s*/, "").trim(); + // Stop at [DONE] message + if (data.match(/\[DONE\]/)) { return; } - return new Promise((resolve) => { - streamedContent.push(chunk.slice("data:".length).trim()); - resolve(); - }); + streamedContent.push(JSON.parse(data)); }, //close() {}, //abort(_err) {} }), ); - // Remove the [DONE] message - streamedContent.pop(); - // Parse json - streamedContent = streamedContent.map((str) => JSON.parse(str)); // Show the streamed content console.log( `[Streamed response] ${JSON.stringify(streamedContent, null, 2)}`, ); + console.log( + `Generated reply: ${streamedContent.map((r) => r.choices[0].delta.content ?? "").join("")}`, + ); // The response body is unusable if consumed by out stream await expect(response.text).rejects.toThrow(); From ff2cf084b2de094af97ef089be2b4883e3e358ab Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Thu, 1 Feb 2024 14:22:55 +0700 Subject: [PATCH 36/49] fix(nitro-node/ci): remove feature branch from trigger --- .github/workflows/test-install-nitro-node.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-install-nitro-node.yml b/.github/workflows/test-install-nitro-node.yml index 260d040e4..de9abb29f 100644 --- a/.github/workflows/test-install-nitro-node.yml +++ b/.github/workflows/test-install-nitro-node.yml @@ -6,7 +6,6 @@ on: push: branches: - main - - feat/1635/decouple-nitro-inference-engine-into-a-library tags: ["v[0-9]+.[0-9]+.[0-9]+"] paths: - ".github/scripts/e2e-test-install-nitro-node.js" From b2b749dcccc9826edaeb7f92363a93ccc5769f9f Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sat, 3 Feb 2024 20:56:00 +0700 Subject: [PATCH 37/49] refactor(nitro-node): nitro release url can be altered upon build --- nitro-node/package.json | 5 +++-- nitro-node/rollup.config.ts | 11 ++++++++++- nitro-node/src/scripts/download-nitro.ts | 17 +++++++---------- nitro-node/src/scripts/types/global.d.ts | 4 ++++ 4 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 nitro-node/src/scripts/types/global.d.ts diff --git a/nitro-node/package.json b/nitro-node/package.json index c21c87341..33a35bf4a 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -10,7 +10,8 @@ "scripts": { "test": "jest --verbose --detectOpenHandles", "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "downloadnitro": "tsx src/scripts/download-nitro.ts", + "predownloadnitro": "npm run build", + "downloadnitro": "node dist/scripts/index.cjs", "build:publish": "npm pack", "postinstall": "node -r @janhq/nitro-node/postinstall" }, @@ -43,6 +44,7 @@ "@rollup/plugin-commonjs": "^25.0.7", "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-replace": "^5.0.5", "@types/download": "^8.0.5", "@types/jest": "^29.5.11", "@types/node": "^20.11.4", @@ -54,7 +56,6 @@ "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^29.1.2", "ts-node": "^10.9.2", - "tsx": "^4.7.0", "typescript": "^5.3.3" }, "dependencies": { diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts index a55f126ef..ddf90b39b 100644 --- a/nitro-node/rollup.config.ts +++ b/nitro-node/rollup.config.ts @@ -3,6 +3,7 @@ import commonjs from "@rollup/plugin-commonjs"; import sourceMaps from "rollup-plugin-sourcemaps"; import typescript from "rollup-plugin-typescript2"; import json from "@rollup/plugin-json"; +import replace from "@rollup/plugin-replace"; export default [ { @@ -67,6 +68,14 @@ export default [ include: "src/scripts/**", }, plugins: [ + replace({ + RELEASE_URL_PREFIX: JSON.stringify( + "https://api.github.com/repos/janhq/nitro/releases/", + ), + TAGGED_RELEASE_URL_PREFIX: JSON.stringify( + "https://api.github.com/repos/janhq/nitro/releases/tags", + ), + }), // Allow json resolution json(), @@ -82,7 +91,7 @@ export default [ commonjs(), // Compile TypeScript files typescript({ - useTsconfigDeclarationDir: true + useTsconfigDeclarationDir: true, }), // Resolve source maps to the original source diff --git a/nitro-node/src/scripts/download-nitro.ts b/nitro-node/src/scripts/download-nitro.ts index b83f575c6..938ed3c6f 100644 --- a/nitro-node/src/scripts/download-nitro.ts +++ b/nitro-node/src/scripts/download-nitro.ts @@ -10,17 +10,12 @@ const PLATFORM = process.env.npm_config_platform || process.platform; // The platform architecture //const ARCH = process.env.npm_config_arch || process.arch; -const releaseUrlPrefixVariants = { - latest: "https://api.github.com/repos/janhq/nitro/releases/", - tag: "https://api.github.com/repos/janhq/nitro/releases/tags/", -}; - const getReleaseInfo = async (taggedVersion: string): Promise => { const releaseUrlPrefix = taggedVersion === "latest" - ? releaseUrlPrefixVariants.latest + ? RELEASE_URL_PREFIX : taggedVersion.match(/^v?\d+\.\d+\.\d+$/gi) - ? releaseUrlPrefixVariants.tag + ? TAGGED_RELEASE_URL_PREFIX : undefined; if (!releaseUrlPrefix) throw Error(`Invalid version: ${taggedVersion}`); const url = `${releaseUrlPrefix}${taggedVersion}`; @@ -59,17 +54,19 @@ const extractDownloadInfo = async ( suffixVariants: Record, ): Promise> => { console.log(`[Release URL][prerelease=${prerelease}] ${html_url}`); - const assetNames = assets.map(({ name }: { name: string }) => name); const assetsToDownloads = Object.entries(suffixVariants).reduce( ( dict: Record, [suffix, dir]: [string, string], ) => { + const assetInfo = assets.find( + (a) => a.name === `nitro-${name}-${suffix}.tar.gz`, + ); // Skip if suffix is not in asset names - if (!assetNames.includes(`nitro-${name}-${suffix}.tar.gz`)) return dict; + if (!assetInfo) return dict; // Else add the download url dict[suffix] = { - url: `https://github.com/janhq/nitro/releases/download/${tag_name}/nitro-${name}-${suffix}.tar.gz`, + url: assetInfo.browser_download_url, dir, }; return dict; diff --git a/nitro-node/src/scripts/types/global.d.ts b/nitro-node/src/scripts/types/global.d.ts new file mode 100644 index 000000000..c7dc35437 --- /dev/null +++ b/nitro-node/src/scripts/types/global.d.ts @@ -0,0 +1,4 @@ +// These constants are defined in `rollup.config.ts` at the root of the repository +// In plugin `replace`, you can change the URL to a different fork if you must +declare const RELEASE_URL_PREFIX: string; +declare const TAGGED_RELEASE_URL_PREFIX: string; From 91e2c2430a3ec8affa7e33a3e1b9cc2a31449ffe Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sun, 4 Feb 2024 00:25:44 +0700 Subject: [PATCH 38/49] refactor(nitro-node): move checkMagicBytes functions out to utils --- nitro-node/jest.config.ts | 17 +++++---- nitro-node/src/index.ts | 1 + nitro-node/src/logger.ts | 14 ++++++++ nitro-node/src/nitro.ts | 52 +++++---------------------- nitro-node/src/types/index.ts | 2 +- nitro-node/src/utils/index.ts | 25 +++++++++++++ nitro-node/test/nitro-process.test.ts | 30 +++++++--------- 7 files changed, 74 insertions(+), 67 deletions(-) create mode 100644 nitro-node/src/logger.ts create mode 100644 nitro-node/src/utils/index.ts diff --git a/nitro-node/jest.config.ts b/nitro-node/jest.config.ts index 9dbaf2efa..56b9acee9 100644 --- a/nitro-node/jest.config.ts +++ b/nitro-node/jest.config.ts @@ -1,9 +1,14 @@ -import type { JestConfigWithTsJest } from 'ts-jest' +import type { JestConfigWithTsJest } from "ts-jest"; const jestConfig: JestConfigWithTsJest = { - preset: 'ts-jest', - testEnvironment: 'node', - transformIgnorePatterns: ['/node_modules/'] -} + preset: "ts-jest", + testEnvironment: "node", + transformIgnorePatterns: ["/node_modules/"], + globals: { + RELEASE_URL_PREFIX: "https://api.github.com/repos/janhq/nitro/releases/", + TAGGED_RELEASE_URL_PREFIX: + "https://api.github.com/repos/janhq/nitro/releases/tags", + }, +}; -export default jestConfig +export default jestConfig; diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index c44d2e13c..e3deb1532 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -1,2 +1,3 @@ export * from "./types"; export * from "./nitro"; +export { setLogger } from "./logger"; diff --git a/nitro-node/src/logger.ts b/nitro-node/src/logger.ts new file mode 100644 index 000000000..cf19955b6 --- /dev/null +++ b/nitro-node/src/logger.ts @@ -0,0 +1,14 @@ +import os from "node:os"; +import { NitroLogger } from "./types"; + +// The logger to use, default to stdout +export let log: NitroLogger = (message, ..._) => + process.stdout.write(message + os.EOL); + +/** + * Set logger before running nitro + * @param {NitroLogger} logger The logger to use + */ +export async function setLogger(logger: NitroLogger): Promise { + log = logger; +} diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index 43dabf162..b3857a383 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -1,4 +1,3 @@ -import os from "node:os"; import fs from "node:fs"; import path from "node:path"; import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; @@ -14,12 +13,13 @@ import { NitroNvidiaConfig, NitroModelSetting, NitroPromptSetting, - NitroLogger, NitroModelOperationResponse, NitroModelInitOptions, ResourcesInfo, } from "./types"; import { downloadNitro } from "./scripts"; +import { checkMagicBytes } from "./utils"; +import { log } from "./logger"; // Polyfill fetch with retry const fetchRetry = fetchRT(fetch); @@ -58,7 +58,7 @@ const NVIDIA_DEFAULT_CONFIG: NitroNvidiaConfig = { const SUPPORTED_MODEL_FORMATS = [".gguf"]; // The supported model magic number -const SUPPORTED_MODEL_MAGIC_NUMBERS = ["GGUF"]; +const SUPPORTED_MODEL_MAGIC_BYTES = ["GGUF"]; // The subprocess instance for Nitro let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; @@ -68,9 +68,6 @@ let currentModelFile: string = ""; let currentSettings: NitroModelSetting | undefined = undefined; // The Nvidia info file for checking for CUDA support on the system let nvidiaConfig: NitroNvidiaConfig = NVIDIA_DEFAULT_CONFIG; -// The logger to use, default to stdout -let log: NitroLogger = (message, ..._) => - process.stdout.write(message + os.EOL); // The absolute path to bin directory let binPath: string = path.join(__dirname, "..", "bin"); @@ -118,14 +115,6 @@ export async function setNvidiaConfig( nvidiaConfig = config; } -/** - * Set logger before running nitro - * @param {NitroLogger} logger The logger to use - */ -export async function setLogger(logger: NitroLogger): Promise { - log = logger; -} - /** * Stops a Nitro subprocess. * @param wrapper - The model wrapper. @@ -135,29 +124,6 @@ export function stopModel(): Promise { return killSubprocess(); } -/** - * Read the magic bytes from a file and check if they match the provided magic bytes - */ -export async function checkMagicBytes( - filePath: string, - magicBytes: string, -): Promise { - const desired = Buffer.from(magicBytes); - const nBytes = desired.byteLength; - const chunks = []; - for await (let chunk of fs.createReadStream(filePath, { - start: 0, - end: nBytes - 1, - })) { - chunks.push(chunk); - } - const actual = Buffer.concat(chunks); - log( - `Comparing file's magic bytes <${actual.toString()}> and desired <${desired.toString()}>`, - ); - return Buffer.compare(actual, desired) === 0; -} - /** * Initializes a Nitro subprocess to load a machine learning model. * @param modelFullPath - The absolute full path to model directory. @@ -165,25 +131,25 @@ export async function checkMagicBytes( * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. */ export async function runModel({ - modelFullPath, + modelPath, promptTemplate, }: NitroModelInitOptions): Promise { // Download nitro binaries if it's not already downloaded await downloadNitro(binPath); - const files: string[] = fs.readdirSync(modelFullPath); + const files: string[] = fs.readdirSync(modelPath); // Look for model file with supported format let ggufBinFile = files.find( (file) => - file === path.basename(modelFullPath) || + file === path.basename(modelPath) || SUPPORTED_MODEL_FORMATS.some((ext) => file.toLowerCase().endsWith(ext)), ); // If not found from path and extension, try from magic number if (!ggufBinFile) { for (const f of files) { - for (const magicNum of SUPPORTED_MODEL_MAGIC_NUMBERS) { - if (await checkMagicBytes(path.join(modelFullPath, f), magicNum)) { + for (const magicBytes of SUPPORTED_MODEL_MAGIC_BYTES) { + if (await checkMagicBytes(path.join(modelPath, f), magicBytes)) { ggufBinFile = f; break; } @@ -194,7 +160,7 @@ export async function runModel({ if (!ggufBinFile) throw new Error("No GGUF model file found"); - currentModelFile = path.join(modelFullPath, ggufBinFile); + currentModelFile = path.join(modelPath, ggufBinFile); const nitroResourceProbe = await getResourcesInfo(); // Convert promptTemplate to system_prompt, user_prompt, ai_prompt diff --git a/nitro-node/src/types/index.ts b/nitro-node/src/types/index.ts index 0c9e5c749..86de45462 100644 --- a/nitro-node/src/types/index.ts +++ b/nitro-node/src/types/index.ts @@ -33,7 +33,7 @@ export interface NitroModelSetting extends NitroPromptSetting { * The response object for model init operation. */ export interface NitroModelInitOptions { - modelFullPath: string; + modelPath: string; promptTemplate?: string; } diff --git a/nitro-node/src/utils/index.ts b/nitro-node/src/utils/index.ts new file mode 100644 index 000000000..df86451cf --- /dev/null +++ b/nitro-node/src/utils/index.ts @@ -0,0 +1,25 @@ +import fs from "node:fs"; +import { log } from "../logger"; + +/** + * Read the magic bytes from a file and check if they match the provided magic bytes + */ +export async function checkMagicBytes( + filePath: string, + magicBytes: string, +): Promise { + const desired = Buffer.from(magicBytes); + const nBytes = desired.byteLength; + const chunks = []; + for await (let chunk of fs.createReadStream(filePath, { + start: 0, + end: nBytes - 1, + })) { + chunks.push(chunk); + } + const actual = Buffer.concat(chunks); + log( + `Comparing file's magic bytes <${actual.toString()}> and desired <${desired.toString()}>`, + ); + return Buffer.compare(actual, desired) === 0; +} diff --git a/nitro-node/test/nitro-process.test.ts b/nitro-node/test/nitro-process.test.ts index 72fb7be6a..64841b8ba 100644 --- a/nitro-node/test/nitro-process.test.ts +++ b/nitro-node/test/nitro-process.test.ts @@ -14,8 +14,8 @@ import { loadLLMModel, validateModelStatus, chatCompletion, - checkMagicBytes, } from "../src"; +import { checkMagicBytes } from "../src/utils"; // FIXME: Shorthand only possible for es6 targets and up //import * as model from './model.json' assert {type: 'json'} @@ -103,9 +103,7 @@ const sleep = async (ms: number): Promise => */ describe("Manage nitro process", () => { /// BEGIN SUITE CONFIG - const modelFullPath = fs.mkdtempSync( - path.join(os.tmpdir(), "nitro-node-test"), - ); + const modelPath = fs.mkdtempSync(path.join(os.tmpdir(), "nitro-node-test")); let modelCfg: Record = {}; // Setup steps before running the suite @@ -113,14 +111,14 @@ describe("Manage nitro process", () => { // Get model config from json getModelConfigHook((cfg) => Object.assign(modelCfg, cfg)), // Download model before starting tests - downloadModelHook(modelCfg, modelFullPath), + downloadModelHook(modelCfg, modelPath), ]; // Teardown steps after running the suite const teardownHooks = [ // Stop nitro after running, regardless of error or not () => stopModel(), // On teardown, cleanup tmp directory that was created earlier - cleanupTargetDirHook(modelFullPath), + cleanupTargetDirHook(modelPath), ]; /// END SUITE CONFIG @@ -145,7 +143,7 @@ describe("Manage nitro process", () => { async () => { // Start nitro await runModel({ - modelFullPath, + modelPath, promptTemplate: modelCfg.settings.prompt_template, }); // Wait 5s for nitro to start @@ -161,14 +159,14 @@ describe("Manage nitro process", () => { async () => { // Start nitro await runModel({ - modelFullPath, + modelPath, promptTemplate: modelCfg.settings.prompt_template, }); // Wait 5s for nitro to start await sleep(5 * 1000); // Load LLM model await loadLLMModel({ - llama_model_path: modelFullPath, + llama_model_path: modelPath, ctx_len: modelCfg.settings.ctx_len, ngl: modelCfg.settings.ngl, embedding: false, @@ -240,27 +238,25 @@ describe("Manage nitro process", () => { const fileName = modelCfg.source_url.split("/")?.pop() ?? "model.gguf"; // Rename the extension of model file fs.renameSync( - path.join(modelFullPath, fileName), - path.join(modelFullPath, `${fileName.replace(/\.gguf$/gi, ".bak")}`), + path.join(modelPath, fileName), + path.join(modelPath, `${fileName.replace(/\.gguf$/gi, ".bak")}`), ); }); afterEach(async () => { const fileName = modelCfg.source_url.split("/")?.pop() ?? "model.gguf"; // Restore the extension of model file fs.renameSync( - path.join(modelFullPath, `${fileName.replace(/\.gguf$/gi, ".bak")}`), - path.join(modelFullPath, fileName), + path.join(modelPath, `${fileName.replace(/\.gguf$/gi, ".bak")}`), + path.join(modelPath, fileName), ); }); test( "should be able to detect model file by magic number", async () => { - const files = fs.readdirSync(modelFullPath) as string[]; + const files = fs.readdirSync(modelPath) as string[]; // Test checking magic bytes const res = await Promise.all( - files.map((f) => - checkMagicBytes(path.join(modelFullPath, f), "GGUF"), - ), + files.map((f) => checkMagicBytes(path.join(modelPath, f), "GGUF")), ); expect(res).toContain(true); }, From 049ef1dd16ea205426ae657ee2acf5f04189833f Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sun, 4 Feb 2024 00:31:14 +0700 Subject: [PATCH 39/49] fix(nitro-node): more sensible wait time for tcp port when starting nitro --- nitro-node/src/nitro.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index b3857a383..1cf01d4ac 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -444,7 +444,7 @@ export function spawnNitroProcess(): Promise { reject(`child process exited with code ${code}`); }); - tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { + tcpPortUsed.waitUntilUsed(PORT, 300, 5000).then(() => { log(`[NITRO]::Debug: Nitro is ready`); resolve({}); }); From f5bc098257ce0dd9244b59a24c8260ef5bb229bc Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sun, 4 Feb 2024 01:51:53 +0700 Subject: [PATCH 40/49] fix(nitro-node): default run mode should be based on CUDA availability --- nitro-node/src/execute.ts | 10 +- nitro-node/src/index.ts | 1 + nitro-node/src/nitro.ts | 179 ++++++++------------------ nitro-node/src/nvidia.ts | 87 +++++++------ nitro-node/src/prompt.ts | 50 +++++++ nitro-node/src/types/index.ts | 8 +- nitro-node/src/utils/index.ts | 15 +++ nitro-node/test/nitro-process.test.ts | 5 + 8 files changed, 187 insertions(+), 168 deletions(-) create mode 100644 nitro-node/src/prompt.ts diff --git a/nitro-node/src/execute.ts b/nitro-node/src/execute.ts index b713ddb1f..1c02cfa19 100644 --- a/nitro-node/src/execute.ts +++ b/nitro-node/src/execute.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { NitroNvidiaConfig } from "./types"; +import { getNvidiaConfig } from "./nvidia"; export interface NitroExecutableOptions { executablePath: string; @@ -11,9 +11,11 @@ export interface NitroExecutableOptions { * @returns The name of the executable file to run. */ export const executableNitroFile = ( - nvidiaSettings: NitroNvidiaConfig, binaryFolder: string, + // Default to GPU if CUDA is available when calling + runMode: "cpu" | "gpu" = getNvidiaConfig().cuda.exist ? "gpu" : "cpu", ): NitroExecutableOptions => { + const nvidiaSettings = getNvidiaConfig(); let cudaVisibleDevices = ""; let binaryName = "nitro"; /** @@ -23,7 +25,7 @@ export const executableNitroFile = ( /** * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 */ - if (nvidiaSettings["run_mode"] === "cpu") { + if (runMode === "cpu") { binaryFolder = path.join(binaryFolder, "win-cpu"); } else { if (nvidiaSettings["cuda"].version === "12") { @@ -44,7 +46,7 @@ export const executableNitroFile = ( binaryFolder = path.join(binaryFolder, "mac-x64"); } } else { - if (nvidiaSettings["run_mode"] === "cpu") { + if (runMode === "cpu") { binaryFolder = path.join(binaryFolder, "linux-cpu"); } else { if (nvidiaSettings["cuda"].version === "12") { diff --git a/nitro-node/src/index.ts b/nitro-node/src/index.ts index e3deb1532..3eed4619c 100644 --- a/nitro-node/src/index.ts +++ b/nitro-node/src/index.ts @@ -1,3 +1,4 @@ export * from "./types"; export * from "./nitro"; +export { getNvidiaConfig, setNvidiaConfig } from "./nvidia"; export { setLogger } from "./logger"; diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index 1cf01d4ac..0793ed3f6 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -3,23 +3,19 @@ import path from "node:path"; import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; import tcpPortUsed from "tcp-port-used"; import fetchRT from "fetch-retry"; -import osUtils from "os-utils"; -import { - getNitroProcessInfo, - updateNvidiaInfo as _updateNvidiaInfo, -} from "./nvidia"; import { executableNitroFile } from "./execute"; import { - NitroNvidiaConfig, NitroModelSetting, NitroPromptSetting, NitroModelOperationResponse, NitroModelInitOptions, - ResourcesInfo, + NitroProcessInfo, } from "./types"; import { downloadNitro } from "./scripts"; -import { checkMagicBytes } from "./utils"; +import { checkMagicBytes, getResourcesInfo } from "./utils"; import { log } from "./logger"; +import { updateNvidiaInfo } from "./nvidia"; +import { promptTemplateConverter } from "./prompt"; // Polyfill fetch with retry const fetchRetry = fetchRT(fetch); @@ -38,22 +34,6 @@ const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; // The URL for the Nitro subprocess to run chat completion const NITRO_HTTP_CHAT_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/chat_completion`; -// The default config for using Nvidia GPU -const NVIDIA_DEFAULT_CONFIG: NitroNvidiaConfig = { - notify: true, - run_mode: "cpu", - nvidia_driver: { - exist: false, - version: "", - }, - cuda: { - exist: false, - version: "", - }, - gpus: [], - gpu_highest_vram: "", -}; - // The supported model format const SUPPORTED_MODEL_FORMATS = [".gguf"]; @@ -62,12 +42,18 @@ const SUPPORTED_MODEL_MAGIC_BYTES = ["GGUF"]; // The subprocess instance for Nitro let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; +/** + * Retrieve current nitro process + */ +const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => ({ + isRunning: subprocess != null, +}); +const getCurrentNitroProcessInfo = () => getNitroProcessInfo(subprocess); + // The current model file url let currentModelFile: string = ""; // The current model settings let currentSettings: NitroModelSetting | undefined = undefined; -// The Nvidia info file for checking for CUDA support on the system -let nvidiaConfig: NitroNvidiaConfig = NVIDIA_DEFAULT_CONFIG; // The absolute path to bin directory let binPath: string = path.join(__dirname, "..", "bin"); @@ -75,13 +61,13 @@ let binPath: string = path.join(__dirname, "..", "bin"); * Get current bin path * @returns {string} The bin path */ -export function getBinPath(): string { +function getBinPath(): string { return binPath; } /** * Set custom bin path */ -export async function setBinPath(customBinPath: string): Promise { +async function setBinPath(customBinPath: string): Promise { // Check if the path is a directory if ( fs.existsSync(customBinPath) && @@ -96,23 +82,13 @@ export async function setBinPath(customBinPath: string): Promise { } /** - * Get current Nvidia config - * @returns {NitroNvidiaConfig} A copy of the config object - * The returned object should be used for reading only - * Writing to config should be via the function {@setNvidiaConfig} - */ -export function getNvidiaConfig(): NitroNvidiaConfig { - return Object.assign({}, nvidiaConfig); -} - -/** - * Set custom Nvidia config for running inference over GPU - * @param {NitroNvidiaConfig} config The new config to apply + * Initializes the library. Must be called before any other function. + * This loads the neccesary system information and set some defaults before running model */ -export async function setNvidiaConfig( - config: NitroNvidiaConfig, -): Promise { - nvidiaConfig = config; +async function initialize(): Promise { + // Update nvidia info + await updateNvidiaInfo(); + log("[NITRO]::Debug: Nitro initialized"); } /** @@ -120,7 +96,7 @@ export async function setNvidiaConfig( * @param wrapper - The model wrapper. * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. */ -export function stopModel(): Promise { +function stopModel(): Promise { return killSubprocess(); } @@ -130,10 +106,10 @@ export function stopModel(): Promise { * @param promptTemplate - The template to use for generating prompts. * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. */ -export async function runModel({ - modelPath, - promptTemplate, -}: NitroModelInitOptions): Promise { +async function runModel( + { modelPath, promptTemplate }: NitroModelInitOptions, + runMode?: "cpu" | "gpu", +): Promise { // Download nitro binaries if it's not already downloaded await downloadNitro(binPath); const files: string[] = fs.readdirSync(modelPath); @@ -178,7 +154,7 @@ export async function runModel({ Math.round(nitroResourceProbe.numCpuPhysicalCore / 2), ), }; - return runNitroAndLoadModel(); + return runNitroAndLoadModel(runMode); } /** @@ -187,7 +163,9 @@ export async function runModel({ * 3. Validate model status * @returns */ -export async function runNitroAndLoadModel(): Promise { +async function runNitroAndLoadModel( + runMode?: "cpu" | "gpu", +): Promise { try { // Gather system information for CPU physical cores and memory await killSubprocess(); @@ -200,7 +178,7 @@ export async function runNitroAndLoadModel(): Promise setTimeout(() => resolve({}), 500)); } - const spawnResult = await spawnNitroProcess(); + const spawnResult = await spawnNitroProcess(runMode); if (spawnResult.error) { return spawnResult; } @@ -218,60 +196,14 @@ export async function runNitroAndLoadModel(): Promise { +async function loadLLMModel(settings: any): Promise { + // The nitro subprocess must be started before loading model + if (!subprocess) throw Error("Calling loadLLMModel without running nitro"); + log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`); try { const res = await fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { @@ -301,7 +233,7 @@ export async function loadLLMModel(settings: any): Promise { * @returns {Promise} A Promise that resolves when the chat completion success, or rejects with an error if the completion fails. * @description If outStream is specified, the response body is consumed and cannot be used to reconstruct the data */ -export async function chatCompletion( +async function chatCompletion( request: any, outStream?: WritableStream, ): Promise { @@ -350,7 +282,7 @@ export async function chatCompletion( * If the model is loaded successfully, the object is empty. * If the model is not loaded successfully, the object contains an error message. */ -export async function validateModelStatus(): Promise { +async function validateModelStatus(): Promise { // Send a GET request to the validation URL. // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. const response = await fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { @@ -382,12 +314,13 @@ export async function validateModelStatus(): Promise { +async function killSubprocess(): Promise { const controller = new AbortController(); setTimeout(() => controller.abort(), 5000); log(`[NITRO]::Debug: Request to kill Nitro`); try { + // FIXME: should use this response? const _response = await fetch(NITRO_HTTP_KILL_URL, { method: "DELETE", signal: controller.signal, @@ -406,11 +339,13 @@ export async function killSubprocess(): Promise { * Spawns a Nitro subprocess. * @returns A promise that resolves when the Nitro subprocess is started. */ -export function spawnNitroProcess(): Promise { +function spawnNitroProcess( + runMode?: "cpu" | "gpu", +): Promise { log(`[NITRO]::Debug: Spawning Nitro subprocess...`); return new Promise(async (resolve, reject) => { - const executableOptions = executableNitroFile(nvidiaConfig, binPath); + const executableOptions = executableNitroFile(binPath, runMode); const args: string[] = ["1", LOCAL_HOST, PORT.toString()]; // Execute the binary @@ -451,23 +386,6 @@ export function spawnNitroProcess(): Promise { }); } -/** - * Get the system resources information - */ -export async function getResourcesInfo(): Promise { - const cpu = osUtils.cpuCount(); - log(`[NITRO]::CPU informations - ${cpu}`); - const response: ResourcesInfo = { - numCpuPhysicalCore: cpu, - memAvailable: 0, - }; - return response; -} - -export const updateNvidiaInfo = async () => - await _updateNvidiaInfo(nvidiaConfig); -export const getCurrentNitroProcessInfo = () => getNitroProcessInfo(subprocess); - /** * Trap for system signal so we can stop nitro process on exit */ @@ -475,3 +393,16 @@ process.on("SIGTERM", async () => { log(`[NITRO]::Debug: Received SIGTERM signal`); await killSubprocess(); }); + +export { + getCurrentNitroProcessInfo, + getBinPath, + setBinPath, + initialize, + stopModel, + runModel, + loadLLMModel, + chatCompletion, + validateModelStatus, + killSubprocess, +}; diff --git a/nitro-node/src/nvidia.ts b/nitro-node/src/nvidia.ts index 8ae6d03c9..57e32741f 100644 --- a/nitro-node/src/nvidia.ts +++ b/nitro-node/src/nvidia.ts @@ -3,57 +3,71 @@ import { exec } from "node:child_process"; import { NitroNvidiaConfig } from "./types"; import path from "node:path"; +// The default config for using Nvidia GPU +const NVIDIA_DEFAULT_CONFIG: NitroNvidiaConfig = { + notify: true, + nvidia_driver: { + exist: false, + version: "", + }, + cuda: { + exist: false, + version: "", + }, + gpus: [], + gpu_highest_vram: "", +}; + +// The Nvidia info config for checking for CUDA support on the system +let nvidiaConfig: NitroNvidiaConfig = NVIDIA_DEFAULT_CONFIG; + /** - * Current nitro process + * Get current Nvidia config + * @returns {NitroNvidiaConfig} A copy of the config object + * The returned object should be used for reading only + * Writing to config should be via the function {@setNvidiaConfig} */ -let nitroProcessInfo: NitroProcessInfo | undefined = undefined; +export function getNvidiaConfig(): NitroNvidiaConfig { + return Object.assign({}, nvidiaConfig); +} /** - * Nitro process info + * Set custom Nvidia config for running inference over GPU + * @param {NitroNvidiaConfig} config The new config to apply */ -export interface NitroProcessInfo { - isRunning: boolean; +export async function setNvidiaConfig( + config: NitroNvidiaConfig, +): Promise { + nvidiaConfig = config; } /** - * This will retrive GPU informations and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported + * This will retrieve GPU informations + * Should be called when the library is loaded to turn on GPU acceleration if supported */ -export async function updateNvidiaInfo(nvidiaSettings: NitroNvidiaConfig) { +export async function updateNvidiaInfo() { if (process.platform !== "darwin") { await Promise.all([ - updateNvidiaDriverInfo(nvidiaSettings), - updateCudaExistence(nvidiaSettings), - updateGpuInfo(nvidiaSettings), + updateNvidiaDriverInfo(), + updateCudaExistence(), + updateGpuInfo(), ]); } } -/** - * Retrieve current nitro process - */ -export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { - nitroProcessInfo = { - isRunning: subprocess != null, - }; - return nitroProcessInfo; -}; - /** * Validate nvidia and cuda for linux and windows */ -export async function updateNvidiaDriverInfo( - nvidiaSettings: NitroNvidiaConfig, -): Promise { +async function updateNvidiaDriverInfo(): Promise { exec( "nvidia-smi --query-gpu=driver_version --format=csv,noheader", (error, stdout) => { if (!error) { const firstLine = stdout.split("\n")[0].trim(); - nvidiaSettings["nvidia_driver"].exist = true; - nvidiaSettings["nvidia_driver"].version = firstLine; + nvidiaConfig["nvidia_driver"].exist = true; + nvidiaConfig["nvidia_driver"].version = firstLine; } else { - nvidiaSettings["nvidia_driver"].exist = false; + nvidiaConfig["nvidia_driver"].exist = false; } }, ); @@ -72,7 +86,7 @@ export function checkFileExistenceInPaths( /** * Validate cuda for linux and windows */ -export function updateCudaExistence(nvidiaSettings: NitroNvidiaConfig) { +function updateCudaExistence() { let filesCuda12: string[]; let filesCuda11: string[]; let paths: string[]; @@ -106,19 +120,14 @@ export function updateCudaExistence(nvidiaSettings: NitroNvidiaConfig) { cudaVersion = "12"; } - nvidiaSettings["cuda"].exist = cudaExists; - nvidiaSettings["cuda"].version = cudaVersion; - if (cudaExists) { - nvidiaSettings.run_mode = "gpu"; - } + nvidiaConfig["cuda"].exist = cudaExists; + nvidiaConfig["cuda"].version = cudaVersion; } /** * Get GPU information */ -export async function updateGpuInfo( - nvidiaSettings: NitroNvidiaConfig, -): Promise { +async function updateGpuInfo(): Promise { exec( "nvidia-smi --query-gpu=index,memory.total --format=csv,noheader,nounits", (error, stdout) => { @@ -139,10 +148,10 @@ export async function updateGpuInfo( return { id, vram }; }); - nvidiaSettings["gpus"] = gpus; - nvidiaSettings["gpu_highest_vram"] = highestVramId; + nvidiaConfig["gpus"] = gpus; + nvidiaConfig["gpu_highest_vram"] = highestVramId; } else { - nvidiaSettings["gpus"] = []; + nvidiaConfig["gpus"] = []; } }, ); diff --git a/nitro-node/src/prompt.ts b/nitro-node/src/prompt.ts new file mode 100644 index 000000000..65b0c8c4c --- /dev/null +++ b/nitro-node/src/prompt.ts @@ -0,0 +1,50 @@ +import { NitroPromptSetting } from "./types"; + +/** + * Parse prompt template into agrs settings + * @param {string} promptTemplate Template as string + * @returns {(NitroPromptSetting | never)} parsed prompt setting + * @throws {Error} if cannot split promptTemplate + */ +export function promptTemplateConverter( + promptTemplate: string, +): NitroPromptSetting | never { + // Split the string using the markers + const systemMarker = "{system_message}"; + const promptMarker = "{prompt}"; + + if ( + promptTemplate.includes(systemMarker) && + promptTemplate.includes(promptMarker) + ) { + // Find the indices of the markers + const systemIndex = promptTemplate.indexOf(systemMarker); + const promptIndex = promptTemplate.indexOf(promptMarker); + + // Extract the parts of the string + const system_prompt = promptTemplate.substring(0, systemIndex); + const user_prompt = promptTemplate.substring( + systemIndex + systemMarker.length, + promptIndex, + ); + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length, + ); + + // Return the split parts + return { system_prompt, user_prompt, ai_prompt }; + } else if (promptTemplate.includes(promptMarker)) { + // Extract the parts of the string for the case where only promptMarker is present + const promptIndex = promptTemplate.indexOf(promptMarker); + const user_prompt = promptTemplate.substring(0, promptIndex); + const ai_prompt = promptTemplate.substring( + promptIndex + promptMarker.length, + ); + + // Return the split parts + return { user_prompt, ai_prompt }; + } + + // Throw error if none of the conditions are met + throw Error("Cannot split prompt template"); +} diff --git a/nitro-node/src/types/index.ts b/nitro-node/src/types/index.ts index 86de45462..de732b2e6 100644 --- a/nitro-node/src/types/index.ts +++ b/nitro-node/src/types/index.ts @@ -12,6 +12,13 @@ export interface ResourcesInfo { memAvailable: number; } +/** + * Nitro process info + */ +export interface NitroProcessInfo { + isRunning: boolean; +} + /** * Setting for prompts when inferencing with Nitro */ @@ -49,7 +56,6 @@ export interface NitroLogger { */ export interface NitroNvidiaConfig { notify: boolean; - run_mode: "cpu" | "gpu"; nvidia_driver: { exist: boolean; version: string; diff --git a/nitro-node/src/utils/index.ts b/nitro-node/src/utils/index.ts index df86451cf..bd9cbfb4e 100644 --- a/nitro-node/src/utils/index.ts +++ b/nitro-node/src/utils/index.ts @@ -1,5 +1,20 @@ import fs from "node:fs"; +import osUtils from "os-utils"; import { log } from "../logger"; +import { ResourcesInfo } from "../types"; + +/** + * Get the system resources information + */ +export async function getResourcesInfo(): Promise { + const cpu = osUtils.cpuCount(); + log(`[NITRO]::CPU informations - ${cpu}`); + const response: ResourcesInfo = { + numCpuPhysicalCore: cpu, + memAvailable: 0, + }; + return response; +} /** * Read the magic bytes from a file and check if they match the provided magic bytes diff --git a/nitro-node/test/nitro-process.test.ts b/nitro-node/test/nitro-process.test.ts index 64841b8ba..84ea83768 100644 --- a/nitro-node/test/nitro-process.test.ts +++ b/nitro-node/test/nitro-process.test.ts @@ -14,6 +14,7 @@ import { loadLLMModel, validateModelStatus, chatCompletion, + initialize, } from "../src"; import { checkMagicBytes } from "../src/utils"; @@ -141,6 +142,8 @@ describe("Manage nitro process", () => { test( "start/stop nitro process normally", async () => { + // Init the library + await initialize(); // Start nitro await runModel({ modelPath, @@ -157,6 +160,8 @@ describe("Manage nitro process", () => { test( "chat completion", async () => { + // Init the library + await initialize(); // Start nitro await runModel({ modelPath, From cdbd4a19801d351937b5ecfab295a72c5baffabf Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sun, 4 Feb 2024 15:42:24 +0700 Subject: [PATCH 41/49] fix(nitro-node): missing replace for global constants in index script --- nitro-node/rollup.config.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nitro-node/rollup.config.ts b/nitro-node/rollup.config.ts index ddf90b39b..1cf67d002 100644 --- a/nitro-node/rollup.config.ts +++ b/nitro-node/rollup.config.ts @@ -23,6 +23,14 @@ export default [ include: "src/**", }, plugins: [ + replace({ + RELEASE_URL_PREFIX: JSON.stringify( + "https://api.github.com/repos/janhq/nitro/releases/", + ), + TAGGED_RELEASE_URL_PREFIX: JSON.stringify( + "https://api.github.com/repos/janhq/nitro/releases/tags", + ), + }), // Allow json resolution json(), // Allow node_modules resolution, so you can use 'external' to control From b6727a0aa9e126323d6ecce614a00bd1f7f738e1 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sun, 4 Feb 2024 17:06:07 +0700 Subject: [PATCH 42/49] fix(nitro-node): fetch is not available by default on some environment --- nitro-node/package.json | 1 + nitro-node/src/nitro.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/nitro-node/package.json b/nitro-node/package.json index 33a35bf4a..67a6dfa66 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -61,6 +61,7 @@ "dependencies": { "download": "^8.0.0", "fetch-retry": "^5.0.6", + "isomorphic-fetch": "^3.0.0", "os-utils": "^0.0.14", "tcp-port-used": "^1.0.2" }, diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index 0793ed3f6..d476dcb12 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; import tcpPortUsed from "tcp-port-used"; +import "isomorphic-fetch"; import fetchRT from "fetch-retry"; import { executableNitroFile } from "./execute"; import { From d10bbbbdeda2991e9e400221501467adb3fef2eb Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sun, 4 Feb 2024 20:43:18 +0700 Subject: [PATCH 43/49] Revert "fix(nitro-node): fetch is not available by default on some environment" This reverts commit b6727a0aa9e126323d6ecce614a00bd1f7f738e1. Polyfill alternatives for fetch are not fully supported ReadableStream. --- nitro-node/package.json | 1 - nitro-node/src/nitro.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/nitro-node/package.json b/nitro-node/package.json index 67a6dfa66..33a35bf4a 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -61,7 +61,6 @@ "dependencies": { "download": "^8.0.0", "fetch-retry": "^5.0.6", - "isomorphic-fetch": "^3.0.0", "os-utils": "^0.0.14", "tcp-port-used": "^1.0.2" }, diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index d476dcb12..0793ed3f6 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -2,7 +2,6 @@ import fs from "node:fs"; import path from "node:path"; import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; import tcpPortUsed from "tcp-port-used"; -import "isomorphic-fetch"; import fetchRT from "fetch-retry"; import { executableNitroFile } from "./execute"; import { From 142a60dab94bc2a4b0aa082dba895e3372581b7c Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Sun, 4 Feb 2024 22:13:20 +0700 Subject: [PATCH 44/49] fix(nitro-node): use cross-fetch on environments which do not have fetch --- nitro-node/package.json | 1 + nitro-node/src/nitro.ts | 3 ++- nitro-node/src/scripts/download-nitro.ts | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nitro-node/package.json b/nitro-node/package.json index 33a35bf4a..3c0f04515 100644 --- a/nitro-node/package.json +++ b/nitro-node/package.json @@ -59,6 +59,7 @@ "typescript": "^5.3.3" }, "dependencies": { + "cross-fetch": "^4.0.0", "download": "^8.0.0", "fetch-retry": "^5.0.6", "os-utils": "^0.0.14", diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index 0793ed3f6..3ffdbadf8 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -16,8 +16,9 @@ import { checkMagicBytes, getResourcesInfo } from "./utils"; import { log } from "./logger"; import { updateNvidiaInfo } from "./nvidia"; import { promptTemplateConverter } from "./prompt"; +import crossFetch from "cross-fetch"; // Polyfill fetch with retry -const fetchRetry = fetchRT(fetch); +const fetchRetry = fetchRT(globalThis.fetch ?? crossFetch); // The PORT to use for the Nitro subprocess const PORT = 3928; diff --git a/nitro-node/src/scripts/download-nitro.ts b/nitro-node/src/scripts/download-nitro.ts index 938ed3c6f..89a413720 100644 --- a/nitro-node/src/scripts/download-nitro.ts +++ b/nitro-node/src/scripts/download-nitro.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import download from "download"; import { Duplex } from "node:stream"; +import crossFetch from "cross-fetch"; // Define nitro version to download in env variable ("latest" or tag "v1.2.3") const NITRO_VERSION = process.env.NITRO_VERSION || "latest"; @@ -20,7 +21,7 @@ const getReleaseInfo = async (taggedVersion: string): Promise => { if (!releaseUrlPrefix) throw Error(`Invalid version: ${taggedVersion}`); const url = `${releaseUrlPrefix}${taggedVersion}`; console.log(`[Getting release info] ${url}`); - const response = await fetch(url, { + const response = await crossFetch(url, { headers: { Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", From 21e3026dea7994504bb9eb7637473010350f2367 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Mon, 5 Feb 2024 17:49:24 +0700 Subject: [PATCH 45/49] feat(nitro-node): allow programmatically specifying the nitro version tag in code --- nitro-node/src/scripts/download-nitro.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nitro-node/src/scripts/download-nitro.ts b/nitro-node/src/scripts/download-nitro.ts index 89a413720..e166f7454 100644 --- a/nitro-node/src/scripts/download-nitro.ts +++ b/nitro-node/src/scripts/download-nitro.ts @@ -198,13 +198,16 @@ if (!(PLATFORM in variantMapping)) { // Get download config for this platform const variantConfig: Record = variantMapping[PLATFORM]; // Call the download function with version and config -export const downloadNitro = async (absBinDirPath: string) => { +export const downloadNitro = async ( + absBinDirPath: string, + version?: string, +) => { // Return early without downloading if nitro binaries are already downloaded if (verifyDownloadedBinaries(absBinDirPath)) { //console.log("Nitro binaries are already downloaded!"); return; } - const releaseInfo = await getReleaseInfo(NITRO_VERSION); + const releaseInfo = await getReleaseInfo(version ?? NITRO_VERSION); const downloadInfo = await extractDownloadInfo(releaseInfo, variantConfig); return await downloadBinaries(downloadInfo, absBinDirPath); }; From c6a265637ac82d3fe253b0f5143362232bb79977 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 6 Feb 2024 03:16:16 +0700 Subject: [PATCH 46/49] fix(nitro-node): add missing settings when loading model --- nitro-node/src/nitro.ts | 26 +++++++++++++++++++------- nitro-node/src/types/index.ts | 11 ++++++++++- nitro-node/test/nitro-process.test.ts | 10 ++++------ 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index 3ffdbadf8..8cdfc7720 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -108,7 +108,15 @@ function stopModel(): Promise { * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. */ async function runModel( - { modelPath, promptTemplate }: NitroModelInitOptions, + { + modelPath, + promptTemplate, + ctx_len = 2048, + ngl = 100, + cont_batching = false, + embedding = true, + cpu_threads, + }: NitroModelInitOptions, runMode?: "cpu" | "gpu", ): Promise { // Download nitro binaries if it's not already downloaded @@ -149,11 +157,15 @@ async function runModel( currentSettings = { ...prompt, llama_model_path: currentModelFile, + ctx_len, + ngl, + cont_batching, + embedding, // This is critical and requires real system information - cpu_threads: Math.max( - 1, - Math.round(nitroResourceProbe.numCpuPhysicalCore / 2), - ), + cpu_threads: + cpu_threads && cpu_threads > 0 + ? cpu_threads + : Math.max(1, Math.round(nitroResourceProbe.numCpuPhysicalCore / 2)), }; return runNitroAndLoadModel(runMode); } @@ -184,7 +196,7 @@ async function runNitroAndLoadModel( return spawnResult; } // TODO: Use this response? - const _loadModelResponse = await loadLLMModel(currentSettings); + const _loadModelResponse = await loadLLMModel(currentSettings!); const validationResult = await validateModelStatus(); if (validationResult.error) { return validationResult; @@ -201,7 +213,7 @@ async function runNitroAndLoadModel( * Loads a LLM model into the Nitro subprocess by sending a HTTP POST request. * @returns A Promise that resolves when the model is loaded successfully, or rejects with an error message if the model is not found or fails to load. */ -async function loadLLMModel(settings: any): Promise { +async function loadLLMModel(settings: NitroModelSetting): Promise { // The nitro subprocess must be started before loading model if (!subprocess) throw Error("Calling loadLLMModel without running nitro"); diff --git a/nitro-node/src/types/index.ts b/nitro-node/src/types/index.ts index de732b2e6..0c98bdbc3 100644 --- a/nitro-node/src/types/index.ts +++ b/nitro-node/src/types/index.ts @@ -33,15 +33,24 @@ export interface NitroPromptSetting { */ export interface NitroModelSetting extends NitroPromptSetting { llama_model_path: string; + ctx_len: number; + ngl: number; + cont_batching: boolean; + embedding: boolean; cpu_threads: number; } /** - * The response object for model init operation. + * The parameters for model init operation. */ export interface NitroModelInitOptions { modelPath: string; promptTemplate?: string; + ctx_len?: number; + ngl?: number; + cont_batching?: boolean; + embedding?: boolean; + cpu_threads?: number; } /** diff --git a/nitro-node/test/nitro-process.test.ts b/nitro-node/test/nitro-process.test.ts index 84ea83768..cd2740c23 100644 --- a/nitro-node/test/nitro-process.test.ts +++ b/nitro-node/test/nitro-process.test.ts @@ -166,16 +166,14 @@ describe("Manage nitro process", () => { await runModel({ modelPath, promptTemplate: modelCfg.settings.prompt_template, - }); - // Wait 5s for nitro to start - await sleep(5 * 1000); - // Load LLM model - await loadLLMModel({ - llama_model_path: modelPath, ctx_len: modelCfg.settings.ctx_len, ngl: modelCfg.settings.ngl, + cont_batching: false, embedding: false, + cpu_threads: -1, // Default to auto }); + // Wait 5s for nitro to start + await sleep(5 * 1000); // Validate model status await validateModelStatus(); // Arrays of all the chunked response From 8a309a31cf253628f3f8b4a8e22a0650ffab4437 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 6 Feb 2024 07:01:58 +0700 Subject: [PATCH 47/49] feat(nitro-node): add stdio and event handler for nitro subprocess This expose exit code, termination signal, and stdio for nitro subprocess This should help solving janhq/nitro#369 --- nitro-node/src/nitro.ts | 45 ++++++++++++++++++++++++++++------- nitro-node/src/types/index.ts | 25 +++++++++++++++++++ 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/nitro-node/src/nitro.ts b/nitro-node/src/nitro.ts index 8cdfc7720..2c61954cc 100644 --- a/nitro-node/src/nitro.ts +++ b/nitro-node/src/nitro.ts @@ -1,5 +1,6 @@ import fs from "node:fs"; import path from "node:path"; +import stream from "node:stream"; import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; import tcpPortUsed from "tcp-port-used"; import fetchRT from "fetch-retry"; @@ -10,6 +11,8 @@ import { NitroModelOperationResponse, NitroModelInitOptions, NitroProcessInfo, + NitroProcessEventHandler, + NitroProcessStdioHanler, } from "./types"; import { downloadNitro } from "./scripts"; import { checkMagicBytes, getResourcesInfo } from "./utils"; @@ -50,6 +53,28 @@ const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => ({ isRunning: subprocess != null, }); const getCurrentNitroProcessInfo = () => getNitroProcessInfo(subprocess); +// Default event handler: do nothing +let processEventHandler: NitroProcessEventHandler = {}; +// Default stdio handler: log stdout and stderr +let processStdioHandler: NitroProcessStdioHanler = { + stdout: (stdout: stream.Readable | null | undefined) => { + stdout?.on("data", (data: any) => { + log(`[NITRO]::Debug: ${data}`); + }); + }, + stderr: (stderr: stream.Readable | null | undefined) => { + stderr?.on("data", (data: any) => { + log(`[NITRO]::Error: ${data}`); + }); + }, +}; + +const registerEventHandler = (handler: NitroProcessEventHandler) => { + processEventHandler = handler; +}; +const registerStdioHandler = (handler: NitroProcessStdioHanler) => { + processStdioHandler = handler; +}; // The current model file url let currentModelFile: string = ""; @@ -378,16 +403,16 @@ function spawnNitroProcess( ); // Handle subprocess output - subprocess.stdout.on("data", (data: any) => { - log(`[NITRO]::Debug: ${data}`); - }); - - subprocess.stderr.on("data", (data: any) => { - log(`[NITRO]::Error: ${data}`); - }); + processStdioHandler.stdout(subprocess.stdout); + processStdioHandler.stderr(subprocess.stderr); + // Handle events + let evt: keyof NitroProcessEventHandler; + for (evt in processEventHandler) { + subprocess.on(evt, processEventHandler[evt]!); + } - subprocess.on("close", (code: any) => { - log(`[NITRO]::Debug: Nitro exited with code: ${code}`); + subprocess.on("close", (code: number, signal: string) => { + log(`[NITRO]::Debug: Nitro exited with code: ${code}, signal: ${signal}`); subprocess = undefined; reject(`child process exited with code ${code}`); }); @@ -411,6 +436,8 @@ export { getCurrentNitroProcessInfo, getBinPath, setBinPath, + registerStdioHandler, + registerEventHandler, initialize, stopModel, runModel, diff --git a/nitro-node/src/types/index.ts b/nitro-node/src/types/index.ts index 0c98bdbc3..8d1d5a6ac 100644 --- a/nitro-node/src/types/index.ts +++ b/nitro-node/src/types/index.ts @@ -1,3 +1,7 @@ +import { ChildProcessWithoutNullStreams } from "node:child_process"; +import net from "node:net"; +import stream from "node:stream"; + /** * The response from the initModel function. * @property error - An error message if the model fails to load. @@ -19,6 +23,27 @@ export interface NitroProcessInfo { isRunning: boolean; } +export interface NitroProcessEventHandler { + close?: (code: number, signal: string) => void; + disconnect?: () => void; + error?: (e: Error) => void; + exit?: (code: number, signal: string) => void; + message?: ( + message: object, + sendHandle: net.Socket | net.Server | undefined, + ) => void; + spawn?: () => void; +} + +export interface NitroProcessStdioHanler { + stdout: (_: stream.Readable | null | undefined) => void; + stderr: (_: stream.Readable | null | undefined) => void; +} + +export interface NitroProcessCloseHandler { + (code: number, signal: string): void; +} + /** * Setting for prompts when inferencing with Nitro */ From 1d79084046cf07b7c11a23199b3eaed2a98b40d3 Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 6 Feb 2024 13:20:13 +0700 Subject: [PATCH 48/49] chore(nitro-node): split github workflow files into separate commit --- .../scripts/e2e-test-install-nitro-node.js | 138 ------- .github/workflows/build-nitro-node.yml | 355 ------------------ .github/workflows/test-install-nitro-node.yml | 163 -------- 3 files changed, 656 deletions(-) delete mode 100644 .github/scripts/e2e-test-install-nitro-node.js delete mode 100644 .github/workflows/build-nitro-node.yml delete mode 100644 .github/workflows/test-install-nitro-node.yml diff --git a/.github/scripts/e2e-test-install-nitro-node.js b/.github/scripts/e2e-test-install-nitro-node.js deleted file mode 100644 index ae2339aec..000000000 --- a/.github/scripts/e2e-test-install-nitro-node.js +++ /dev/null @@ -1,138 +0,0 @@ -const os = require("node:os"); -const path = require("node:path"); -const fs = require("node:fs"); -const { spawn } = require("node:child_process"); - -// Test on both npm and yarn -const PACKAGE_MANAGERS = ["npm", "yarn"]; -const ADD_DEP_CMDS = { - // Need to copy dependency instead of linking so test logic can check the bin - npm: "install --install-links", - yarn: "add", -}; -// Path to the package to install -const NITRO_NODE_PKG = - process.env.NITRO_NODE_PKG || - path.resolve(path.normalize(path.join(__dirname, "..", "..", "nitro-node"))); -// Prefixes of downloaded nitro bin subdirectories -const BIN_DIR_PREFIXES = { - darwin: "mac", - win32: "win", - linux: "linux", -}; - -// Utility to check for a file with nitro in name in the corresponding directory -const checkBinaries = (repoDir) => { - // FIXME: Check for unsupported platforms - const binDirPrefix = BIN_DIR_PREFIXES[process.platform]; - const searchRoot = path.join( - repoDir, - "node_modules", - "@janhq", - "nitro-node", - "bin", - ); - // Get the dir and files that indicate successful download of binaries - const matched = fs.readdirSync(searchRoot, { recursive: true }).filter( - // FIXME: the result of readdirSync with recursive option is filename - // with intermediate subdirectories so this logic might not be correct - (fname) => fname.startsWith(binDirPrefix) || fname.includes("nitro"), - ); - console.log(`Downloaded bin paths:`, matched); - - // Must have both the directory for the platform and the binary - return matched.length > 1; -}; - -// Wrapper to wait for child process to finish -const childProcessPromise = (childProcess) => - new Promise((resolve, reject) => { - childProcess.on("exit", (exitCode) => { - const exitNum = Number(exitCode); - if (0 == exitNum) { - resolve(); - } else { - reject(exitNum); - } - }); - }); - -// Create a temporary directory for testing -const createTestDir = () => - fs.mkdtempSync(path.join(os.tmpdir(), "dummy-project-")); - -// First test, create empty project dir and add nitro-node as dependency -const firstTest = async (packageManager, repoDir) => { - console.log(`[First test @ ${repoDir}] install with ${packageManager}`); - // Init project with default package.json - const cmd1 = `npm init -y`; - console.log("🖥️ " + cmd1); - await childProcessPromise( - spawn(cmd1, [], { cwd: repoDir, shell: true, stdio: "inherit" }), - ); - - // Add nitro-node as dependency - const cmd2 = `${packageManager} ${ADD_DEP_CMDS[packageManager]} ${NITRO_NODE_PKG}`; - console.log("🖥️ " + cmd2); - await childProcessPromise( - spawn(cmd2, [], { cwd: repoDir, shell: true, stdio: "inherit" }), - ); - - // Check that the binaries exists - if (!checkBinaries(repoDir)) process.exit(3); - - // Cleanup node_modules after success - //fs.rmSync(path.join(repoDir, "node_modules"), { recursive: true }); -}; - -// Second test, install the wrapper from another project dir -const secondTest = async (packageManager, repoDir, wrapperDir) => { - console.log( - `[Second test @ ${repoDir}] install ${wrapperDir} with ${packageManager}`, - ); - // Init project with default package.json - const cmd1 = `npm init -y`; - console.log("🖥️ " + cmd1); - await childProcessPromise( - spawn(cmd1, [], { cwd: repoDir, shell: true, stdio: "inherit" }), - ); - - // Add wrapper as dependency - const cmd2 = `${packageManager} ${ADD_DEP_CMDS[packageManager]} ${wrapperDir}`; - console.log("🖥️ " + cmd2); - await childProcessPromise( - spawn(cmd2, [], { cwd: repoDir, shell: true, stdio: "inherit" }), - ); - - // Check that the binaries exists - if (!checkBinaries(repoDir)) process.exit(3); -}; - -// Run all the tests for the chosen package manger -const run = async (packageManager) => { - const firstRepoDir = createTestDir(); - - // Run first test - await firstTest(packageManager, firstRepoDir); - console.log("First test ran success"); - - // FIXME: Currently failed with npm due to wrong path being resolved. - //const secondRepoDir = createTestDir(); - - // Run second test - //await secondTest(packageManager, secondRepoDir, firstRepoDir); - //console.log("Second test ran success"); -}; - -// Main, run tests for npm and yarn -const main = async () => { - await PACKAGE_MANAGERS.reduce( - (p, pkgMng) => p.then(() => run(pkgMng)), - Promise.resolve(), - ); -}; - -// Run script if called directly instead of import as module -if (require.main === module) { - main(); -} diff --git a/.github/workflows/build-nitro-node.yml b/.github/workflows/build-nitro-node.yml deleted file mode 100644 index cce5bc310..000000000 --- a/.github/workflows/build-nitro-node.yml +++ /dev/null @@ -1,355 +0,0 @@ -name: Build nitro-node - -on: - #schedule: - # - cron: "0 20 * * *" # At 8 PM UTC, which is 3 AM UTC+7 - push: - branches: - - main - tags: ["v[0-9]+.[0-9]+.[0-9]+"] - paths: [".github/workflows/build-nitro-node.yml", "nitro-node"] - pull_request: - types: [opened, synchronize, reopened] - paths: [".github/workflows/build-nitro-node.yml", "nitro-node"] - workflow_dispatch: - -jobs: - ubuntu-amd64-non-cuda-build: - runs-on: ubuntu-latest - steps: - - name: Clone - id: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - name: Restore cached model file - id: cache-model-restore - uses: actions/cache/restore@v4 - with: - path: | - nitro-node/test/test_assets/*.gguf - key: ${{ runner.os }}-model-gguf - - - uses: suisei-cn/actions-download-file@v1.4.0 - id: download-model-file - name: Download model file - with: - url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - target: nitro-node/test/test_assets/ - auto-match: true - retry-times: 3 - - - name: Save downloaded model file to cache - id: cache-model-save - uses: actions/cache/save@v4 - with: - path: | - nitro-node/test/test_assets/*.gguf - key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - - - name: Run tests - id: test_nitro_node - run: | - cd nitro-node - make clean test-ci - - #ubuntu-amd64-build: - # runs-on: ubuntu-18-04-cuda-11-7 - # steps: - # - name: Clone - # id: checkout - # uses: actions/checkout@v4 - # with: - # submodules: recursive - - # - uses: actions/setup-node@v4 - # with: - # node-version: 18 - - # - name: Restore cached model file - # id: cache-model-restore - # uses: actions/cache/restore@v4 - # with: - # path: | - # nitro-node/test/test_assets/*.gguf - # key: ${{ runner.os }}-model-gguf - - # - uses: suisei-cn/actions-download-file@v1.4.0 - # id: download-model-file - # name: Download model file - # with: - # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: nitro-node/test/test_assets/ - # auto-match: true - # retry-times: 3 - - # - name: Save downloaded model file to cache - # id: cache-model-save - # uses: actions/cache/save@v4 - # with: - # path: | - # nitro-node/test/test_assets/*.gguf - # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - - # - name: Run tests - # id: test_nitro_node - # run: | - # cd nitro-node - # make clean test-ci - - #ubuntu-amd64-cuda-build: - # runs-on: ubuntu-18-04-cuda-${{ matrix.cuda }} - # strategy: - # matrix: - # cuda: ["12-0", "11-7"] - - # steps: - # - name: Clone - # id: checkout - # uses: actions/checkout@v4 - # with: - # submodules: recursive - - # - uses: actions/setup-node@v4 - # with: - # node-version: 18 - - # - name: Restore cached model file - # id: cache-model-restore - # uses: actions/cache/restore@v4 - # with: - # path: | - # nitro-node/test/test_assets/*.gguf - # key: ${{ runner.os }}-model-gguf - - # - uses: suisei-cn/actions-download-file@v1.4.0 - # id: download-model-file - # name: Download model file - # with: - # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: nitro-node/test/test_assets/ - # auto-match: true - # retry-times: 3 - - # - name: Save downloaded model file to cache - # id: cache-model-save - # uses: actions/cache/save@v4 - # with: - # path: | - # nitro-node/test/test_assets/*.gguf - # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - - # - name: Run tests - # id: test_nitro_node - # run: | - # cd nitro-node - # make clean test-ci - - macOS-M-build: - runs-on: macos-14 - steps: - - name: Clone - id: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - name: Restore cached model file - id: cache-model-restore - uses: actions/cache/restore@v4 - with: - path: | - nitro-node/test/test_assets/*.gguf - key: ${{ runner.os }}-model-gguf - - - uses: suisei-cn/actions-download-file@v1.4.0 - id: download-model-file - name: Download model file - with: - url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - target: nitro-node/test/test_assets/ - auto-match: true - retry-times: 3 - - - name: Save downloaded model file to cache - id: cache-model-save - uses: actions/cache/save@v4 - with: - path: | - nitro-node/test/test_assets/*.gguf - key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - - - name: Run tests - id: test_nitro_node - run: | - cd nitro-node - make clean test-ci - - macOS-Intel-build: - runs-on: macos-latest - steps: - - name: Clone - id: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - name: Restore cached model file - id: cache-model-restore - uses: actions/cache/restore@v4 - with: - path: | - nitro-node/test/test_assets/*.gguf - key: ${{ runner.os }}-model-gguf - - - uses: suisei-cn/actions-download-file@v1.4.0 - id: download-model-file - name: Download model file - with: - url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - target: nitro-node/test/test_assets/ - auto-match: true - retry-times: 3 - - - name: Save downloaded model file to cache - id: cache-model-save - uses: actions/cache/save@v4 - with: - path: | - nitro-node/test/test_assets/*.gguf - key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - - - name: Run tests - id: test_nitro_node - run: | - cd nitro-node - make clean test-ci - - #windows-amd64-build: - # runs-on: windows-latest - # steps: - # - name: Clone - - # id: checkout - # uses: actions/checkout@v4 - # with: - # submodules: recursive - - # - uses: actions/setup-node@v4 - # with: - # node-version: 18 - - # - name: Setup VSWhere.exe - # uses: warrenbuckley/Setup-VSWhere@v1 - # with: - # version: latest - # silent: true - # env: - # ACTIONS_ALLOW_UNSECURE_COMMANDS: true - - # - name: Restore cached model file - # id: cache-model-restore - # uses: actions/cache/restore@v4 - # with: - # path: | - # nitro-node/test/test_assets/*.gguf - # key: ${{ runner.os }}-model-gguf - - # - uses: suisei-cn/actions-download-file@v1.4.0 - # id: download-model-file - # name: Download model file - # with: - # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: nitro-node/test/test_assets/ - # auto-match: true - # retry-times: 3 - - # - name: Save downloaded model file to cache - # id: cache-model-save - # uses: actions/cache/save@v4 - # with: - # path: | - # nitro-node/test/test_assets/*.gguf - # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - - # - name: Run tests - # id: test_nitro_node - # run: | - # cd nitro-node - # make clean test-ci - - #windows-amd64-cuda-build: - # runs-on: windows-cuda-${{ matrix.cuda }} - # strategy: - # matrix: - # cuda: ["12-0", "11-7"] - - # steps: - # - name: Clone - # id: checkout - # uses: actions/checkout@v4 - # with: - # submodules: recursive - - # - uses: actions/setup-node@v4 - # with: - # node-version: 18 - - # - name: actions-setup-cmake - # uses: jwlawson/actions-setup-cmake@v1.14.1 - - # - name: Setup VSWhere.exe - # uses: warrenbuckley/Setup-VSWhere@v1 - # with: - # version: latest - # silent: true - # env: - # ACTIONS_ALLOW_UNSECURE_COMMANDS: true - - # - uses: actions/setup-dotnet@v3 - # with: - # dotnet-version: "6.0.x" - - # - name: Restore cached model file - # id: cache-model-restore - # uses: actions/cache/restore@v4 - # with: - # path: | - # nitro-node/test/test_assets/*.gguf - # key: ${{ runner.os }}-model-gguf - - # - uses: suisei-cn/actions-download-file@v1.4.0 - # id: download-model-file - # name: Download model file - # with: - # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" - # target: nitro-node/test/test_assets/ - # auto-match: true - # retry-times: 3 - - # - name: Save downloaded model file to cache - # id: cache-model-save - # uses: actions/cache/save@v4 - # with: - # path: | - # nitro-node/test/test_assets/*.gguf - # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} - - # - name: Run tests - # id: test_nitro_node - # run: | - # cd nitro-node - # make clean test-ci diff --git a/.github/workflows/test-install-nitro-node.yml b/.github/workflows/test-install-nitro-node.yml deleted file mode 100644 index de9abb29f..000000000 --- a/.github/workflows/test-install-nitro-node.yml +++ /dev/null @@ -1,163 +0,0 @@ -name: Test install nitro-node - -on: - #schedule: - # - cron: "0 20 * * *" # At 8 PM UTC, which is 3 AM UTC+7 - push: - branches: - - main - tags: ["v[0-9]+.[0-9]+.[0-9]+"] - paths: - - ".github/scripts/e2e-test-install-nitro-node.js" - - ".github/workflows/test-install-nitro-node.yml" - - "nitro-node/**" - pull_request: - types: [opened, synchronize, reopened] - paths: - - ".github/scripts/e2e-test-install-nitro-node.js" - - ".github/workflows/test-install-nitro-node.yml" - - "nitro-node/**" - workflow_dispatch: - -jobs: - linux-pack-tarball: - runs-on: ubuntu-latest - outputs: - tarball-url: ${{ steps.upload.outputs.artifact-url }} - steps: - - name: Clone - id: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - name: Build tarball - id: build - run: | - cd nitro-node - make pack - find . -type f -name 'janhq-nitro-node-*.tgz' -exec mv {} janhq-nitro-node.tgz \; - - - name: Upload tarball as artifact - id: upload - uses: actions/upload-artifact@master - with: - name: janhq-nitro-node - path: nitro-node/janhq-nitro-node.tgz - if-no-files-found: error - - ubuntu-install: - runs-on: ubuntu-latest - needs: [linux-pack-tarball] - if: always() && needs.linux-pack-tarball.result == 'success' - steps: - - name: Clone - id: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - name: Download prebuilt tarball - uses: actions/download-artifact@master - with: - name: janhq-nitro-node - path: .github/scripts/ - - - name: List tarball content - id: tar-tf - run: | - cd .github - cd scripts - tar tf janhq-nitro-node.tgz - - - name: Run tests - id: test_install_nitro_node - env: - NITRO_NODE_PKG: ${{ github.workspace }}/.github/scripts/janhq-nitro-node.tgz - run: | - cd .github - cd scripts - node e2e-test-install-nitro-node.js - - macOS-install: - runs-on: macos-latest - needs: [linux-pack-tarball] - if: always() && needs.linux-pack-tarball.result == 'success' - steps: - - name: Clone - id: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - name: Download prebuilt tarball - uses: actions/download-artifact@master - with: - name: janhq-nitro-node - path: .github/scripts/ - - - name: List tarball content - id: tar-tf - run: | - cd .github - cd scripts - tar tf janhq-nitro-node.tgz - - - name: Run tests - id: test_install_nitro_node - env: - NITRO_NODE_PKG: ${{ github.workspace }}/.github/scripts/janhq-nitro-node.tgz - run: | - cd .github - cd scripts - node e2e-test-install-nitro-node.js - - windows-install: - runs-on: windows-latest - needs: [linux-pack-tarball] - if: always() && needs.linux-pack-tarball.result == 'success' - steps: - - name: Clone - - id: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - name: Download prebuilt tarball - uses: actions/download-artifact@master - with: - name: janhq-nitro-node - path: .github/scripts/ - - - name: List tarball content - id: tar-tf - run: | - cd .github - cd scripts - tar tf janhq-nitro-node.tgz - - - name: Run tests - id: test_install_nitro_node - env: - NITRO_NODE_PKG: ${{ github.workspace }}\.github\scripts\janhq-nitro-node.tgz - run: | - cd .github - cd scripts - node e2e-test-install-nitro-node.js From 81246f2fabf4b874a9b183dcaa52189d5c08f78b Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 6 Feb 2024 17:43:24 +0700 Subject: [PATCH 49/49] fix(nitro-node): remove unused interface --- nitro-node/src/types/index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nitro-node/src/types/index.ts b/nitro-node/src/types/index.ts index 8d1d5a6ac..e12ebf489 100644 --- a/nitro-node/src/types/index.ts +++ b/nitro-node/src/types/index.ts @@ -40,10 +40,6 @@ export interface NitroProcessStdioHanler { stderr: (_: stream.Readable | null | undefined) => void; } -export interface NitroProcessCloseHandler { - (code: number, signal: string): void; -} - /** * Setting for prompts when inferencing with Nitro */