👷 Changes to build inference types (#464)

Follow up to #449 

Review with whitespaces off

.gitignore

@@ -103,4 +103,7 @@ dist
 # TernJS port file
 .tern-port
 
-.DS_Store
+.DS_Store
+
+# Generated by doc-internal
+docs

package.json

@@ -18,7 +18,7 @@
 		"prettier": "^3.1.0",
 		"prettier-plugin-svelte": "^3.1.2",
 		"semver": "^7.5.0",
-		"ts-node": "^10.9.1",
+		"tsx": "^4.7.0",
 		"tsup": "^6.7.0",
 		"typescript": "^5.0.4",
 		"vite": "^5.0.2",

packages/doc-internal/package.json

@@ -13,10 +13,10 @@
 		"doc-hub": "typedoc --tsconfig ../hub/tsconfig.json --githubPages false --plugin typedoc-plugin-markdown --out ../../docs/hub --hideBreadcrumbs --hideInPageTOC --sourceLinkTemplate https://github.com/huggingface/huggingface.js/blob/main/{path}#L{line} ../hub/index.ts",
 		"doc-inference": "typedoc --tsconfig ../inference/tsconfig.json --githubPages false --plugin typedoc-plugin-markdown --out ../../docs/inference --hideBreadcrumbs --hideInPageTOC --sourceLinkTemplate https://github.com/huggingface/huggingface.js/blob/main/{path}#L{line} ../inference/src/index.ts",
 		"doc-agents": "typedoc --tsconfig ../agents/tsconfig.json --githubPages false --plugin typedoc-plugin-markdown --out ../../docs/agents --hideBreadcrumbs --hideInPageTOC --sourceLinkTemplate https://github.com/huggingface/huggingface.js/blob/main/{path}#L{line} ../agents/src/index.ts",
-		"update-toc": "node --experimental-specifier-resolution=node --loader ts-node/esm update-toc.ts",
-		"fix-cdn-versions": "node --experimental-specifier-resolution=node --loader ts-node/esm fix-cdn-versions.ts",
-		"fix-md-links": "node --experimental-specifier-resolution=node --loader ts-node/esm fix-md-links.ts",
-		"fix-md-headinghashlinks": "node --experimental-specifier-resolution=node --loader ts-node/esm fix-md-headinghashlinks.ts"
+		"update-toc": "tsx update-toc.ts",
+		"fix-cdn-versions": "tsx fix-cdn-versions.ts",
+		"fix-md-links": "tsx fix-md-links.ts",
+		"fix-md-headinghashlinks": "tsx fix-md-headinghashlinks.ts"
 	},
 	"type": "module",
 	"license": "MIT",

packages/inference/package.json

@@ -30,16 +30,17 @@
 	],
 	"source": "src/index.ts",
 	"types": "./dist/index.d.ts",
-	"main": "./dist/index.js",
-	"module": "./dist/index.mjs",
+	"main": "./dist/index.cjs",
+	"module": "./dist/index.js",
 	"exports": {
 		"types": "./dist/index.d.ts",
-		"require": "./dist/index.js",
-		"import": "./dist/index.mjs"
+		"require": "./dist/index.cjs",
+		"import": "./dist/index.js"
 	},
+	"type": "module",
 	"scripts": {
 		"build": "tsup src/index.ts --format cjs,esm --clean && pnpm run dts",
-		"dts": "ts-node scripts/generate-dts.ts",
+		"dts": "tsx scripts/generate-dts.ts",
 		"lint": "eslint --quiet --fix --ext .cjs,.ts .",
 		"lint:check": "eslint --ext .cjs,.ts .",
 		"format": "prettier --write .",

packages/inference/scripts/generate-dts.ts

@@ -174,8 +174,6 @@ function* extractAsyncFunctions(fileContent: string): Iterable<string> {
 	}
 }
 
-writeFileSync("./dist/index.js", '/// <reference path="./index.d.ts" />\n' + readFileSync("./dist/index.js", "utf-8"));
-writeFileSync(
-	"./dist/index.mjs",
-	'/// <reference path="./index.d.ts" />\n' + readFileSync("./dist/index.mjs", "utf-8")
-);
+for (const distPath of ["./dist/index.js", "./dist/index.cjs"]) {
+	writeFileSync(distPath, '/// <reference path="./index.d.ts" />\n' + readFileSync(distPath, "utf-8"));
+}

packages/inference/tsconfig.json

@@ -11,11 +11,6 @@
 		"skipLibCheck": true,
 		"noImplicitOverride": true
 	},
-	"ts-node": {
-		"compilerOptions": {
-			"module": "commonjs"
-		}
-	},
 	"include": ["src", "test", "index.ts", "../shared/src"],
 	"exclude": ["dist"]
 }

packages/tasks/.eslintignore

@@ -0,0 +1 @@
+dist

packages/tasks/package.json

@@ -7,14 +7,14 @@
 	"publishConfig": {
 		"access": "public"
 	},
-	"main": "./dist/index.js",
-	"module": "./dist/index.mjs",
+	"main": "./dist/index.cjs",
+	"module": "./dist/index.js",
 	"types": "./dist/index.d.ts",
 	"exports": {
 		".": {
 			"types": "./dist/index.d.ts",
-			"require": "./dist/index.js",
-			"import": "./dist/index.mjs"
+			"require": "./dist/index.cjs",
+			"import": "./dist/index.js"
 		}
 	},
 	"source": "src/index.ts",
@@ -24,11 +24,12 @@
 		"format": "prettier --write .",
 		"format:check": "prettier --check .",
 		"prepublishOnly": "pnpm run build",
-		"build": "tsup src/index.ts src/scripts/**.ts --format cjs,esm --clean --dts",
+		"build": "tsup src/index.ts --format cjs,esm --clean --dts && pnpm run inference-codegen",
 		"prepare": "pnpm run build",
 		"check": "tsc",
-		"inference-codegen": "pnpm run build && node dist/scripts/inference-codegen.js"
+		"inference-codegen": "tsx scripts/inference-codegen.ts && prettier --write src/tasks/*/inference.ts"
 	},
+	"type": "module",
 	"files": [
 		"dist",
 		"src",

packages/tasks/src/scripts/inference-codegen.ts → packages/tasks/scripts/inference-codegen.ts

@@ -1,9 +1,9 @@
 import type { SerializedRenderResult } from "quicktype-core";
 import { quicktype, InputData, JSONSchemaInput, FetchingJSONSchemaStore } from "quicktype-core";
-import * as fs from "fs/promises";
-import { existsSync as pathExists } from "fs";
-import * as path from "path";
-import * as ts from "typescript";
+import * as fs from "node:fs/promises";
+import { existsSync as pathExists } from "node:fs";
+import * as path from "node:path/posix";
+import ts from "typescript";
 
 const TYPESCRIPT_HEADER_FILE = `
 /**
@@ -15,16 +15,17 @@ const TYPESCRIPT_HEADER_FILE = `
 `;
 
 const rootDirFinder = function (): string {
-	const parts = __dirname.split("/");
-	let level = parts.length - 1;
-	while (level > 0) {
-		const currentPath = parts.slice(0, level).join("/");
-		if (pathExists(`${currentPath}/package.json`)) {
-			return path.normalize(currentPath);
+	let currentPath = path.normalize(import.meta.url);
+
+	while (currentPath !== "/") {
+		if (pathExists(path.join(currentPath, "package.json"))) {
+			return currentPath;
 		}
-		level--;
+
+		currentPath = path.normalize(path.join(currentPath, ".."));
 	}
-	return "";
+
+	return "/";
 };
 
 /**
@@ -53,6 +54,7 @@ async function generateTypescript(inputData: InputData): Promise<SerializedRende
 		inputData,
 		lang: "typescript",
 		alphabetizeProperties: true,
+		indentation: "\t",
 		rendererOptions: {
 			"just-types": true,
 			"nice-property-names": true,
@@ -139,54 +141,44 @@ async function postProcessOutput(path2generated: string, outputSpec: Record<stri
 	return;
 }
 
-async function main() {
-	const rootDir = rootDirFinder();
-	const tasksDir = path.join(rootDir, "src", "tasks");
-	const allTasks = await Promise.all(
-		(await fs.readdir(tasksDir, { withFileTypes: true }))
-			.filter((entry) => entry.isDirectory())
-			.filter((entry) => entry.name !== "placeholder")
-			.map(async (entry) => ({ task: entry.name, dirPath: path.join(entry.path, entry.name) }))
-	);
-	const allSpecFiles = [
-		path.join(tasksDir, "common-definitions.json"),
-		...allTasks
-			.flatMap(({ dirPath }) => [path.join(dirPath, "spec", "input.json"), path.join(dirPath, "spec", "output.json")])
-			.filter((filepath) => pathExists(filepath)),
-	];
-
-	for (const { task, dirPath } of allTasks) {
-		const taskSpecDir = path.join(dirPath, "spec");
-		if (!(pathExists(path.join(taskSpecDir, "input.json")) && pathExists(path.join(taskSpecDir, "output.json")))) {
-			console.debug(`No spec found for task ${task} - skipping`);
-			continue;
-		}
-		console.debug(`✨ Generating types for task`, task);
-
-		console.debug("   📦 Building input data");
-		const inputData = await buildInputData(task, taskSpecDir, allSpecFiles);
-
-		console.debug("   🏭 Generating typescript code");
-		{
-			const { lines } = await generateTypescript(inputData);
-			await fs.writeFile(`${dirPath}/inference.ts`, [TYPESCRIPT_HEADER_FILE, ...lines].join(`\n`), {
-				flag: "w+",
-				encoding: "utf-8",
-			});
-		}
+const rootDir = rootDirFinder();
+const tasksDir = path.join(rootDir, "src", "tasks");
+const allTasks = await Promise.all(
+	(await fs.readdir(tasksDir, { withFileTypes: true }))
+		.filter((entry) => entry.isDirectory())
+		.filter((entry) => entry.name !== "placeholder")
+		.map(async (entry) => ({ task: entry.name, dirPath: path.join(entry.path, entry.name) }))
+);
+const allSpecFiles = [
+	path.join(tasksDir, "common-definitions.json"),
+	...allTasks
+		.flatMap(({ dirPath }) => [path.join(dirPath, "spec", "input.json"), path.join(dirPath, "spec", "output.json")])
+		.filter((filepath) => pathExists(filepath)),
+];
+
+for (const { task, dirPath } of allTasks) {
+	const taskSpecDir = path.join(dirPath, "spec");
+	if (!(pathExists(path.join(taskSpecDir, "input.json")) && pathExists(path.join(taskSpecDir, "output.json")))) {
+		console.debug(`No spec found for task ${task} - skipping`);
+		continue;
+	}
+	console.debug(`✨ Generating types for task`, task);
+
+	console.debug("   📦 Building input data");
+	const inputData = await buildInputData(task, taskSpecDir, allSpecFiles);
+
+	console.debug("   🏭 Generating typescript code");
+	{
+		const { lines } = await generateTypescript(inputData);
+		await fs.writeFile(`${dirPath}/inference.ts`, [TYPESCRIPT_HEADER_FILE, ...lines].join(`\n`), {
+			flag: "w+",
+			encoding: "utf-8",
+		});
+	}
 
-		const outputSpec = JSON.parse(await fs.readFile(`${taskSpecDir}/output.json`, { encoding: "utf-8" }));
+	const outputSpec = JSON.parse(await fs.readFile(`${taskSpecDir}/output.json`, { encoding: "utf-8" }));
 
-		console.log("   🩹 Post-processing the generated code");
-		await postProcessOutput(`${dirPath}/inference.ts`, outputSpec);
-	}
-	console.debug("✅ All done!");
+	console.log("   🩹 Post-processing the generated code");
+	await postProcessOutput(`${dirPath}/inference.ts`, outputSpec);
 }
-
-let exit = 0;
-main()
-	.catch((err) => {
-		console.error("Failure", err);
-		exit = 1;
-	})
-	.finally(() => process.exit(exit));
+console.debug("✅ All done!");

packages/tasks/tsconfig.json

@@ -2,9 +2,9 @@
 	"compilerOptions": {
 		"allowSyntheticDefaultImports": true,
 		"lib": ["ES2022", "DOM"],
-		"module": "CommonJS",
+		"module": "ESNext",
+		"target": "ESNext",
 		"moduleResolution": "node",
-		"target": "ES2022",
 		"forceConsistentCasingInFileNames": true,
 		"strict": true,
 		"noImplicitAny": true,
@@ -13,6 +13,6 @@
 		"noImplicitOverride": true,
 		"outDir": "./dist"
 	},
-	"include": ["src"],
+	"include": ["src", "scripts"],
 	"exclude": ["dist"]
 }