feat: add data file generator script

Author: luxass

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 dist
+data-files

package.json

@@ -42,6 +42,7 @@
   "scripts": {
     "build": "tsup",
     "dev": "tsup --watch",
+    "generate:data-files": "tsx scripts/generate-data-files.ts",
     "test": "vitest run",
     "test:watch": "vitest",
     "lint": "eslint .",

scripts/generate-data-files.ts

@@ -0,0 +1,78 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { extname } from "node:path";
+import process from "node:process";
+
+const root = new URL("../data-files/", import.meta.url);
+
+interface Entry {
+  type: "directory" | "file";
+  name: string;
+  path: string;
+}
+
+async function run() {
+  const rootResponse = await fetch("https://unicode-proxy.mojis.dev/proxy/emoji/");
+
+  if (!rootResponse.ok) {
+    throw new Error("failed to fetch root entry");
+  }
+
+  const rootEntries: Entry[] = await rootResponse.json() as Entry[];
+
+  await mkdir(root, { recursive: true });
+
+  async function processDirectory(entry: Entry, basePath: string) {
+    const dirResponse = await fetch(`https://unicode-proxy.mojis.dev/proxy/emoji/${entry.path}`);
+    const dirEntries: Entry[] = await dirResponse.json() as Entry[];
+
+    const fileEntries = dirEntries.filter(
+      (e) =>
+        e.type === "file"
+        && e.name !== "ReadMe.txt"
+        && !e.path.includes("latest")
+        && !e.path.includes("draft"),
+    );
+
+    await Promise.all(
+      fileEntries.map(async (fileEntry) => {
+        const fullPath = basePath + fileEntry.path;
+
+        if (fullPath.includes("latest") || fullPath.includes("draft")) {
+          return;
+        }
+
+        const [_, version, ...rest] = fullPath.replace(/^\//, "").split("/");
+
+        if (!version) {
+          throw new Error(`version is undefined, ${fullPath}`);
+        }
+
+        const fileExt = extname(fileEntry.name);
+        const type = rest.join("").replace(fileExt, "");
+
+        await mkdir(new URL(type, root), { recursive: true });
+
+        const content = await fetch(`https://unicode-proxy.mojis.dev/proxy/${fullPath}`).then((res) => res.text());
+        await writeFile(
+          new URL(`${type}/v${version.toString()}${fileExt}`, root),
+          content,
+        );
+      }),
+    );
+
+    const dirEntriesToProcess = dirEntries.filter((e) => e.type === "directory");
+    await Promise.all(
+      dirEntriesToProcess.map((dir) => processDirectory(dir, basePath + entry.path)),
+    );
+  }
+
+  const directories = rootEntries.filter((e) => e.type === "directory");
+  await Promise.all(directories.map((dir) => processDirectory(dir, `emoji/${dir.path}/`)));
+
+  console.log("done");
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});