feat(vite-plugin-warpdrive): new Vite plugin for uploading large assets to S3 compatible storages

Author: nekomeowww

apps/stage-tamagotchi/electron.vite.config.ts

@@ -15,6 +15,9 @@ import { DownloadLive2DSDK } from '@proj-airi/unplugin-live2d-sdk'
 import { templateCompilerOptions } from '@tresjs/core'
 import { defineConfig, externalizeDepsPlugin } from 'electron-vite'
 
+const stageUIAssetsRoot = resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-ui', 'src', 'assets'))
+const sharedCacheDir = resolve(join(import.meta.dirname, '..', '..', '.cache'))
+
 export default defineConfig({
   main: {
     plugins: [externalizeDepsPlugin()],
@@ -130,10 +133,10 @@ export default defineConfig({
       }),
 
       DownloadLive2DSDK(),
-      Download('https://dist.ayaka.moe/live2d-models/hiyori_free_zh.zip', 'hiyori_free_zh.zip', 'assets/live2d/models'),
-      Download('https://dist.ayaka.moe/live2d-models/hiyori_pro_zh.zip', 'hiyori_pro_zh.zip', 'assets/live2d/models'),
-      Download('https://dist.ayaka.moe/vrm-models/VRoid-Hub/AvatarSample-A/AvatarSample_A.vrm', 'AvatarSample_A.vrm', 'assets/vrm/models/AvatarSample-A'),
-      Download('https://dist.ayaka.moe/vrm-models/VRoid-Hub/AvatarSample-B/AvatarSample_B.vrm', 'AvatarSample_B.vrm', 'assets/vrm/models/AvatarSample-B'),
+      Download('https://dist.ayaka.moe/live2d-models/hiyori_free_zh.zip', 'hiyori_free_zh.zip', 'live2d/models', { parentDir: stageUIAssetsRoot, cacheDir: sharedCacheDir }),
+      Download('https://dist.ayaka.moe/live2d-models/hiyori_pro_zh.zip', 'hiyori_pro_zh.zip', 'live2d/models', { parentDir: stageUIAssetsRoot, cacheDir: sharedCacheDir }),
+      Download('https://dist.ayaka.moe/vrm-models/VRoid-Hub/AvatarSample-A/AvatarSample_A.vrm', 'AvatarSample_A.vrm', 'vrm/models/AvatarSample-A', { parentDir: stageUIAssetsRoot, cacheDir: sharedCacheDir }),
+      Download('https://dist.ayaka.moe/vrm-models/VRoid-Hub/AvatarSample-B/AvatarSample_B.vrm', 'AvatarSample_B.vrm', 'vrm/models/AvatarSample-B', { parentDir: stageUIAssetsRoot, cacheDir: sharedCacheDir }),
     ],
   },
 })

apps/stage-tamagotchi/package.json

@@ -135,7 +135,7 @@
     "@proj-airi/lobe-icons": "^1.0.14",
     "@proj-airi/stage-shared": "workspace:^",
     "@proj-airi/ui-transitions": "workspace:^",
-    "@proj-airi/unplugin-fetch": "^0.1.7",
+    "@proj-airi/unplugin-fetch": "^0.2.0",
     "@proj-airi/unplugin-live2d-sdk": "^0.1.6",
     "@shikijs/markdown-it": "^3.15.0",
     "@types/culori": "^4.0.1",

apps/stage-web/package.json

@@ -105,8 +105,9 @@
     "@iconify-json/vscode-icons": "^1.2.34",
     "@intlify/unplugin-vue-i18n": "^11.0.1",
     "@proj-airi/lobe-icons": "^1.0.14",
-    "@proj-airi/unplugin-fetch": "^0.1.7",
+    "@proj-airi/unplugin-fetch": "^0.2.0",
     "@proj-airi/unplugin-live2d-sdk": "^0.1.6",
+    "@proj-airi/vite-plugin-warpdrive": "workspace:*",
     "@shikijs/markdown-it": "^3.15.0",
     "@types/audioworklet": "^0.0.91",
     "@types/culori": "^4.0.1",

apps/stage-web/vite.config.ts

@@ -13,11 +13,15 @@ import Layouts from 'vite-plugin-vue-layouts'
 
 import { Download } from '@proj-airi/unplugin-fetch/vite'
 import { DownloadLive2DSDK } from '@proj-airi/unplugin-live2d-sdk/vite'
+import { createS3Provider, WarpDrivePlugin } from '@proj-airi/vite-plugin-warpdrive'
 import { templateCompilerOptions } from '@tresjs/core'
 import { LFS, SpaceCard } from 'hfup/vite'
 import { defineConfig } from 'vite'
 import { VitePWA } from 'vite-plugin-pwa'
 
+const stageUIAssetsRoot = resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-ui', 'src', 'assets'))
+const sharedCacheDir = resolve(join(import.meta.dirname, '..', '..', '.cache'))
+
 export default defineConfig({
   optimizeDeps: {
     exclude: [
@@ -63,6 +67,7 @@ export default defineConfig({
       ],
     },
   },
+
   plugins: [
     Info(),
 
@@ -153,10 +158,10 @@ export default defineConfig({
     VueDevTools(),
 
     DownloadLive2DSDK(),
-    Download('https://dist.ayaka.moe/live2d-models/hiyori_free_zh.zip', 'hiyori_free_zh.zip', 'assets/live2d/models'),
-    Download('https://dist.ayaka.moe/live2d-models/hiyori_pro_zh.zip', 'hiyori_pro_zh.zip', 'assets/live2d/models'),
-    Download('https://dist.ayaka.moe/vrm-models/VRoid-Hub/AvatarSample-A/AvatarSample_A.vrm', 'AvatarSample_A.vrm', 'assets/vrm/models/AvatarSample-A'),
-    Download('https://dist.ayaka.moe/vrm-models/VRoid-Hub/AvatarSample-B/AvatarSample_B.vrm', 'AvatarSample_B.vrm', 'assets/vrm/models/AvatarSample-B'),
+    Download('https://dist.ayaka.moe/live2d-models/hiyori_free_zh.zip', 'hiyori_free_zh.zip', 'live2d/models', { parentDir: stageUIAssetsRoot, cacheDir: sharedCacheDir }),
+    Download('https://dist.ayaka.moe/live2d-models/hiyori_pro_zh.zip', 'hiyori_pro_zh.zip', 'live2d/models', { parentDir: stageUIAssetsRoot, cacheDir: sharedCacheDir }),
+    Download('https://dist.ayaka.moe/vrm-models/VRoid-Hub/AvatarSample-A/AvatarSample_A.vrm', 'AvatarSample_A.vrm', 'vrm/models/AvatarSample-A', { parentDir: stageUIAssetsRoot, cacheDir: sharedCacheDir }),
+    Download('https://dist.ayaka.moe/vrm-models/VRoid-Hub/AvatarSample-B/AvatarSample_B.vrm', 'AvatarSample_B.vrm', 'vrm/models/AvatarSample-B', { parentDir: stageUIAssetsRoot, cacheDir: sharedCacheDir }),
 
     // HuggingFace Spaces
     LFS({ root: cwd(), extraGlobs: ['*.vrm', '*.vrma', '*.hdr', '*.cmo3', '*.png', '*.jpg', '*.jpeg', '*.gif', '*.webp', '*.bmp', '*.ttf'] }),
@@ -175,5 +180,47 @@ export default defineConfig({
       ],
       short_description: 'AI driven VTuber & Companion, supports Live2D and VRM.',
     }),
+
+    // For the following example assets:
+    //
+    // dist/assets/ort-wasm-simd-threaded.jsep-B0T3yYHD.wasm                21,596.01 kB │ gzip: 5,121.95 kB
+    // dist/assets/XiaolaiSC-Regular-SNWuh554.ttf                           22,183.94 kB
+    // dist/assets/cjkFonts_allseto_v1.11-ByBdljxl.ttf                      31,337.14 kB
+    // dist/assets/duckdb-coi-CSr8FQO4.wasm                                 32,320.49 kB │ gzip: 7,194.65 kB
+    // dist/assets/duckdb-eh-BJOC5S4x.wasm                                  32,604.02 kB │ gzip: 7,133.37 kB
+    // dist/assets/duckdb-mvp-8HYqhb4i.wasm                                 37,345.64 kB │ gzip: 8,099.69 kB
+    //
+    // they are too large to be able to put into deployments like Cloudflare Workers or Pages,
+    // we need to upload them to external storage and use renderBuiltUrl to rewrite their URLs.
+    ...((!env.S3_ENDPOINT || !env.S3_ACCESS_KEY_ID || !env.S3_SECRET_ACCESS_KEY)
+      ? []
+      : [
+          WarpDrivePlugin({
+            prefix: 'proj-airi/stage-web/',
+            include: [/\.wasm$/i, /\.ttf$/i, /\.vrm$/i, /\.zip$/i], // in existing assets, wasm, ttf, vrm files are the largest ones
+            manifest: true,
+            contentTypeBy: (filename: string) => {
+              if (filename.endsWith('.wasm')) {
+                return 'application/wasm'
+              }
+              if (filename.endsWith('.ttf')) {
+                return 'font/ttf'
+              }
+              if (filename.endsWith('.vrm')) {
+                return 'application/octet-stream'
+              }
+              if (filename.endsWith('.zip')) {
+                return 'application/zip'
+              }
+            },
+            provider: createS3Provider({
+              endpoint: env.S3_ENDPOINT,
+              accessKeyId: env.S3_ACCESS_KEY_ID,
+              secretAccessKey: env.S3_SECRET_ACCESS_KEY,
+              region: env.S3_REGION,
+              publicBaseUrl: env.WARP_DRIVE_PUBLIC_BASE ?? env.S3_ENDPOINT,
+            }),
+          }),
+        ]),
   ],
 })

apps/stage-web/wrangler.toml

@@ -2,9 +2,6 @@ name = "airi"
 compatibility_date = "2025-04-01"
 preview_urls = true
 
-[build]
-command = "pnpm -F @proj-airi/stage-web run build && pnpm -F @proj-airi/docs run build:base && mv ./docs/.vitepress/dist ./apps/stage-web/dist/docs && cp ./apps/stage-web/dist/docs/sitemap.xml ./apps/stage-web/dist/sitemap.xml  && pnpm -F @proj-airi/stage-ui run story:build && mv ./packages/stage-ui/.histoire/dist ./apps/stage-web/dist/ui"
-
 [assets]
 directory = "./dist"
 not_found_handling = "single-page-application"

cspell.config.yaml

@@ -289,6 +289,7 @@ words:
   - vtuber
   - vueuse
   - waapi
+  - warpdrive
   - Warudo
   - wavefile
   - webgpu

packages/stage-ui/package.json

@@ -156,6 +156,7 @@
     "@iconify-json/vscode-icons": "^1.2.34",
     "@moeru/std": "catalog:",
     "@proj-airi/lobe-icons": "^1.0.14",
+    "@proj-airi/vite-plugin-warpdrive": "workspace:*",
     "@types/audioworklet": "^0.0.91",
     "@types/culori": "^4.0.1",
     "@types/three": "^0.181.0",

packages/stage-ui/vite.config.ts

@@ -1,12 +1,14 @@
 import type { Plugin } from 'vite'
 
 import { join, resolve } from 'node:path'
+import { env } from 'node:process'
 
 import Vue from '@vitejs/plugin-vue'
 import Unocss from 'unocss/vite'
 import Yaml from 'unplugin-yaml/vite'
 import Inspect from 'vite-plugin-inspect'
 
+import { createS3Provider, WarpDrivePlugin } from '@proj-airi/vite-plugin-warpdrive'
 import { defineConfig } from 'vite'
 
 // For Histoire
@@ -47,5 +49,44 @@ export default defineConfig({
     // TODO: Type wrong for `unplugin-yaml` in Histoire required
     // Vite version, wait until Histoire updates to support Vite 7
     Inspect() as Plugin,
+
+    // For the following example assets:
+    //
+    // dist/assets/ort-wasm-simd-threaded.jsep-B0T3yYHD.wasm                21,596.01 kB │ gzip: 5,121.95 kB
+    // dist/assets/XiaolaiSC-Regular-SNWuh554.ttf                           22,183.94 kB
+    // dist/assets/cjkFonts_allseto_v1.11-ByBdljxl.ttf                      31,337.14 kB
+    //
+    // they are too large to be able to put into deployments like Cloudflare Workers or Pages,
+    // we need to upload them to external storage and use renderBuiltUrl to rewrite their URLs.
+    ...((!env.S3_ENDPOINT || !env.S3_ACCESS_KEY_ID || !env.S3_SECRET_ACCESS_KEY)
+      ? []
+      : [
+          WarpDrivePlugin({
+            prefix: 'proj-airi/stage-ui/',
+            include: [/\.wasm$/i, /\.ttf$/i, /\.vrm$/i, /\.zip$/i], // in existing assets, wasm, ttf, vrm files are the largest ones
+            manifest: true,
+            contentTypeBy: (filename: string) => {
+              if (filename.endsWith('.wasm')) {
+                return 'application/wasm'
+              }
+              if (filename.endsWith('.ttf')) {
+                return 'font/ttf'
+              }
+              if (filename.endsWith('.vrm')) {
+                return 'application/octet-stream'
+              }
+              if (filename.endsWith('.zip')) {
+                return 'application/zip'
+              }
+            },
+            provider: createS3Provider({
+              endpoint: env.S3_ENDPOINT,
+              accessKeyId: env.S3_ACCESS_KEY_ID,
+              secretAccessKey: env.S3_SECRET_ACCESS_KEY,
+              region: env.S3_REGION,
+              publicBaseUrl: env.WARP_DRIVE_PUBLIC_BASE ?? env.S3_ENDPOINT,
+            }),
+          }),
+        ]),
   ],
 })

packages/vite-plugin-warpdrive/README.md

@@ -0,0 +1,66 @@
+# @proj-airi/vite-plugin-warpdrive
+
+Vite plugin that rewrites selected build assets (large WASM/TTF/VRM, etc.) to remote object storage and uploads them after build. It uses Vite's `renderBuiltUrl` hook so the generated bundles reference the remote URL while keeping the local file for upload.
+
+## Why
+
+- Keep HTML/JS bundles lean while serving heavy assets (WASM, fonts, models) from object storage/CDN.
+- Simple provider abstraction; ships with an S3-compatible implementation via [`s3mini`](https://github.com/good-lly/s3mini).
+- Emits an optional manifest (`remote-assets.manifest.json`) that maps built filenames to remote URLs plus hostId/hostType for debugging.
+
+## Install
+
+```bash
+pnpm add -D @proj-airi/vite-plugin-warpdrive
+```
+
+## Usage
+
+```ts
+import { createS3Provider, WarpDrivePlugin } from '@proj-airi/vite-plugin-warpdrive'
+// vite.config.ts
+import { defineConfig } from 'vite'
+
+export default defineConfig({
+  plugins: [
+    WarpDrivePlugin({
+      prefix: 'remote-assets', // optional path prefix in the bucket
+      include: [/\.wasm$/i, /\.ttf$/i, /\.vrm$/i], // which assets to rewrite/upload
+      // includeBy: (file, ctx) => ctx.hostId?.includes('duckdb'),
+      // contentType: (file) => file.endsWith('.wasm') ? 'application/wasm' : undefined,
+      manifest: true, // emit remote-assets.manifest.json in dist
+      provider: createS3Provider({
+        endpoint: process.env.S3_ENDPOINT!,
+        accessKeyId: process.env.S3_ACCESS_KEY_ID!,
+        secretAccessKey: process.env.S3_SECRET_ACCESS_KEY!,
+        region: process.env.S3_REGION,
+        publicBaseUrl: process.env.WARP_DRIVE_PUBLIC_BASE, // defaults to endpoint
+      }),
+    }),
+  ],
+})
+```
+
+### Options
+
+- `prefix`: string path prefix for uploaded keys and URLs (e.g. `remote-assets` -> `remote-assets/assets/foo.wasm`).
+- `include`: array of regex or predicate functions to decide which assets to rewrite/upload.
+- `includeBy`: optional `(filename, ctx) => boolean` for finer control (ctx has `hostId`, `hostType`).
+- `manifest`: when true, emits `remote-assets.manifest.json` describing fileName/key/url/hostId/hostType/size.
+- `contentType`: optional `(filename) => string | undefined` resolver passed to the provider upload.
+- `logger`: optional logger ({ info, warn, error }) for custom logging sinks.
+- `provider`: any object implementing `{ getPublicUrl(key): string; upload(localPath, key, contentType?): Promise<void> }`.
+
+### createS3Provider
+
+Light wrapper around `s3mini`. Required fields:
+
+- `endpoint`: full bucket URL (e.g. `https://s3.example.com/my-bucket`).
+- `accessKeyId`, `secretAccessKey`: credentials.
+- Optional: `region`, `requestSizeInBytes`, `requestAbortTimeout`, `publicBaseUrl` (override public URL base).
+
+## How it works
+
+1. `renderBuiltUrl` returns the remote URL for matching assets while remembering the key/hostId/hostType.
+2. In `generateBundle`, local artifacts are uploaded via the provider.
+3. Optional manifest is emitted for traceability/debugging.

packages/vite-plugin-warpdrive/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@proj-airi/vite-plugin-warpdrive",
+  "type": "module",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Vite plugin to rewrite and upload heavy build assets to remote object storage",
+  "author": {
+    "name": "Moeru AI Project AIRI Team",
+    "email": "airi@moeru.ai",
+    "url": "https://github.com/moeru-ai"
+  },
+  "license": "MIT",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "files": [
+    "README.md",
+    "dist",
+    "package.json"
+  ],
+  "scripts": {
+    "build": "tsdown",
+    "typecheck": "tsc --noEmit"
+  },
+  "peerDependencies": {
+    "vite": "^5.0.0 || ^6.0.0"
+  },
+  "dependencies": {
+    "rolldown": "1.0.0-beta.51",
+    "s3mini": "^0.7.0"
+  },
+  "devDependencies": {
+    "@moeru/std": "catalog:"
+  }
+}