ideapad

next/.env.example

@@ -3,3 +3,5 @@
 MY_HOST="example.com"
 TZ=UTC
 DISABLE_X_ROBOTS_TAG='NO'
+GOOGLE_APPLICATION_CREDENTIALS='PATH'
+OPENAI_API_KEY='sk_***'

next/jest.config.js

@@ -1,5 +1,5 @@
 /**
- * @type {import('ts-jest').InitialOptionsTsJest}
+ * @type {import('jest').Config}
  */
 const customJestConfig = {
   roots: ['pages', 'app', 'src', 'shared', 'server'],
@@ -8,26 +8,29 @@
   collectCoverageFrom: ['**/*.(ts|tsx)', '!build/', '!**/node_modules', '!/coverage'],
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
   coverageReporters: ['json', 'lcov', 'text', 'html'],
-  // moduleNameMapper: {
-  //   '\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$':
-  //     '<rootDir>/src/test/mocks/resolves-to-path.json',
-  //   '\\.(css|less|scss|sass)$': '<rootDir>/src/test/mocks/resolves-to-path.json',
-  // },
-  // transform: {
-  //   '^.+\\.(js|jsx|ts|tsx)$': [
-  //     'ts-jest',
-  //     {
-  //       isolatedModules: true,
-  //       tsconfig: {
-  //         jsx: 'react',
-  //       },
-  //     },
-  //   ],
-  // },
-  // setupFilesAfterEnv: ['<rootDir>/jest.setup.js'],
+};
+
+const unusedConfig = {
+  moduleNameMapper: {
+    '\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$':
+      '<rootDir>/src/test/mocks/resolves-to-path.json',
+    '\\.(css|less|scss|sass)$': '<rootDir>/src/test/mocks/resolves-to-path.json',
+  },
+  transform: {
+    '^.+\\.(js|jsx|ts|tsx)$': [
+      'ts-jest',
+      {
+        isolatedModules: true,
+        tsconfig: {
+          jsx: 'react',
+        },
+      },
+    ],
+  },
+  setupFilesAfterEnv: ['<rootDir>/jest.setup.js'],
 };
 
 const nextJest = require('next/jest');
 
 // Providing the path to your Next.js app which will enable loading next.config.js and .env files
 const createJestConfig = nextJest({ dir: './' });

next/next.config.mjs

@@ -1,4 +1,8 @@
 import { PHASE_DEVELOPMENT_SERVER, PHASE_PRODUCTION_BUILD } from 'next/constants.js';
+import path from 'node:path';
+import url from 'node:url';
+
+const ___dirname = path.dirname(url.fileURLToPath(import.meta.url));
 
 /**
  * when in problem, try to sync with {@link https://github.com/vercel/next.js/tree/canary/packages/create-next-app/templates/typescript}
@@ -14,6 +18,7 @@ const nextConf = {
    */
   serverRuntimeConfig: {
     serverStartAt: new Date().toISOString(),
+    projectRoot: ___dirname,
   },
   /**
    * runtime shared configuration

next/package.json

@@ -16,6 +16,11 @@
     "typecheck:watch": "tsc --watch --noEmit"
   },
   "dependencies": {
+    "@blueprintjs/core": "^5.9.1",
+    "@chakra-ui/react": "^2.8.2",
+    "@emotion/react": "^11.11.3",
+    "@emotion/styled": "^11.11.0",
+    "@google-cloud/vision": "^4.0.3",
     "@jokester/ts-commonutil": "^0.5.0",
     "@trpc/client": "^10.45.1",
     "@trpc/next": "^10.45.1",
@@ -24,10 +29,14 @@
     "date-fns": "<3",
     "debug": "^4.3.4",
     "foxact": "^0.2.31",
+    "framer-motion": "^11.0.5",
     "lodash-es": "^4.17.21",
     "next": "^14.1",
+    "openai": "^4.27.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
+    "react-use": "^17.5.0",
+    "really-relaxed-json": "^0.3.2",
     "superjson": "^2.2.1",
     "zod": "^3.22"
   },

next/pages/moeflow-ait.tsx

@@ -0,0 +1,48 @@
+import { useEffect, useState } from 'react';
+import { trpcClient } from '../src/api/trpc-client';
+import { createDebugLogger } from '../shared/logger';
+
+const debugLogger = createDebugLogger(__filename)
+
+function FileList(props: { path: string; onFileSelected: (path: string) => void }) {
+  const files = trpcClient.moeflow.listImages.useQuery({ dir: props.path });
+  if (files.status !== 'success') {
+    return <div>Loading...</div>;
+  }
+
+  return (
+    <div>
+      <h2>File List</h2>
+      <ul>
+        {files.data.files.map((f) => (
+          <li key={f} onClick={() => props.onFileSelected(f)} className="inline-block">
+            <img src={f} className="object-cover w-32 h-32 cursor-pointer" />
+          </li>
+        ))}
+      </ul>
+    </div>
+  );
+}
+
+function ImgPreview(props: { path: string }) {
+  const mutated = trpcClient.moeflow.extractText.useQuery({ file: props.path });
+  useEffect(() => {
+    if (mutated.status === 'success') {
+      debugLogger('mutated', mutated.data);
+    }
+  }, [mutated])
+  return <img src={props.path} />;
+}
+
+function MoeflowAssistedTranslatorPage() {
+  const [imgPath, setImagPath] = useState('');
+  return (
+    <div>
+      <h1>Moeflow Assisted Translator</h1>
+      {imgPath && <ImgPreview key={imgPath} path={imgPath} />}
+      <FileList path="demo-images" onFileSelected={setImagPath} />
+    </div>
+  );
+}
+
+export default MoeflowAssistedTranslatorPage;

next/pages/moeflow.tsx

@@ -0,0 +1,14 @@
+import { MoeflowPreTransWorkarea } from '../src/moeflow-auto-translate/moetrans';
+
+function MoeflowPreTransPage() {
+  return (
+    <div className="container mx-auto py-8">
+      <h1 className="text-center text-xl">moeflow pre-translator / 萌翻实验版：机器翻译</h1>
+      <h2 className="text-center">给萌翻加入翻译辅助功能的实验</h2>
+      <hr className="my-4" />
+      <MoeflowPreTransWorkarea />
+    </div>
+  );
+}
+
+export default MoeflowPreTransPage;

next/server/api/index.ts

@@ -4,6 +4,7 @@ import { createDebugLogger } from '../../shared/logger';
 import { z } from 'zod';
 import { ApiError } from './errors';
 import { TRPCError } from '@trpc/server';
+import { moeflowRouter } from './moeflow-router';
 
 const debugLogger = createDebugLogger(__filename);
 
@@ -43,6 +44,7 @@ export const appRouter = t.router({
       };
     }),
   }),
+  moeflow: moeflowRouter,
 });
 
 // Export type router type signature,

next/server/api/moeflow-router.ts

@@ -0,0 +1,176 @@
+import { z } from 'zod';
+import { t } from './_base';
+import vision, { type v1 } from '@google-cloud/vision';
+import { createDebugLogger } from '../../shared/logger';
+import path from 'node:path';
+import fsp from 'node:fs/promises';
+import { TRPCError } from '@trpc/server';
+import { serverRuntimeConfig } from '../runtime-config';
+import openai from 'openai';
+import { withRetry } from '@jokester/ts-commonutil/lib/util/with-retry';
+// @ts-ignore
+import { toJson } from 'really-relaxed-json';
+
+const publicDir = path.join(serverRuntimeConfig.projectRoot, 'public');
+
+async function ocrText(bytes: Buffer) {
+  const client = new vision.ImageAnnotatorClient();
+
+  // TEXT_DETECTION detects and extracts text from any image
+  const [result] = await client.textDetection(bytes);
+  return {
+    textAnnotations: (result.textAnnotations ?? []).map((i) => i),
+    fullTextAnnotations: result.fullTextAnnotation && {},
+  };
+}
+
+type OcrResult = Awaited<ReturnType<typeof ocrText>>;
+
+function orgText(
+  annotations: OcrResult,
+): { text: string; leftTop: { x: number; y: number }; rightBottom: { x: number; y: number } }[] {
+  return annotations.textAnnotations.map((i) => {
+    const aabb: { x: number; y: number }[] = i.boundingPoly?.vertices ?? ([] as any);
+
+    const minX = Math.min(...aabb.map((v) => v.x));
+    const maxX = Math.max(...aabb.map((v) => v.x));
+    const minY = Math.min(...aabb.map((v) => v.y));
+    const maxY = Math.max(...aabb.map((v) => v.y));
+
+    return {
+      text: i.description!,
+      leftTop: { x: minX, y: minY },
+      rightBottom: { x: maxX, y: maxY },
+    };
+  });
+}
+
+async function aiRebuild(annotations: OcrResult): Promise<{ x: number; y: number; text: string }[]> {
+  const blocks = orgText(annotations);
+  const prompt = `
+これは漫画ページから、吹き出しごとに文言を抽出するための処理です。
+入力テキストは以下の形式に従います: (x座標, y座標): {テキスト}。
+漫画の文字方向と配置を考慮しつつ、テキストの内容と座標を確認しながら、吹き出しの境界を検出し、吹き出しごとのテキストを抽出します。抽出されたテキストは以下のJSON配列で返してください：
+{
+    "x": number, // 吹き出しのx座標
+    "y": number, // 吹き出しのy座標
+    "text": string // 吹き出しの内容
+}
+
+入力テキストは ### の後に続くものとします。
+###
+
+${blocks.map((b) => `(${b.leftTop.x}, ${b.leftTop.y}): {${b.text}}`).join('\n')}
+
+  `.trim();
+
+  const client = new openai.OpenAI({ apiKey: serverRuntimeConfig.openaiApiKey });
+
+  const completion = await client.chat.completions.create({
+    messages: [
+      {
+        role: 'user',
+        content: prompt,
+      },
+    ],
+    // model: `gpt-4`,
+    model: 'gpt-3.5-turbo-0125',
+  });
+
+  debugLogger('completion', completion);
+  const shouldBeJson = `[` + completion.choices[0]?.message?.content + `]`;
+
+  debugLogger('shouldBeJson', shouldBeJson);
+  const jsonized = JSON.parse(toJson(shouldBeJson));
+  debugLogger('toJson(shouldBeJson)', jsonized);
+  if (!Array.isArray(jsonized)) {
+    throw new Error(`fail early`);
+  }
+  return jsonized;
+}
+
+async function openaiTranslate(texts: string[]): Promise<string[]> {
+  const prompt = `
+你是一位资深漫画翻译, 请将以下日文文本翻译成繁体中文. 请注意, 本文本是从漫画中提取的, 有可能包含一些特殊的用语和表达, 请尽量保持原意.
+输入文本以JSON string[] 的形式给出. 请将请将翻译结果按照原文本的顺序, 同样以JSON string[] 的形式返回.
+
+${JSON.stringify(texts)}
+    `;
+  const client = new openai.OpenAI({ apiKey: serverRuntimeConfig.openaiApiKey });
+
+  const completion = await client.chat.completions.create({
+    messages: [
+      {
+        role: 'user',
+        content: prompt,
+      },
+    ],
+    model: `gpt-4-0125-preview`,
+    // model: 'gpt-3.5-turbo-0125',
+  });
+  return JSON.parse(completion.choices[0]?.message!.content!);
+}
+
+async function ocrDocumentText(bytes: Buffer) {
+  const client = new vision.ImageAnnotatorClient();
+
+  // DOCUMENT_TEXT_DETECTION: optimized for dense text and documents
+  const [result] = await client.documentTextDetection(bytes);
+  return result;
+}
+
+const debugLogger = createDebugLogger(__filename);
+
+export const moeflowRouter = t.router({
+  listImages: t.procedure
+    .input(
+      z.object({
+        dir: z.string(),
+      }),
+    )
+    .query(async ({ input }) => {
+      const dir = path.join(publicDir, input.dir);
+      if (!dir.startsWith(publicDir)) {
+        throw new TRPCError({ message: 'Invalid directory', code: 'FORBIDDEN' });
+      }
+
+      const files = await fsp.readdir(dir, { withFileTypes: true });
+
+      debugLogger('files', files);
+
+      return {
+        /**
+         * List of files in the directory, relative to site root
+         */
+        files: files
+          .filter((f) => f.isFile() && ['.jpg', '.jpeg', '.png'].includes(path.extname(f.name)))
+          .map((f) => '/' + path.join(input.dir, f.name)),
+      };
+    }),
+
+  extractText: t.procedure.input(z.object({ imgBytes: z.string() })).mutation(async ({ input }) => {
+    const ocrTextResult = await withRetry(() => ocrText(Buffer.from(input.imgBytes, 'base64')));
+    const rebuilt = await withRetry(() => aiRebuild(ocrTextResult), {
+      maxAttempts: 10,
+      shouldBreak(error: unknown, tried: number): boolean | PromiseLike<boolean> {
+        console.error('aiRebuild failed', error, tried);
+        return false;
+      },
+    });
+    debugLogger('rebuilt', typeof rebuilt, rebuilt);
+    const translated = await withRetry(() => openaiTranslate(rebuilt.map((b) => b.text)), {
+      maxAttempts: 10,
+      shouldBreak(error: unknown, tried: number): boolean | PromiseLike<boolean> {
+        console.error('openaiTranslate failed', error, tried);
+        return false;
+      },
+    });
+    return {
+      ...ocrTextResult,
+      blocks: orgText(ocrTextResult),
+      translated: rebuilt.map((b, i) => ({ ...b, translated: translated[i] ?? '????' })),
+    };
+  }),
+
+  translateBalloon: t.procedure.input(z.object({ text: z.string(), targetLang: z.string() })).query(({ input }) => {}),
+});

next/server/runtime-config.ts

@@ -2,5 +2,7 @@ import getConfig from 'next/config';
 
 export interface ServerRuntimeConfig {
   serverStartAt: string;
+  projectRoot: string;
+  openaiApiKey: string;
 }
 export const serverRuntimeConfig: ServerRuntimeConfig = getConfig().serverRuntimeConfig;

next/src/api/trpc-client.ts

@@ -1,8 +1,9 @@
-import { httpLink } from '@trpc/client';
+import { createTRPCClient, createTRPCProxyClient, httpLink } from '@trpc/client';
 import { createTRPCNext } from '@trpc/next';
 import type { AppRouter } from '../../server/api';
 import { inBrowser } from '../../shared/runtime-config';
 import superjson from 'superjson';
+
 function getBaseUrl() {
   if (inBrowser) {
     // browser should use relative path
@@ -34,3 +35,8 @@ export const trpcClient = createTRPCNext<AppRouter>({
    **/
   ssr: false,
 });
+
+export const trpcClient$ = createTRPCProxyClient<AppRouter>({
+  links,
+  transformer: superjson,
+});

next/src/app.scss

@@ -1,3 +1,5 @@
 @tailwind base;
 @tailwind components;
 @tailwind utilities;
+
+@import "@blueprintjs/core/lib/css/blueprint.css";