assume ext instead of name for ime files

LibreService · Mar 5, 2023 · 6e7cabb · 6e7cabb
1 parent a18aa93
commit 6e7cabb
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 63 deletions.
diff --git a/.gitignore b/.gitignore
@@ -34,7 +34,9 @@ build
 *.js.map
 public/worker.js
 .vercel
+dependency-map.json
 opencc-configs.json
 schema-files.json
 schema-target.json
+target-files.json
 *.woff2
diff --git a/package.json b/package.json
@@ -16,7 +16,7 @@
     "worker": "rollup -c rollup.worker-config.mjs",
     "font": "bash scripts/download_font.sh",
     "native": "bash scripts/build_native.sh",
-    "schema": "bash scripts/install_schemas.sh",
+    "schema": "node scripts/install_schemas.mjs",
     "lib": "bash scripts/build_lib.sh",
     "wasm": "node scripts/build_wasm.mjs",
     "dev": "vite --host",

diff --git a/scripts/install_schemas.mjs b/scripts/install_schemas.mjs
@@ -1,5 +1,5 @@
 import { spawnSync } from 'child_process'
-import { readFileSync, writeFileSync, readdirSync, mkdirSync, copyFileSync } from 'fs'
+import { readFileSync, writeFileSync, mkdirSync, copyFileSync, existsSync, readdirSync } from 'fs'
 import { cwd, chdir, exit } from 'process'
 import yaml from 'js-yaml'
 import { ensure } from './util.mjs'
@@ -9,16 +9,23 @@ const { version } = JSON.parse(readFileSync('package.json'))
 const RIME_DIR = 'build/librime_native/bin'
 const defaultPath = `${RIME_DIR}/default.yaml`
 
+// input file
 const schemas = JSON.parse(readFileSync('schemas.json'))
-const schemaFiles = {} // maps schema_id to a list of files with hash, and optionally a root schema_id
-const schemaTarget = {} // maps schema_id to target
-const targetFiles = {} // maps target to files
+
+// output files
+const schemaFiles = {} // maps schema to dict and prism
+const schemaTarget = {} // maps schema to target
+const dependencyMap = {} // maps schema to dependent schemas
+const openccConfigs = ['t2s.json']
+
+// temp data structures
+const targetSchemas = {} // maps target to a list of schemas
+const targetFiles = {} // maps target to files with hash
 const targetLicense = {}
 const ids = []
-const rootMap = {}
 
-function install (arg) {
-  ensure(spawnSync('plum/rime-install', [arg], {
+function install (target) {
+  ensure(spawnSync('plum/rime-install', [target], {
     stdio: 'inherit',
     env: {
       ...process.env,
@@ -27,40 +34,60 @@ function install (arg) {
   }))
 }
 
-const openccConfigs = ['t2s.json']
 function loadOpenCC (config) {
   if (!openccConfigs.includes(config)) {
     openccConfigs.push(config)
   }
 }
 
-function collectOpenCC (schemaId) {
-  const content = yaml.load(readFileSync(`${RIME_DIR}/${schemaId}.schema.yaml`, { encoding: 'utf-8' }))
-  for (const value of Object.values(content)) {
+function isOfficialIME (target) {
+  return !target.includes('/')
+}
+
+function parseYaml (schemaId) {
+  const content = yaml.load(readFileSync(`${RIME_DIR}/build/${schemaId}.schema.yaml`, { encoding: 'utf-8' }))
+  for (const [key, value] of Object.entries(content)) {
+    if (key === 'translator') {
+      const { dictionary, prism } = value
+      schemaFiles[schemaId] = {}
+      // By default, dictionary equals to schemaId, and prism equals to dictionary (not schemaId, see luna_pinyin_fluency)
+      if (dictionary !== schemaId) {
+        schemaFiles[schemaId].dict = dictionary
+      }
+      if (prism && prism !== dictionary) {
+        schemaFiles[schemaId].prism = prism
+      }
+    }
     if (value && typeof value === 'object' && 'opencc_config' in value) {
       loadOpenCC(value.opencc_config)
     }
   }
 }
 
+['prelude', 'essay', 'emoji'].forEach(install)
+
 for (const schema of schemas) {
-  install(schema.target)
+  const { target } = schema
+  if (!(target in targetSchemas)) {
+    targetSchemas[target] = []
+    targetFiles[target] = []
+    targetLicense[target] = schema.license
+    install(target)
+  }
   ids.push(schema.id)
-  collectOpenCC(schema.id)
-  schemaFiles[schema.id] = []
-  schemaTarget[schema.id] = schema.target
-  targetFiles[schema.target] = []
-  targetLicense[schema.target] = schema.license
+  schemaTarget[schema.id] = target
+  targetSchemas[target].push(schema.id)
   if (schema.dependencies) {
-    rootMap[schema.id] = schema.dependencies
+    dependencyMap[schema.id] = schema.dependencies
   }
   if (schema.family) {
-    for (const { id } of schema.family) {
+    for (const { id, disabled } of schema.family) {
       ids.push(id)
-      collectOpenCC(id)
-      schemaFiles[id] = []
-      schemaTarget[id] = schema.target
-      rootMap[id] = [schema.id]
+      schemaTarget[id] = target
+      targetSchemas[target].push(id)
+      if (!disabled) {
+        dependencyMap[id] = schema.dependencies
+      }
     }
   }
   if (schema.emoji) {
@@ -70,9 +97,9 @@ for (const schema of schemas) {
 }
 
 // check schemas.json integrity
-for (const [schemaId, dependencies] of Object.entries(rootMap)) {
+for (const [schemaId, dependencies] of Object.entries(dependencyMap)) {
   for (const id of dependencies) {
-    if (!(id in schemaFiles)) {
+    if (!ids.includes(id)) {
       console.error(`Integrity check fails. Dependency '${id}' of '${schemaId}' should be defined in schemas.json.`)
       exit(1)
     }
@@ -90,61 +117,75 @@ ensure(spawnSync('./rime_console', [], {
   input: ''
 }))
 
-const fileNames = readdirSync('build')
-for (const fileName of fileNames) {
-  const id = fileName.split('.')[0]
-  if (id in schemaFiles) {
-    targetFiles[schemaTarget[id]].push(fileName)
-    const md5 = ensure(spawnSync('md5sum', [`build/${fileName}`], {
-      encoding: 'utf-8'
-    })).stdout.slice(0, 32)
-    schemaFiles[id].push({ name: fileName, md5 })
-    for (const schemaId of rootMap[id] || []) {
-      if (!schemaFiles[id].includes(schemaId)) {
-        schemaFiles[id].push(schemaId)
-      }
+chdir(root)
+ids.forEach(parseYaml)
+
+for (const [target, schemaIds] of Object.entries(targetSchemas)) {
+  // find all files that belongs to a target('s npm package)
+  // wtf https://github.com/rime/plum/blob/6f502ff6fa87789847fa18200415318e705bffa4/scripts/resolver.sh#L22
+  const repoDir = `plum/package/${isOfficialIME(target) ? 'rime/' : ''}${target}`.replace('rime-', '')
+  const fileNames = []
+  for (const schemaId of schemaIds) {
+    fileNames.push(`${schemaId}.schema.yaml`)
+    const { dict, prism } = schemaFiles[schemaId]
+    const dictionary = dict || schemaId
+    const dictYaml = `${dictionary}.dict.yaml`
+    const tableBin = `${dictionary}.table.bin`
+    const reverseBin = `${dictionary}.reverse.bin`
+    const prismBin = `${prism || dictionary}.prism.bin`
+    if (!fileNames.includes(tableBin) && existsSync(`${repoDir}/${dictYaml}`)) {
+      fileNames.push(tableBin, reverseBin)
+    }
+    if (!fileNames.includes(prismBin)) {
+      fileNames.push(prismBin)
     }
   }
-}
-
-chdir(root)
+  fileNames.sort()
 
-for (const [target, fileNames] of Object.entries(targetFiles)) {
+  // make npm package and calculate hash
   const packageDir = `public/ime/${target}`
   mkdirSync(packageDir, { recursive: true })
   const packageJson = {
-    name: `@rime-contrib/${target}`,
+    name: `@${isOfficialIME(target) ? 'rime-contrib' : ''}/${target}`,
     version,
     files: fileNames,
     license: targetLicense[target]
   }
   writeFileSync(`${packageDir}/package.json`, JSON.stringify(packageJson))
   for (const fileName of fileNames) {
-    copyFileSync(`${RIME_DIR}/build/${fileName}`, `${packageDir}/${fileName}`)
+    const fullPath = `${RIME_DIR}/build/${fileName}`
+    copyFileSync(fullPath, `${packageDir}/${fileName}`)
+
+    const md5 = ensure(spawnSync('md5sum', [fullPath], {
+      encoding: 'utf-8'
+    })).stdout.slice(0, 32)
+    targetFiles[target].push({ name: fileName, md5 })
   }
 }
 
-let oldSchemaFiles
+let oldTargetFiles
 try {
-  oldSchemaFiles = JSON.parse(readFileSync('schema-files.json', 'utf-8'))
+  oldTargetFiles = JSON.parse(readFileSync('target-files.json', 'utf-8'))
 } catch (e) {
-  oldSchemaFiles = {}
+  oldTargetFiles = {}
 }
 
 const updatedTargets = []
-for (const schemaId of Object.keys(schemaFiles)) {
-  const target = schemaTarget[schemaId]
-  if (!updatedTargets.includes(target) && JSON.stringify(oldSchemaFiles[schemaId]) !== JSON.stringify(schemaFiles[schemaId])) {
+for (const [target, files] of Object.entries(targetFiles)) {
+  if (JSON.stringify(files) !== JSON.stringify(oldTargetFiles[target])) {
     updatedTargets.push(target)
   }
 }
+
 if (updatedTargets.length) {
   console.log('Updated targets:')
-  for (const updatedTarget of updatedTargets) {
-    console.log(updatedTarget)
+  for (const target of updatedTargets) {
+    console.log(target)
   }
+  writeFileSync('target-files.json', JSON.stringify(targetFiles))
   writeFileSync('schema-files.json', JSON.stringify(schemaFiles))
   writeFileSync('schema-target.json', JSON.stringify(schemaTarget))
+  writeFileSync('dependency-map.json', JSON.stringify(dependencyMap))
 } else {
   console.log('All targets are already up to date.')
 }
@@ -161,3 +202,7 @@ if (oldOpenccConfigs !== sortedOpenccConfig) {
   writeFileSync('opencc-configs.json', sortedOpenccConfig)
   console.log("opencc-configs.json is updated. You need to run 'pnpm run wasm' again.")
 }
+
+for (const fileName of readdirSync(`${RIME_DIR}/opencc`)) {
+  copyFileSync(`${RIME_DIR}/opencc/${fileName}`, `build/sysroot/usr/local/share/opencc/${fileName}`)
+}
diff --git a/scripts/install_schemas.sh b/scripts/install_schemas.sh
diff --git a/src/worker.ts b/src/worker.ts
@@ -2,6 +2,8 @@ import { expose, loadWasm } from '@libreservice/my-worker'
 import { openDB } from 'idb'
 import schemaFiles from '../schema-files.json'
 import schemaTarget from '../schema-target.json'
+import dependencyMap from '../dependency-map.json'
+import targetFiles from '../target-files.json'
 
 const HASH = 'hash'
 const CONTENT = 'content'
@@ -26,11 +28,23 @@ async function setIME (schemaId: string) {
       md5: string
       target: string
     }[] = []
-    for (const item of (schemaFiles as {[key: string]: ({ name: string, md5: string } | string)[]})[key]) {
-      if (typeof item === 'string') { // root schema_id
-        files.push(...getFiles(item))
-      } else {
-        files.push({ ...item, target: (schemaTarget as {[key: string]: string})[key] })
+
+    for (const dependency of (dependencyMap as {[key: string]: string[] | undefined})[key] || []) {
+      files.push(...getFiles(dependency))
+    }
+    const { dict, prism } = (schemaFiles as {[key: string]: { dict?: string, prism?: string }})[key]
+    const dictionary = dict || key
+    const tableBin = `${dictionary}.table.bin`
+    const reverseBin = `${dictionary}.reverse.bin`
+    const prismBin = `${prism || dictionary}.prism.bin`
+    const schemaYaml = `${key}.schema.yaml`
+    const target = (schemaTarget as {[key: string]: string})[key]
+    for (const fileName of [tableBin, reverseBin, prismBin, schemaYaml]) {
+      for (const { name, md5 } of (targetFiles as { [key: string]: { name: string, md5: string }[]})[target]) {
+        if (fileName === name) {
+          files.push({ name, md5, target })
+          break
+        }
       }
     }
     return files