spx-backend: store only object keys in the DB (#505)

Updates #411
Fixes #506

spx-backend/internal/controller/controller.go

@@ -8,7 +8,6 @@ import (
 	"net/url"
 	"os"
 	"regexp"
-	"strings"
 	"time"
 
 	_ "image/png"
@@ -505,10 +504,10 @@ type UpInfo struct {
 	Expires uint64 `json:"expires"`
 	// Maximum file size allowed in bytes
 	MaxSize int64 `json:"maxSize"`
+	// Bucket name
+	Bucket string `json:"bucket"`
 	// Bucket Region
 	Region string `json:"region"`
-	// Base URL to fetch file
-	BaseUrl string `json:"baseUrl"`
 }
 
 func (ctrl *Controller) GetUpInfo(ctx context.Context) (*UpInfo, error) {
@@ -528,18 +527,18 @@ func (ctrl *Controller) GetUpInfo(ctx context.Context) (*UpInfo, error) {
 		Token:   upToken,
 		Expires: putPolicy.Expires,
 		MaxSize: putPolicy.FsizeLimit,
+		Bucket:  ctrl.kodo.bucket,
 		Region:  ctrl.kodo.bucketRegion,
-		BaseUrl: ctrl.kodo.baseUrl,
 	}, nil
 }
 
 type MakeFileURLsParams struct {
-	// Objects is a list of object keys.
+	// Objects is a list of universal URLs of the objects.
 	Objects []string `json:"objects"`
 }
 
 type FileURLs struct {
-	// ObjectURLs is a map from object keys to signed URLs for the objects.
+	// ObjectURLs is a map from universal URLs to signed web URLs for the objects.
 	ObjectURLs map[string]string `json:"objectUrls"`
 }
 
@@ -550,27 +549,28 @@ func (ctrl *Controller) MakeFileURLs(ctx context.Context, params *MakeFileURLsPa
 		ObjectURLs: make(map[string]string, len(params.Objects)),
 	}
 	for _, object := range params.Objects {
-		if !strings.HasPrefix(object, ctrl.kodo.baseUrl) {
-			fileURLs.ObjectURLs[object] = object // not a Kodo object
-			continue
-		}
 		u, err := url.Parse(object)
 		if err != nil {
-			logger.Printf("failed to parse object key: %s: %v", object, err)
+			logger.Printf("invalid object: %s: %v", object, err)
+			return nil, err
+		}
+		if u.Scheme != "kodo" || u.Host != ctrl.kodo.bucket {
+			err := fmt.Errorf("unrecognized object: %s", object)
+			logger.Printf("%v", err)
+			return nil, err
+		}
+
+		objectURL, err := url.JoinPath(ctrl.kodo.baseUrl, u.Path)
+		if err != nil {
+			logger.Printf("url.JoinPath failed: [%q, %q]: %v", ctrl.kodo.baseUrl, object, err)
 			return nil, err
 		}
-		u.Fragment = ""
 
 		// INFO: Workaround for browser caching issue with signed URLs, causing redundant downloads.
 		now := time.Now().UTC()
 		e := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC).Unix() + expires
 
-		if u.RawQuery != "" {
-			u.RawQuery += "&"
-		}
-		u.RawQuery += fmt.Sprintf("e=%d", e)
-
-		objectURL := u.String()
+		objectURL += fmt.Sprintf("?e=%d", e)
 		objectURL += "&token=" + ctrl.kodo.cred.Sign([]byte(objectURL))
 		fileURLs.ObjectURLs[object] = objectURL
 	}

spx-gui/src/apis/asset.ts

@@ -1,4 +1,4 @@
-import type { ByPage, PaginationParams, FileUrls } from './common'
+import type { FileCollection, ByPage, PaginationParams } from './common'
 import { client, IsPublic } from './common'
 
 export { IsPublic }
@@ -21,7 +21,7 @@ export type AssetData = {
   /** Public status */
   isPublic: IsPublic
   /** Files the asset contains */
-  files: FileUrls
+  files: FileCollection
   /** Preview URL for the asset, e.g., a gif for a sprite */
   preview: string
   /** Asset Type */

spx-gui/src/apis/common/index.ts

@@ -17,9 +17,20 @@ export enum IsPublic {
   public = 1
 }
 
-/** Map from relative path to URL */
-export type FileUrls = {
-  [path: string]: string
+/** Url with 'http://' or 'https://' scheme, used for web resources */
+export type WebUrl = string
+
+/** Url for universal resources, which could be either a WebUrl or a Url with a custom scheme like 'kodo://' */
+export type UniversalUrl = string
+
+/** Map from UniversalUrl to WebUrl */
+export type UniversalToWebUrlMap = {
+  [universalUrl: UniversalUrl]: WebUrl
+}
+
+/** Map from relative path to UniversalUrl */
+export type FileCollection = {
+  [path: string]: UniversalUrl
 }
 
 export const client = new Client()

spx-gui/src/apis/project.ts

@@ -1,10 +1,8 @@
-import type { ByPage, PaginationParams, FileUrls } from './common'
+import type { FileCollection, ByPage, PaginationParams } from './common'
 import { client, IsPublic } from './common'
 
 export { IsPublic }
 
-export type { FileUrls as FileCollection }
-
 export enum ProjectDataType {
   Sprite = 0,
   Backdrop = 1,
@@ -21,7 +19,7 @@ export type ProjectData = {
   /** Public status */
   isPublic: IsPublic
   /** Files the project contains */
-  files: FileUrls
+  files: FileCollection
   /** Project version */
   version: number
   /** Create time */

spx-gui/src/apis/util.ts

@@ -2,7 +2,7 @@
  * @desc util-related APIs of spx-backend
  */
 
-import { client, type FileUrls } from './common'
+import { client, type UniversalUrl, type UniversalToWebUrlMap } from './common'
 
 export interface FormatError {
   column: number
@@ -26,8 +26,8 @@ export type UpInfo = {
   expires: number
   /** Maximum file size allowed in bytes */
   maxSize: number
-  /** Base URL to fetch file */
-  baseUrl: string
+  /** Bucket name */
+  bucket: string
   /** Bucket Region */
   region: string
 }
@@ -36,11 +36,8 @@ export function getUpInfo() {
   return client.get('/util/upinfo') as Promise<UpInfo>
 }
 
-export type DownloadableFileUrls = {
-  /** Map from object keys to signed URLs for the objects */
-  objectUrls: FileUrls
-}
-
-export function makeDownloadableFileUrls(objects: string[]) {
-  return client.post('/util/fileurls', { objects: objects }) as Promise<DownloadableFileUrls>
+export function makeObjectUrls(objects: UniversalUrl[]) {
+  return client.post('/util/fileurls', { objects: objects }) as Promise<{
+    objectUrls: UniversalToWebUrlMap
+  }>
 }

spx-gui/src/components/project/ProjectCreateModal.vue

@@ -106,11 +106,11 @@ const handleSubmit = useMessageHandle(
     await sprite.autoFit()
     // upload project content & call API addProject, TODO: maybe this should be extracted to `@/models`?
     const files = project.export()[1]
-    const fileUrls = await uploadFiles(files)
+    const fileCollection = await uploadFiles(files)
     const projectData = await addProject({
       name: form.value.name,
       isPublic: IsPublic.personal,
-      files: fileUrls
+      files: fileCollection
     })
     emit('resolved', projectData)
   },

spx-gui/src/models/common/asset.ts

@@ -19,11 +19,11 @@ export type AssetModel<T extends AssetType = AssetType> = T extends AssetType.So
       : never
 
 export async function sprite2Asset(sprite: Sprite): Promise<PartialAssetData> {
-  const fileUrls = await uploadFiles(sprite.export())
+  const fileCollection = await uploadFiles(sprite.export())
   return {
     displayName: sprite.name,
     assetType: AssetType.Sprite,
-    files: fileUrls
+    files: fileCollection
   }
 }
 
@@ -40,11 +40,11 @@ const virtualBackdropConfigFileName = 'assets/__backdrop__.json'
 export async function backdrop2Asset(backdrop: Backdrop): Promise<PartialAssetData> {
   const [config, files] = backdrop.export()
   files[virtualBackdropConfigFileName] = fromConfig(virtualBackdropConfigFileName, config)
-  const fileUrls = await uploadFiles(files)
+  const fileColleciton = await uploadFiles(files)
   return {
     displayName: backdrop.name,
     assetType: AssetType.Backdrop,
-    files: fileUrls
+    files: fileColleciton
   }
 }
 
@@ -57,11 +57,11 @@ export async function asset2Backdrop(assetData: PartialAssetData) {
 }
 
 export async function sound2Asset(sound: Sound): Promise<PartialAssetData> {
-  const fileUrls = await uploadFiles(sound.export())
+  const fileCollection = await uploadFiles(sound.export())
   return {
     displayName: sound.name,
     assetType: AssetType.Sound,
-    files: fileUrls
+    files: fileCollection
   }
 }
 

spx-gui/src/models/common/cloud.ts

@@ -1,15 +1,15 @@
 import * as qiniu from 'qiniu-js'
 import { filename } from '@/utils/path'
 import { File, toNativeFile, type Files } from './file'
-import type { FileCollection, ProjectData } from '@/apis/project'
+import type { ProjectData } from '@/apis/project'
 import { IsPublic, addProject, getProject, updateProject } from '@/apis/project'
-import {
-  getUpInfo as getRawUpInfo,
-  type UpInfo as RawUpInfo,
-  makeDownloadableFileUrls
-} from '@/apis/util'
+import { getUpInfo as getRawUpInfo, type UpInfo as RawUpInfo, makeObjectUrls } from '@/apis/util'
 import { DefaultException } from '@/utils/exception'
 import type { Metadata } from '../project'
+import type { WebUrl, UniversalUrl, FileCollection } from '@/apis/common'
+
+// See https://github.com/goplus/builder/issues/411 for all the supported schemes, future plans, and discussions.
+const kodoScheme = 'kodo://'
 
 export async function load(owner: string, name: string) {
   const projectData = await getProject(owner, name)
@@ -20,66 +20,90 @@ export async function save(metadata: Metadata, files: Files) {
   const { owner, name, id } = metadata
   if (owner == null) throw new Error('owner expected')
   if (!name) throw new DefaultException({ en: 'project name not specified', zh: '未指定项目名' })
-  const fileUrls = await uploadFiles(files)
+  const fileCollection = await uploadFiles(files)
   const isPublic = metadata.isPublic ?? IsPublic.personal
   const projectData = await (id != null
-    ? updateProject(owner, name, { isPublic, files: fileUrls })
-    : addProject({ name, isPublic, files: fileUrls }))
+    ? updateProject(owner, name, { isPublic, files: fileCollection })
+    : addProject({ name, isPublic, files: fileCollection }))
   return await parseProjectData(projectData)
 }
 
-export async function parseProjectData({ files: fileUrls, ...metadata }: ProjectData) {
-  const files = await getFiles(fileUrls)
+export async function parseProjectData({ files: fileCollection, ...metadata }: ProjectData) {
+  const files = await getFiles(fileCollection)
   return { metadata, files }
 }
 
 export async function uploadFiles(files: Files): Promise<FileCollection> {
-  const fileUrls: FileCollection = {}
+  const fileCollection: FileCollection = {}
   const entries = await Promise.all(
     Object.keys(files).map(async (path) => [path, await uploadFile(files[path]!)] as const)
   )
-  for (const [path, fileUrl] of entries) {
-    fileUrls[path] = fileUrl
+  for (const [path, url] of entries) {
+    fileCollection[path] = url
   }
-  return fileUrls
+  return fileCollection
 }
 
-export async function getFiles(fileUrls: FileCollection): Promise<Files> {
-  const objects = Object.values(fileUrls)
-  const { objectUrls } = await makeDownloadableFileUrls(objects)
+export async function getFiles(fileCollection: FileCollection): Promise<Files> {
+  const objects = Object.values(fileCollection).filter((url) => url.startsWith(kodoScheme))
+  const { objectUrls } = await makeObjectUrls(objects)
+
+  // FIXME: Remove legacyObjects related code after migration is done
+  const legacyKodoUrlPrefix = 'https://builder-static-test.goplus.org/'
+  const legacyObjects = Object.values(fileCollection)
+    .filter((url) => url.startsWith(legacyKodoUrlPrefix))
+    .map(
+      (url) =>
+        kodoScheme +
+        'goplus-builder-static-test/' +
+        url.slice(legacyKodoUrlPrefix.length).split('?')[0]
+    )
+  const { objectUrls: legacyObjectUrls } = await makeObjectUrls(legacyObjects)
+
   const files: Files = {}
-  Object.keys(fileUrls).forEach((path) => {
-    const url = objectUrls[fileUrls[path]]
-    files[path] = createFileWithUrl(filename(path), url)
+  Object.keys(fileCollection).forEach((path) => {
+    const universalUrl = fileCollection[path]
+    let webUrl = universalUrl
+    if (universalUrl.startsWith(kodoScheme)) {
+      webUrl = objectUrls[universalUrl]
+    } else if (universalUrl.startsWith(legacyKodoUrlPrefix)) {
+      webUrl =
+        legacyObjectUrls[
+          kodoScheme +
+            'goplus-builder-static-test/' +
+            universalUrl.slice(legacyKodoUrlPrefix.length).split('?')[0]
+        ]
+    }
+    const file = createFileWithWebUrl(filename(path), webUrl)
+    setUniversalUrl(file, universalUrl)
+    files[path] = file
   })
   return files
 }
 
 // A mark to avoid unnecessary uploading for static files
 // TODO: we can apply similar strategy to json or code files
 const fileUrlKey = Symbol('url')
-function setUrl(file: File, url: string) {
+function setUniversalUrl(file: File, url: UniversalUrl) {
   ;(file as any)[fileUrlKey] = url
 }
-function getUrl(file: File): string | null {
+function getUniversalUrl(file: File): UniversalUrl | null {
   return (file as any)[fileUrlKey] ?? null
 }
 
-export function createFileWithUrl(name: string, url: string) {
-  const file = new File(name, async () => {
-    const resp = await fetch(url)
+export function createFileWithWebUrl(name: string, webUrl: WebUrl) {
+  return new File(name, async () => {
+    const resp = await fetch(webUrl)
     const blob = await resp.blob()
     return blob.arrayBuffer()
   })
-  setUrl(file, url)
-  return file
 }
 
 async function uploadFile(file: File) {
-  const uploadedUrl = getUrl(file)
+  const uploadedUrl = getUniversalUrl(file)
   if (uploadedUrl != null) return uploadedUrl
-  const url = await upload(file)
-  setUrl(file, url)
+  const url = await uploadToKodo(file)
+  setUniversalUrl(file, url)
   return url
 }
 
@@ -88,9 +112,9 @@ type QiniuUploadRes = {
   hash: string
 }
 
-async function upload(file: File) {
+async function uploadToKodo(file: File) {
   const nativeFile = await toNativeFile(file)
-  const { token, maxSize, baseUrl, region } = await getUpInfo()
+  const { token, maxSize, bucket, region } = await getUpInfo()
   if (nativeFile.size > maxSize) throw new Error(`file size exceeds the limit (${maxSize} bytes)`)
   const observable = qiniu.upload(
     nativeFile,
@@ -112,7 +136,7 @@ async function upload(file: File) {
       }
     })
   })
-  return baseUrl + '/' + key
+  return (kodoScheme + bucket + '/' + key) as UniversalUrl
 }
 
 type UpInfo = Omit<RawUpInfo, 'expires'> & {