spx-backend: store only object keys in the DB

Updates goplus#411
Fixes goplus#506

spx-backend/internal/controller/controller.go

@@ -8,7 +8,6 @@ import (
 	"net/url"
 	"os"
 	"regexp"
-	"strings"
 	"time"
 
 	_ "image/png"
@@ -505,10 +504,10 @@ type UpInfo struct {
 	Expires uint64 `json:"expires"`
 	// Maximum file size allowed in bytes
 	MaxSize int64 `json:"maxSize"`
+	// Bucket name
+	Bucket string `json:"bucket"`
 	// Bucket Region
 	Region string `json:"region"`
-	// Base URL to fetch file
-	BaseUrl string `json:"baseUrl"`
 }
 
 func (ctrl *Controller) GetUpInfo(ctx context.Context) (*UpInfo, error) {
@@ -528,8 +527,8 @@ func (ctrl *Controller) GetUpInfo(ctx context.Context) (*UpInfo, error) {
 		Token:   upToken,
 		Expires: putPolicy.Expires,
 		MaxSize: putPolicy.FsizeLimit,
+		Bucket:  ctrl.kodo.bucket,
 		Region:  ctrl.kodo.bucketRegion,
-		BaseUrl: ctrl.kodo.baseUrl,
 	}, nil
 }
 
@@ -550,27 +549,28 @@ func (ctrl *Controller) MakeFileURLs(ctx context.Context, params *MakeFileURLsPa
 		ObjectURLs: make(map[string]string, len(params.Objects)),
 	}
 	for _, object := range params.Objects {
-		if !strings.HasPrefix(object, ctrl.kodo.baseUrl) {
-			fileURLs.ObjectURLs[object] = object // not a Kodo object
-			continue
-		}
 		u, err := url.Parse(object)
 		if err != nil {
-			logger.Printf("failed to parse object key: %s: %v", object, err)
+			logger.Printf("invalid object: %s: %v", object, err)
+			return nil, err
+		}
+		if u.Scheme != "kodo" || u.Host != ctrl.kodo.bucket {
+			err := fmt.Errorf("unrecognized object: %s", object)
+			logger.Printf("%v", err)
+			return nil, err
+		}
+
+		objectURL, err := url.JoinPath(ctrl.kodo.baseUrl, u.Path)
+		if err != nil {
+			logger.Printf("url.JoinPath failed: [%q, %q]: %v", ctrl.kodo.baseUrl, object, err)
 			return nil, err
 		}
-		u.Fragment = ""
 
 		// INFO: Workaround for browser caching issue with signed URLs, causing redundant downloads.
 		now := time.Now().UTC()
 		e := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC).Unix() + expires
 
-		if u.RawQuery != "" {
-			u.RawQuery += "&"
-		}
-		u.RawQuery += fmt.Sprintf("e=%d", e)
-
-		objectURL := u.String()
+		objectURL += fmt.Sprintf("?e=%d", e)
 		objectURL += "&token=" + ctrl.kodo.cred.Sign([]byte(objectURL))
 		fileURLs.ObjectURLs[object] = objectURL
 	}

spx-gui/src/apis/util.ts

@@ -26,8 +26,8 @@ export type UpInfo = {
   expires: number
   /** Maximum file size allowed in bytes */
   maxSize: number
-  /** Base URL to fetch file */
-  baseUrl: string
+  /** Bucket name */
+  bucket: string
   /** Bucket Region */
   region: string
 }

spx-gui/src/models/common/cloud.ts

@@ -11,6 +11,9 @@ import {
 import { DefaultException } from '@/utils/exception'
 import type { Metadata } from '../project'
 
+// See https://github.com/goplus/builder/issues/411 for all the supported schemes, future plans, and discussions.
+const kodoScheme = 'kodo://'
+
 export async function load(owner: string, name: string) {
   const projectData = await getProject(owner, name)
   return await parseProjectData(projectData)
@@ -45,12 +48,38 @@ export async function uploadFiles(files: Files): Promise<FileCollection> {
 }
 
 export async function getFiles(fileUrls: FileCollection): Promise<Files> {
-  const objects = Object.values(fileUrls)
+  const objects = Object.values(fileUrls).filter((url) => url.startsWith(kodoScheme))
   const { objectUrls } = await makeDownloadableFileUrls(objects)
+
+  // FIXME: Remove legacyObjects related code after migration is done
+  const legacyKodoUrlPrefix = 'https://builder-static-test.goplus.org/'
+  const legacyObjects = Object.values(fileUrls)
+    .filter((url) => url.startsWith(legacyKodoUrlPrefix))
+    .map(
+      (url) =>
+        kodoScheme +
+        'goplus-builder-static-test/' +
+        url.slice(legacyKodoUrlPrefix.length).split('?')[0]
+    )
+  const { objectUrls: legacyObjectUrls } = await makeDownloadableFileUrls(legacyObjects)
+
   const files: Files = {}
   Object.keys(fileUrls).forEach((path) => {
-    const url = objectUrls[fileUrls[path]]
-    files[path] = createFileWithUrl(filename(path), url)
+    const originalUrl = fileUrls[path]
+    let url = originalUrl
+    if (url.startsWith(kodoScheme)) {
+      url = objectUrls[url]
+    } else if (url.startsWith(legacyKodoUrlPrefix)) {
+      url =
+        legacyObjectUrls[
+          kodoScheme +
+            'goplus-builder-static-test/' +
+            url.slice(legacyKodoUrlPrefix.length).split('?')[0]
+        ]
+    }
+    const file = createFileWithUrl(filename(path), url)
+    setUrl(file, originalUrl)
+    files[path] = file
   })
   return files
 }
@@ -66,13 +95,11 @@ function getUrl(file: File): string | null {
 }
 
 export function createFileWithUrl(name: string, url: string) {
-  const file = new File(name, async () => {
+  return new File(name, async () => {
     const resp = await fetch(url)
     const blob = await resp.blob()
     return blob.arrayBuffer()
   })
-  setUrl(file, url)
-  return file
 }
 
 async function uploadFile(file: File) {
@@ -90,7 +117,7 @@ type QiniuUploadRes = {
 
 async function upload(file: File) {
   const nativeFile = await toNativeFile(file)
-  const { token, maxSize, baseUrl, region } = await getUpInfo()
+  const { token, maxSize, bucket, region } = await getUpInfo()
   if (nativeFile.size > maxSize) throw new Error(`file size exceeds the limit (${maxSize} bytes)`)
   const observable = qiniu.upload(
     nativeFile,
@@ -112,7 +139,7 @@ async function upload(file: File) {
       }
     })
   })
-  return baseUrl + '/' + key
+  return kodoScheme + bucket + '/' + key
 }
 
 type UpInfo = Omit<RawUpInfo, 'expires'> & {