Add support for project manifest.json (#10144)

* Add support for project manifest.json

* license

* bug fix

* remove unnecessary new folder

* fix asset handling, add helpful resolvers for location and asset requests

* rework and simplify asset service

* fix and improve tests

* clean up logs

* fix problems with seeding

* fix scene save as and make studio file browser upload take a path instead of rely on hardcoded state

* remove deprecated scenePath

* fix

* update manifest scene field upon scene saving

* update template manifest.json name field

* populate scene json files into scenes field of generate manifest.json

* fixes for scenes

* fix tests and updating manifest json

* fix save as not updating manifest json

* fix bug with save as

---------

Co-authored-by: Hanzla Mateen <hanzlamateen@live.com>
Co-authored-by: Moiz Adnan <67912355+MoizAdnan@users.noreply.github.com>

packages/client-core/src/admin/components/locations/AddEditLocationModal.tsx

@@ -159,9 +159,8 @@ export default function AddEditLocationModal({ location }: { location?: Location
               : [
                   { value: '', label: t('admin:components.location.selectScene'), disabled: true },
                   ...scenes.data.map((scene) => {
-                    const split = scene.assetURL.split('/')
-                    const project = split.at(1)
-                    const name = split.at(-1)!.split('.').at(0)
+                    const project = scene.projectName
+                    const name = scene.assetURL.split('/').pop()!.split('.').at(0)!
                     return {
                       label: `${name} (${project})`,
                       value: scene.id

packages/client-core/src/admin/components/locations/LocationTable.tsx

@@ -77,7 +77,11 @@ export default function LocationTable({ search }: { search: string }) {
   const createRows = (rows: readonly LocationType[]): LocationRowType[] =>
     rows.map((row) => ({
       name: <a href={`/location/${transformLink(row.name)}`}>{row.name}</a>,
-      sceneId: <a href={`/studio/${row.sceneId.split('/')[0]}`}>{row.sceneId}</a>,
+      sceneId: (
+        <a href={`/studio?projectName=${row.sceneAsset.projectName}&scenePath=${row.sceneAsset.assetURL}`}>
+          {row.sceneId}
+        </a>
+      ),
       maxUsersPerInstance: row.maxUsersPerInstance.toString(),
       scene: row.slugifiedName,
       locationType: row.locationSetting.locationType,

packages/client-core/src/social/services/LocationService.ts

@@ -39,6 +39,7 @@ export const LocationSeed: LocationType = {
   slugifiedName: '',
   maxUsersPerInstance: 10,
   sceneId: '',
+  sceneAsset: {} as any,
   isLobby: false,
   isFeatured: false,
   locationSetting: {

packages/projects/template-code-project/tests/mocha.env.js → packages/common/src/interfaces/ManifestJson.ts

@@ -1,4 +1,3 @@
-
 /*
 CPAL-1.0 License
 
@@ -24,14 +23,37 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20
 Ethereal Engine. All Rights Reserved.
 */
 
-process.env.APP_ENV = 'test'
-process.env.NODE_ENV = 'test'
-process.env.NODE_TLS_REJECT_UNAUTHORIZED='0'
-
-require("ts-node").register({
-  project: './tsconfig.json',
-  files: true,
-  swc: true
-})
-
-require("fix-esm").register()
+export type ManifestJson = {
+  name: string
+  /**
+   * The version of this project
+   * @example "0.0.1"
+   */
+  version: string
+  /**
+   * The version of the engine this project version is compatible with.
+   * @example "1.0.0"
+   */
+  engineVersion: string
+  /**
+   * A short description of the project
+   * @example "A simple project"
+   */
+  description?: string
+  /**
+   * An optional thumbnail image
+   * @example "https://example.com/thumbnail.jpg"
+   */
+  thumbnail?: string
+  /**
+   * project-relative path for scene GLTF files
+   * @example ["public/scenes/default.gltf"]
+   */
+  scenes?: string[]
+  /**
+   * The dependencies of this project. Specify other projects that are to be installed alongside this one.
+   * @todo
+   * @example { "orgname/reponame": "0.1.2" }
+   */
+  // dependencies?: Record<string, string>
+}

packages/common/src/interfaces/ProjectPackageJsonType.ts

@@ -28,6 +28,7 @@ import { IPackageJson } from 'package-json-type'
 export const DefaultUpdateSchedule = '0 * * * *'
 
 export interface ProjectPackageJsonType extends IPackageJson {
+  /** @deprecated */
   etherealEngine: {
     version: string
     thumbnail?: string

packages/common/src/schema.type.module.ts

@@ -259,8 +259,6 @@ export const projectPath = 'project'
 export const projectsPath = 'projects'
 
 export const assetPath = 'asset'
-/** @deprecated use assetPath instead */
-export const scenePath = assetPath
 
 export const builderInfoPath = 'builder-info'
 

packages/common/src/schemas/assets/asset.schema.ts

@@ -24,8 +24,6 @@ import { Type, getValidator } from '@feathersjs/typebox'
 import { dataValidator, queryValidator } from '../validators'
 
 export const assetPath = 'asset'
-/** @deprecated use assetPath instead */
-export const scenePath = assetPath
 
 export const assetMethods = ['get', 'update', 'create', 'find', 'patch', 'remove'] as const
 
@@ -34,6 +32,7 @@ export const assetSchema = Type.Object(
     id: Type.String(),
     assetURL: Type.String(),
     thumbnailURL: Type.String(),
+    projectName: Type.String(),
     projectId: Type.String(),
     createdAt: Type.String({ format: 'date-time' }),
     updatedAt: Type.String({ format: 'date-time' })
@@ -46,8 +45,9 @@ export interface AssetType extends Static<typeof assetSchema> {}
 export const assetDataSchema = Type.Object(
   {
     id: Type.Optional(Type.String()),
-    name: Type.Optional(Type.String()),
     assetURL: Type.Optional(Type.String()),
+    isScene: Type.Optional(Type.Boolean()),
+    sourceURL: Type.Optional(Type.String()),
     thumbnailURL: Type.Optional(Type.Any()),
     project: Type.Optional(Type.String()),
     projectId: Type.Optional(Type.String())
@@ -68,7 +68,6 @@ export interface AssetUpdate extends Static<typeof assetUpdateSchema> {}
 // Schema for updating existing entries
 export const assetPatchSchema = Type.Object(
   {
-    name: Type.Optional(Type.String()),
     project: Type.Optional(Type.String()),
     assetURL: Type.Optional(Type.String()),
     thumbnailURL: Type.Optional(Type.String())
@@ -89,9 +88,7 @@ export const assetQuerySchema = Type.Intersect(
         projectId: Type.Optional(Type.String()),
         assetURL: Type.Optional(Type.String()),
         internal: Type.Optional(Type.Boolean()),
-        paginate: Type.Optional(Type.Boolean()),
-        directory: Type.Optional(Type.String()),
-        localDirectory: Type.Optional(Type.String())
+        paginate: Type.Optional(Type.Boolean())
       },
       { additionalProperties: false }
     )

packages/common/src/schemas/social/location.schema.ts

@@ -29,6 +29,7 @@ import { getValidator, querySyntax, Type } from '@feathersjs/typebox'
 
 import { OpaqueType } from '@etherealengine/common/src/interfaces/OpaqueType'
 import { TypedString } from '../../types/TypeboxUtils'
+import { assetSchema } from '../assets/asset.schema'
 import { dataValidator, queryValidator } from '../validators'
 import { locationAdminSchema } from './location-admin.schema'
 import { locationAuthorizedUserSchema } from './location-authorized-user.schema'
@@ -55,6 +56,7 @@ export const locationSchema = Type.Object(
     isLobby: Type.Boolean(),
     /** @todo review */
     isFeatured: Type.Boolean(),
+    sceneAsset: Type.Ref(assetSchema),
     maxUsersPerInstance: Type.Number(),
     locationSetting: Type.Ref(locationSettingSchema),
     locationAdmin: Type.Optional(Type.Ref(locationAdminSchema)),

packages/editor/src/components/EditorContainer.tsx

@@ -28,7 +28,6 @@ import { RouterState } from '@etherealengine/client-core/src/common/services/Rou
 import multiLogger from '@etherealengine/common/src/logger'
 import { assetPath } from '@etherealengine/common/src/schema.type.module'
 import { Entity, EntityUUID, getComponent, useComponent } from '@etherealengine/ecs'
-import { Engine } from '@etherealengine/ecs/src/Engine'
 import { useQuery } from '@etherealengine/ecs/src/QueryFunctions'
 import { GLTFComponent } from '@etherealengine/engine/src/gltf/GLTFComponent'
 import { GLTFModifiedState } from '@etherealengine/engine/src/gltf/GLTFDocumentState'
@@ -183,7 +182,7 @@ const onCloseProject = () => {
 }
 
 const onSaveAs = async () => {
-  const { sceneAssetID, projectName, sceneName, rootEntity } = getState(EditorState)
+  const { projectName, sceneName, rootEntity } = getState(EditorState)
   const sceneModified = EditorState.isModified()
   const abortController = new AbortController()
   try {
@@ -193,15 +192,10 @@ const onSaveAs = async () => {
       })
       DialogState.setDialog(null)
       if (result?.name && projectName) {
-        await saveSceneGLTF(sceneAssetID, projectName, result.name, abortController.signal)
+        await saveSceneGLTF(null, projectName, result.name, abortController.signal)
 
         const sourceID = getComponent(rootEntity, SourceComponent)
         getMutableState(GLTFModifiedState)[sourceID].set(none)
-
-        const newSceneData = await Engine.instance.api
-          .service(assetPath)
-          .find({ query: { assetURL: getState(EditorState).scenePath! } })
-        getMutableState(EditorState).scenePath.set(newSceneData.data[0].assetURL as any)
       }
     }
   } catch (error) {
@@ -372,7 +366,6 @@ const tabs = [
 const EditorContainer = () => {
   const { sceneAssetID, sceneName, projectName, scenePath, rootEntity } = useHookstate(getMutableState(EditorState))
   const sceneQuery = useFind(assetPath, { query: { assetURL: scenePath.value ?? '' } }).data
-  const sceneURL = sceneQuery?.[0]?.assetURL
 
   const errorState = useHookstate(getMutableState(EditorErrorState).error)
 
@@ -411,15 +404,14 @@ const EditorContainer = () => {
   useHotkeys(`${cmdOrCtrlString}+s`, () => onSaveScene())
 
   useEffect(() => {
-    if (!sceneURL) return
-    const scene = scenePath.value?.substring(scenePath.value.lastIndexOf('/') + 1)
-    const project = scenePath.value?.substring(scenePath.value.indexOf('/') + 1, scenePath.value.lastIndexOf('/'))
+    const scene = sceneQuery[0]
+    if (!scene) return
 
-    sceneName.set(scene ?? null)
-    projectName.set(project ?? null)
+    projectName.set(scene.projectName)
+    sceneName.set(scene.assetURL.split('/').pop() ?? null)
     sceneAssetID.set(sceneQuery[0].id)
-    return setCurrentEditorScene(sceneURL, sceneQuery[0].id! as EntityUUID)
-  }, [sceneURL])
+    return setCurrentEditorScene(scene.assetURL, scene.id as EntityUUID)
+  }, [sceneQuery[0]?.assetURL])
 
   useEffect(() => {
     return () => {

packages/editor/src/components/assets/ScenesPanel.tsx

@@ -35,7 +35,7 @@ import { ClickAwayListener, IconButton, InputBase, Menu, MenuItem, Paper } from
 
 import { LoadingCircle } from '@etherealengine/client-core/src/components/LoadingCircle'
 import config from '@etherealengine/common/src/config'
-import { AssetType, scenePath } from '@etherealengine/common/src/schema.type.module'
+import { AssetType, assetPath } from '@etherealengine/common/src/schema.type.module'
 import { getComponent } from '@etherealengine/ecs'
 import { getTextureAsync } from '@etherealengine/engine/src/assets/functions/resourceLoaderHooks'
 import { GLTFModifiedState } from '@etherealengine/engine/src/gltf/GLTFDocumentState'
@@ -62,7 +62,7 @@ const logger = multiLogger.child({ component: 'editor:ScenesPanel' })
 export default function ScenesPanel() {
   const { t } = useTranslation()
   const editorState = useHookstate(getMutableState(EditorState))
-  const scenesQuery = useFind(scenePath, { query: { project: editorState.projectName.value } })
+  const scenesQuery = useFind(assetPath, { query: { project: editorState.projectName.value } })
   const scenes = scenesQuery.data
 
   const [isContextMenuOpen, setContextMenuOpen] = useState(false)
@@ -138,7 +138,9 @@ export default function ScenesPanel() {
 
   const finishRenaming = async (id: string) => {
     setRenaming(false)
-    const newData = await renameScene(id, newName)
+    const currentURL = loadedScene!.assetURL
+    const newURL = currentURL.replace(currentURL.split('/').pop()!, newName + '.gltf')
+    const newData = await renameScene(id, newURL)
     if (loadedScene) getMutableState(EditorState).scenePath.set(newData.assetURL)
     setNewName('')
   }

packages/editor/src/components/materials/MaterialLibraryPanelContainer.tsx

@@ -51,6 +51,7 @@ import MaterialLibraryEntry, { MaterialLibraryEntryType } from './MaterialLibrar
 
 import exportMaterialsGLTF from '@etherealengine/engine/src/assets/functions/exportMaterialsGLTF'
 import { SelectionState } from '../../services/SelectionServices'
+import { ImportSettingsState } from '../assets/ImportSettingsPanel'
 
 export default function MaterialLibraryPanel() {
   const srcPath = useState('/mat/material-test')
@@ -131,7 +132,11 @@ export default function MaterialLibraryPanel() {
                 })!) as { [key: string]: any }
                 const blob = [JSON.stringify(gltf)]
                 const file = new File(blob, libraryName)
-                const urls = await Promise.all(uploadProjectFiles(projectName, [file], true).promises)
+                const importSettings = getState(ImportSettingsState)
+                const urls = await Promise.all(
+                  uploadProjectFiles(projectName, [file], [`projects/${projectName}${importSettings.importFolder}`])
+                    .promises
+                )
                 console.log('exported material data to ', ...urls)
               }}
             >

packages/editor/src/components/properties/SceneSettingsEditor.tsx

@@ -80,7 +80,8 @@ export const SceneSettingsEditor: EditorComponentType = (props) => {
     state.uploadingThumbnail.set(true)
     const editorState = getState(EditorState)
     const projectName = editorState.projectName!
-    const { promises } = uploadProjectFiles(projectName, [state.thumbnail.value])
+    const currentSceneDirectory = getState(EditorState).scenePath!.split('/').slice(0, -1).join('/')
+    const { promises } = uploadProjectFiles(projectName, [state.thumbnail.value], [currentSceneDirectory])
     const [[savedThumbnailURL]] = await Promise.all(promises)
     commitProperty(SceneSettingsComponent, 'thumbnailURL')(savedThumbnailURL)
     state.merge({
@@ -119,10 +120,12 @@ export const SceneSettingsEditor: EditorComponentType = (props) => {
     const envmapFilename = `${sceneName}.envmap.ktx2`
     const loadingScreenFilename = `${sceneName}.loadingscreen.ktx2`
 
-    const promises = uploadProjectFiles(projectName, [
-      new File([envmap], envmapFilename),
-      new File([loadingScreen], loadingScreenFilename)
-    ])
+    const currentSceneDirectory = getState(EditorState).scenePath!.split('/').slice(0, -1).join('/')
+    const promises = uploadProjectFiles(
+      projectName,
+      [new File([envmap], envmapFilename), new File([loadingScreen], loadingScreenFilename)],
+      [currentSceneDirectory, currentSceneDirectory]
+    )
 
     const [[envmapURL], [loadingScreenURL]] = await Promise.all(promises.promises)
 

packages/editor/src/functions/assetFunctions.ts

@@ -70,9 +70,16 @@ export const inputFileWithAddToScene = async ({
         if (el.files && el.files.length > 0) {
           const files = Array.from(el.files)
           if (projectName) {
-            uploadedURLs = (await Promise.all(uploadProjectFiles(projectName, files, true).promises)).map(
-              (url) => url[0]
-            )
+            const importSettings = getState(ImportSettingsState)
+            uploadedURLs = (
+              await Promise.all(
+                uploadProjectFiles(
+                  projectName,
+                  files,
+                  files.map(() => `projects/${projectName}${importSettings.importFolder}`)
+                ).promises
+              )
+            ).map((url) => url[0])
             for (const url of uploadedURLs) {
               if (url.endsWith('.gltf') || url.endsWith('.glb') || url.endsWith('.wrm')) {
                 const importSettings = getState(ImportSettingsState)
@@ -123,15 +130,12 @@ export const inputFileWithAddToScene = async ({
     el.remove()
   })
 
-export const uploadProjectFiles = (projectName: string, files: File[], isAsset = false, onProgress?) => {
+export const uploadProjectFiles = (projectName: string, files: File[], paths: string[], onProgress?) => {
   const promises: CancelableUploadPromiseReturnType<string>[] = []
-  const importSettings = getState(ImportSettingsState)
 
-  for (const file of files) {
-    const path = `projects/${projectName}${isAsset ? importSettings.importFolder : ''}`
-    // if (importSettings.LODsEnabled) {
-    //   path = `projects/${projectName}${isAsset ? importSettings.LODFolder : ''}`
-    // }
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i]
+    const path = paths[i]
     promises.push(
       uploadToFeathersService(fileBrowserUploadPath, [file], { fileName: file.name, path, contentType: '' }, onProgress)
     )

packages/editor/src/functions/exportGLTF.ts

@@ -46,6 +46,6 @@ export async function exportRelativeGLTF(entity: Entity, projectName: string, re
   })
   const blob = isGLTF ? [JSON.stringify(gltf, null, 2)] : [gltf]
   const file = new File(blob, relativePath)
-  const urls = await Promise.all(uploadProjectFiles(projectName, [file]).promises)
+  const urls = await Promise.all(uploadProjectFiles(projectName, [file], [`projects/${projectName}`]).promises)
   console.log('exported model data to ', ...urls)
 }