From 6cfe5c05a5e8b42b6400e21d710d32383fdf28c5 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Fri, 1 Dec 2023 18:02:44 -0800 Subject: [PATCH 01/21] seed static resources with project assets improve file properties panel --- .../assets/FileBrowserContentPanel.tsx | 38 +------- .../components/assets/FilePropertiesPanel.tsx | 95 +++++++++++++++++++ .../schemas/media/static-resource.schema.ts | 5 +- .../src/projects/project/project-helper.ts | 50 +++++++++- 4 files changed, 150 insertions(+), 38 deletions(-) create mode 100644 packages/editor/src/components/assets/FilePropertiesPanel.tsx diff --git a/packages/editor/src/components/assets/FileBrowserContentPanel.tsx b/packages/editor/src/components/assets/FileBrowserContentPanel.tsx index da1cd18fe2e..4738a7bae1b 100644 --- a/packages/editor/src/components/assets/FileBrowserContentPanel.tsx +++ b/packages/editor/src/components/assets/FileBrowserContentPanel.tsx @@ -59,9 +59,6 @@ import ViewInArIcon from '@mui/icons-material/ViewInAr' import VolumeUpIcon from '@mui/icons-material/VolumeUp' import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' -import Dialog from '@etherealengine/ui/src/primitives/mui/Dialog' -import DialogTitle from '@etherealengine/ui/src/primitives/mui/DialogTitle' -import Grid from '@etherealengine/ui/src/primitives/mui/Grid' import Typography from '@etherealengine/ui/src/primitives/mui/Typography' import { Breadcrumbs, Link, PopoverPosition, TablePagination } from '@mui/material' @@ -79,6 +76,7 @@ import { AssetSelectionChangePropsType } from './AssetsPreviewPanel' import CompressionPanel from './CompressionPanel' import { FileBrowserItem } from './FileBrowserGrid' import { FileDataType } from './FileDataType' +import { FilePropertiesPanel } from './FilePropertiesPanel' import ImageConvertPanel from './ImageConvertPanel' import styles from './styles.module.scss' @@ -318,7 +316,7 @@ const FileBrowserContentPanel: React.FC = (props) const showUploadAndDownloadButtons = selectedDirectory.value.slice(1).startsWith('projects/') && !['projects', 'projects/'].includes(selectedDirectory.value.slice(1)) - const showBackButton = selectedDirectory.value !== originalPath + const showBackButton = true //selectedDirectory.value !== originalPath const handleDownloadProject = async () => { const url = selectedDirectory.value @@ -562,37 +560,7 @@ const FileBrowserContentPanel: React.FC = (props) )} {openProperties.value && fileProperties.value && ( - openProperties.set(false)} - classes={{ paper: styles.paperDialog }} - > - - {`${fileProperties.value.name} ${fileProperties.value.type == 'folder' ? 'folder' : 'file'} Properties`} - - - - - {t('editor:layout.filebrowser.fileProperties.name')} - - - {t('editor:layout.filebrowser.fileProperties.type')} - - - {t('editor:layout.filebrowser.fileProperties.size')} - - - {t('editor:layout.filebrowser.fileProperties.url')} - - - - {fileProperties.value.name} - {fileProperties.value.type} - {fileProperties.value.size} - {fileProperties.value.url} - - - + )} + fileProperties: State +}) => { + const { openProperties, fileProperties } = props + const { t } = useTranslation() + if (!fileProperties.value) return null + + const modifiableProperties: State = useHookstate( + JSON.parse(JSON.stringify(fileProperties.get(NO_PROXY))) as FileType + ) + + const onChange = useCallback((state: State) => { + return (e) => { + state.set(e.target.value) + } + }, []) + + const staticResource = useHookstate(() => + Engine.instance.api.service(staticResourcePath).find({ + query: { + key: fileProperties.value!.key + } + }) + ) + + const tags: State = useHookstate([]) + + useEffect(() => { + if (!staticResource.promised) { + const resources = staticResource.data.value + resources[0] && tags.set(JSON.parse(JSON.stringify(resources[0].tags))) + } + }, [staticResource]) + + return ( + openProperties.set(false)} + classes={{ paper: styles.paperDialog }} + > + + {`${fileProperties.value.name} ${fileProperties.value.type == 'folder' ? 'folder' : 'file'} Properties`} + +
+ + + + {t('editor:layout.filebrowser.fileProperties.type')} + {t('editor:layout.filebrowser.fileProperties.size')} + + + {modifiableProperties.type.value} + {modifiableProperties.size.value} + + + + {tags.value.map((tag, index) => ( + + ))} + +
+ ) +} diff --git a/packages/engine/src/schemas/media/static-resource.schema.ts b/packages/engine/src/schemas/media/static-resource.schema.ts index 5faaf96131e..889662feb81 100755 --- a/packages/engine/src/schemas/media/static-resource.schema.ts +++ b/packages/engine/src/schemas/media/static-resource.schema.ts @@ -91,7 +91,7 @@ export const staticResourceQueryProperties = Type.Pick(staticResourceSchema, [ 'driver', 'attribution', 'licensing', - // 'tags', + 'tags', 'url' // 'stats' ]) @@ -103,6 +103,9 @@ export const staticResourceQuerySchema = Type.Intersect( }, mimeType: { $like: Type.String() + }, + tags: { + $like: Type.String() } }), // Add additional query properties here diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index 191dc4a25e8..001c3bbeabc 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -42,7 +42,10 @@ import fs from 'fs' import { PUBLIC_SIGNED_REGEX } from '@etherealengine/common/src/constants/GitHubConstants' import { ProjectPackageJsonType } from '@etherealengine/common/src/interfaces/ProjectPackageJsonType' import { processFileName } from '@etherealengine/common/src/utils/processFileName' +import { AssetLoader } from '@etherealengine/engine/src/assets/classes/AssetLoader' +import { AssetClass } from '@etherealengine/engine/src/assets/enum/AssetClass' import { apiJobPath } from '@etherealengine/engine/src/schemas/cluster/api-job.schema' +import { staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' import { ProjectBuilderTagsType } from '@etherealengine/engine/src/schemas/projects/project-builder-tags.schema' import { ProjectCheckSourceDestinationMatchType } from '@etherealengine/engine/src/schemas/projects/project-check-source-destination-match.schema' import { ProjectCheckUnfetchedCommitType } from '@etherealengine/engine/src/schemas/projects/project-check-unfetched-commit.schema' @@ -68,6 +71,7 @@ import { getCacheDomain } from '../../media/storageprovider/getCacheDomain' import { getCachedURL } from '../../media/storageprovider/getCachedURL' import { getStorageProvider } from '../../media/storageprovider/storageprovider' import { getFileKeysRecursive } from '../../media/storageprovider/storageProviderUtils' +import { createStaticResourceHash } from '../../media/upload-asset/upload-asset.service' import logger from '../../ServerLogger' import { ServerState } from '../../ServerState' import { BUILDER_CHART_REGEX } from '../../setting/helm-setting/helm-setting' @@ -1649,18 +1653,60 @@ export const uploadLocalProjectToProvider = async ( const files = getFilesRecursive(projectRootPath) const filtered = files.filter((file) => !file.includes(`projects/${projectName}/.git/`)) const results = [] as (string | null)[] + const resourceKey = (key, hash) => `${key}#${hash}` + const currentProjectResources = await app + .service(staticResourcePath) + .find({ + query: { + project: projectName + } + }) + .then((res) => { + const set = new Set() + for (const item of res.data) { + set.add(resourceKey(item.key, item.hash)) + } + return set + }) for (const file of filtered) { try { const fileResult = fs.readFileSync(file) const filePathRelative = processFileName(file.slice(projectRootPath.length)) + const contentType = getContentType(file) + const key = `projects/${projectName}${filePathRelative}` + const url = getCachedURL(key, getCacheDomain(storageProvider)) await storageProvider.putObject( { Body: fileResult, - ContentType: getContentType(file), - Key: `projects/${projectName}${filePathRelative}` + ContentType: contentType, + Key: key }, { isDirectory: false } ) + const staticResourceClasses = [ + AssetClass.Audio, + AssetClass.Image, + AssetClass.Model, + AssetClass.Video, + AssetClass.Volumetric + ] + const thisFileClass = AssetLoader.getAssetClass(file) + if (filePathRelative.startsWith('/assets/') && staticResourceClasses.includes(thisFileClass)) { + const hash = createStaticResourceHash(fileResult, { mimeType: contentType, assetURL: key }) + if (currentProjectResources.has(resourceKey(key, hash))) { + logger.info(`Skipping upload of static resource of class ${thisFileClass}: "${key}"`) + } else { + await app.service(staticResourcePath).create({ + key: `projects/${projectName}${filePathRelative}`, + project: projectName, + hash, + url, + mimeType: contentType, + tags: [thisFileClass] + }) + logger.info(`Uploaded static resource of class ${thisFileClass}: "${key}"`) + } + } results.push(getCachedURL(`projects/${projectName}${filePathRelative}`, cacheDomain)) } catch (e) { logger.error(e) From 6ef7c75da346193cfa7e219ab05d2f00ca9cf4ca Mon Sep 17 00:00:00 2001 From: David Gordon Date: Fri, 1 Dec 2023 18:02:53 -0800 Subject: [PATCH 02/21] licensing --- .../components/assets/FilePropertiesPanel.tsx | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/editor/src/components/assets/FilePropertiesPanel.tsx b/packages/editor/src/components/assets/FilePropertiesPanel.tsx index 75573540b8d..993a25f43c0 100644 --- a/packages/editor/src/components/assets/FilePropertiesPanel.tsx +++ b/packages/editor/src/components/assets/FilePropertiesPanel.tsx @@ -1,3 +1,28 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + import { Dialog, DialogTitle, Grid, Typography } from '@mui/material' import React, { useCallback, useEffect } from 'react' From 51e37a886602894093651c7d730c1ce2c57a2763 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Fri, 8 Dec 2023 09:59:56 -0800 Subject: [PATCH 03/21] checkpoint --- .../components/assets/FilePropertiesPanel.tsx | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/editor/src/components/assets/FilePropertiesPanel.tsx b/packages/editor/src/components/assets/FilePropertiesPanel.tsx index 993a25f43c0..9e5ff6cf081 100644 --- a/packages/editor/src/components/assets/FilePropertiesPanel.tsx +++ b/packages/editor/src/components/assets/FilePropertiesPanel.tsx @@ -34,6 +34,7 @@ import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' import { FileType } from './FileBrowserContentPanel' import styles from './styles.module.scss' +import { FileBrowserService } from '@etherealengine/client-core/src/common/services/FileBrowserService' import { staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' import { Button } from '../inputs/Button' @@ -49,12 +50,29 @@ export const FilePropertiesPanel = (props: { JSON.parse(JSON.stringify(fileProperties.get(NO_PROXY))) as FileType ) + const isModified = useHookstate(false) + const onChange = useCallback((state: State) => { + isModified.set(true) return (e) => { state.set(e.target.value) } }, []) + const onSaveChanges = useCallback(() => { + if (isModified.value) { + if (modifiableProperties.name.value != fileProperties.value!.name) { + FileBrowserService.moveContent( + fileProperties.value!.name, + modifiableProperties.name.value, + fileProperties.value!.path, + fileProperties.value!.path + ) + } + isModified.set(false) + } + }, []) + const staticResource = useHookstate(() => Engine.instance.api.service(staticResourcePath).find({ query: { @@ -114,6 +132,11 @@ export const FilePropertiesPanel = (props: { value={tags[index].value} /> ))} + {isModified.value && ( + + )} ) From e75031bef5449946b838af38304c4703aae0198d Mon Sep 17 00:00:00 2001 From: David Gordon Date: Wed, 20 Dec 2023 09:57:25 -0800 Subject: [PATCH 04/21] generate .sql file for project static resources --- packages/server-core/package.json | 1 + packages/server-core/src/media/services.ts | 12 ++- .../project-resource.service.ts | 72 ++++++++++++++++++ .../src/projects/project/project-helper.ts | 76 +++++++++++++------ .../src/projects/project/project.class.ts | 7 +- 5 files changed, 140 insertions(+), 28 deletions(-) create mode 100644 packages/server-core/src/media/static-resource/project-resource.service.ts diff --git a/packages/server-core/package.json b/packages/server-core/package.json index f2ffcebff4c..f940838a806 100755 --- a/packages/server-core/package.json +++ b/packages/server-core/package.json @@ -101,6 +101,7 @@ "multer": "1.4.5-lts.1", "mysql": "2.18.1", "mysql2": "3.2.0", + "mysqldump": "3.2.0", "nanoid": "3.3.4", "node-fetch": "2.6.9", "nodemailer-smtp-transport": "^2.7.4", diff --git a/packages/server-core/src/media/services.ts b/packages/server-core/src/media/services.ts index a1618f087b1..9097ad9b6da 100755 --- a/packages/server-core/src/media/services.ts +++ b/packages/server-core/src/media/services.ts @@ -28,7 +28,17 @@ import FileBrowser from './file-browser/file-browser' import OEmbed from './oembed/oembed' import Archiver from './recursive-archiver/archiver' import StaticResourceFilters from './static-resource-filters/static-resource-filters' +import ProjectResource from './static-resource/project-resource.service' import StaticResource from './static-resource/static-resource' import Upload from './upload-asset/upload-asset.service' -export default [StaticResource, StaticResourceFilters, FileBrowser, FileBrowserUpload, OEmbed, Upload, Archiver] +export default [ + ProjectResource, + StaticResource, + StaticResourceFilters, + FileBrowser, + FileBrowserUpload, + OEmbed, + Upload, + Archiver +] diff --git a/packages/server-core/src/media/static-resource/project-resource.service.ts b/packages/server-core/src/media/static-resource/project-resource.service.ts new file mode 100644 index 00000000000..71df7bc9f83 --- /dev/null +++ b/packages/server-core/src/media/static-resource/project-resource.service.ts @@ -0,0 +1,72 @@ +import { Application } from '@feathersjs/koa' + +import { exec } from 'child_process' +import mysqldump from 'mysqldump' +import { promisify } from 'util' + +export const projectResourcesPath = 'project-resources' + +export type CreateProjectResourceParams = { + project: string +} + +declare module '@etherealengine/common/declarations' { + interface ServiceTypes { + [projectResourcesPath]: any + } +} + +async function createProjectResource({ project }: CreateProjectResourceParams) { + const tableName = `project_${project.replaceAll('-', '_')}` + const user = process.env.MYSQL_USER + const password = process.env.MYSQL_PASSWORD + const host = process.env.MYSQL_HOST + + const cmdPrefix = `mysql -h ${host} -u ${user} -p"${password}"` + + const dropTableIfExistsCmd = `${cmdPrefix} -e 'USE etherealengine; DROP TABLE IF EXISTS ${tableName};'` + const createProjectResourceTableCmd = `${cmdPrefix} -e 'USE etherealengine; CREATE TABLE ${tableName} AS SELECT * FROM \`static-resource\` WHERE project = "${project}";'` + const dropProjectResourceTableCmd = `${cmdPrefix} -e "USE etherealengine; DROP TABLE ${tableName};"` + + const execPromise = promisify(exec) + + function executeCmd(command) { + return execPromise(command) + .then(({ stdout }) => { + console.log(`stdout: ${stdout}`) + }) + .catch((error) => { + console.error(`Error: ${error}`) + throw error // Rethrow the error for upstream catch handling + }) + } + + await executeCmd(dropTableIfExistsCmd) + + await executeCmd(createProjectResourceTableCmd) + + const connectionConfig = { + host: 'localhost', + user: user!, + password: password!, + database: 'etherealengine' + } + + const dumpToFile = `../projects/projects/${project}/resources.sql` + + await mysqldump({ + connection: connectionConfig, + dump: { + tables: [tableName] + }, + dumpToFile + }) + + await executeCmd(dropProjectResourceTableCmd) +} + +export default (app: Application): void => { + app.use(projectResourcesPath, { + create: createProjectResource + }) +} diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index 5bc23e76315..0c4f2dc126b 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -67,6 +67,7 @@ import { v4 } from 'uuid' import { Application } from '../../../declarations' import config from '../../appconfig' import { getPodsData } from '../../cluster/pods/pods-helper' +import { projectResourcesPath } from '../../media/static-resource/project-resource.service' import { getCacheDomain } from '../../media/storageprovider/getCacheDomain' import { getCachedURL } from '../../media/storageprovider/getCachedURL' import { getStorageProvider } from '../../media/storageprovider/storageprovider' @@ -1657,6 +1658,9 @@ export const uploadLocalProjectToProvider = async ( // upload new files to storage provider const projectRootPath = path.resolve(projectsRootFolder, projectName) + const resourceDBPath = path.join(projectRootPath, 'resources.sql') + const hasResourceDB = fs.existsSync(resourceDBPath) + const files = getFilesRecursive(projectRootPath) const filtered = files.filter((file) => !file.includes(`projects/${projectName}/.git/`)) const results = [] as (string | null)[] @@ -1675,6 +1679,28 @@ export const uploadLocalProjectToProvider = async ( } return set }) + + if (hasResourceDB) { + //if we have a resources.sql file, use it to populate static-resource table + const promiseExec = promisify(exec) + const tableName = `project_${projectName.replaceAll('-', '_')}` + const user = process.env.MYSQL_USER + const password = process.env.MYSQL_PASSWORD + const host = process.env.MYSQL_HOST + + const cmdPrefix = `mysql -h ${host} -u ${user} -p"${password}" etherealengine` + + const runResourceCmd = `${cmdPrefix} < ${resourceDBPath}` + const copyIntoStaticResourcesCmd = `${cmdPrefix} -e 'CREATE TABLE IF NOT EXISTS \`${tableName}\` LIKE \`static-resource\`; INSERT IGNORE INTO \`static-resource\` SELECT * FROM \`${tableName}\`;'` + const dropTableCmd = `${cmdPrefix} -e "DROP TABLE ${tableName};"` + + await promiseExec(runResourceCmd) + await promiseExec(copyIntoStaticResourcesCmd) + await promiseExec(dropTableCmd) + + console.log(`Finished populating static-resource table for project ${projectName}`) + } + for (const file of filtered) { try { const fileResult = fs.readFileSync(file) @@ -1690,28 +1716,31 @@ export const uploadLocalProjectToProvider = async ( }, { isDirectory: false } ) - const staticResourceClasses = [ - AssetClass.Audio, - AssetClass.Image, - AssetClass.Model, - AssetClass.Video, - AssetClass.Volumetric - ] - const thisFileClass = AssetLoader.getAssetClass(file) - if (filePathRelative.startsWith('/assets/') && staticResourceClasses.includes(thisFileClass)) { - const hash = createStaticResourceHash(fileResult, { mimeType: contentType, assetURL: key }) - if (currentProjectResources.has(resourceKey(key, hash))) { - logger.info(`Skipping upload of static resource of class ${thisFileClass}: "${key}"`) - } else { - await app.service(staticResourcePath).create({ - key: `projects/${projectName}${filePathRelative}`, - project: projectName, - hash, - url, - mimeType: contentType, - tags: [thisFileClass] - }) - logger.info(`Uploaded static resource of class ${thisFileClass}: "${key}"`) + if (!hasResourceDB) { + //otherwise, upload the files into static resources individually + const staticResourceClasses = [ + AssetClass.Audio, + AssetClass.Image, + AssetClass.Model, + AssetClass.Video, + AssetClass.Volumetric + ] + const thisFileClass = AssetLoader.getAssetClass(file) + if (filePathRelative.startsWith('/assets/') && staticResourceClasses.includes(thisFileClass)) { + const hash = createStaticResourceHash(fileResult, { mimeType: contentType, assetURL: key }) + if (currentProjectResources.has(resourceKey(key, hash))) { + logger.info(`Skipping upload of static resource of class ${thisFileClass}: "${key}"`) + } else { + await app.service(staticResourcePath).create({ + key: `projects/${projectName}${filePathRelative}`, + project: projectName, + hash, + url, + mimeType: contentType, + tags: [thisFileClass] + }) + logger.info(`Uploaded static resource of class ${thisFileClass}: "${key}"`) + } } } results.push(getCachedURL(`projects/${projectName}${filePathRelative}`, cacheDomain)) @@ -1720,6 +1749,9 @@ export const uploadLocalProjectToProvider = async ( results.push(null) } } + if (!hasResourceDB) { + await app.service(projectResourcesPath).create({ project: projectName }) + } logger.info(`uploadLocalProjectToProvider for project "${projectName}" ended at "${new Date()}".`) return results.filter((success) => !!success) as string[] } diff --git a/packages/server-core/src/projects/project/project.class.ts b/packages/server-core/src/projects/project/project.class.ts index 92aba121b2d..0338b6f4820 100644 --- a/packages/server-core/src/projects/project/project.class.ts +++ b/packages/server-core/src/projects/project/project.class.ts @@ -144,12 +144,10 @@ export class ProjectService dirent.isDirectory()) .map((dirent) => dirent.name) - const promises: Promise[] = [] - for (const projectName of locallyInstalledProjects) { if (!data.find((e) => e.name === projectName)) { try { - promises.push(this._seedProject(projectName)) + await this._seedProject(projectName) } catch (e) { logger.error(e) } @@ -161,10 +159,9 @@ export class ProjectService Date: Wed, 20 Dec 2023 09:57:49 -0800 Subject: [PATCH 05/21] working tagging UI in File Browser --- .../assets/FileBrowserContentPanel.tsx | 2 ++ .../components/assets/FilePropertiesPanel.tsx | 14 ++++++++--- .../src/functions/saveProjectResources.ts | 9 +++++++ .../project-resource.service.ts | 25 +++++++++++++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 packages/editor/src/functions/saveProjectResources.ts diff --git a/packages/editor/src/components/assets/FileBrowserContentPanel.tsx b/packages/editor/src/components/assets/FileBrowserContentPanel.tsx index a13b1075503..255f78442e1 100644 --- a/packages/editor/src/components/assets/FileBrowserContentPanel.tsx +++ b/packages/editor/src/components/assets/FileBrowserContentPanel.tsx @@ -69,6 +69,7 @@ import { archiverPath } from '@etherealengine/engine/src/schemas/media/archiver. import { fileBrowserUploadPath } from '@etherealengine/engine/src/schemas/media/file-browser-upload.schema' import { SupportedFileTypes } from '../../constants/AssetTypes' import { inputFileWithAddToScene } from '../../functions/assetFunctions' +import { saveProjectResources } from '../../functions/saveProjectResources' import { bytesToSize, unique } from '../../functions/utils' import { Button } from '../inputs/Button' import StringInput from '../inputs/StringInput' @@ -532,6 +533,7 @@ const FileBrowserContentPanel: React.FC = (props) variant="body2" /> )} +
diff --git a/packages/editor/src/components/assets/FilePropertiesPanel.tsx b/packages/editor/src/components/assets/FilePropertiesPanel.tsx index 9e5ff6cf081..271c0db07ed 100644 --- a/packages/editor/src/components/assets/FilePropertiesPanel.tsx +++ b/packages/editor/src/components/assets/FilePropertiesPanel.tsx @@ -35,7 +35,7 @@ import { FileType } from './FileBrowserContentPanel' import styles from './styles.module.scss' import { FileBrowserService } from '@etherealengine/client-core/src/common/services/FileBrowserService' -import { staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' +import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' import { Button } from '../inputs/Button' export const FilePropertiesPanel = (props: { @@ -69,6 +69,11 @@ export const FilePropertiesPanel = (props: { fileProperties.value!.path ) } + if (tags.value.length > 0) { + Engine.instance.api.service(staticResourcePath).patch(id.value, { + tags: tags.value + }) + } isModified.set(false) } }, []) @@ -82,11 +87,12 @@ export const FilePropertiesPanel = (props: { ) const tags: State = useHookstate([]) - + const id: State = useHookstate('') useEffect(() => { if (!staticResource.promised) { - const resources = staticResource.data.value - resources[0] && tags.set(JSON.parse(JSON.stringify(resources[0].tags))) + const resources = JSON.parse(JSON.stringify(staticResource.data.value[0])) as StaticResourceType + resources && tags.set(resources.tags) + resources && id.set(resources.id) } }, [staticResource]) diff --git a/packages/editor/src/functions/saveProjectResources.ts b/packages/editor/src/functions/saveProjectResources.ts new file mode 100644 index 00000000000..9e511470428 --- /dev/null +++ b/packages/editor/src/functions/saveProjectResources.ts @@ -0,0 +1,9 @@ +import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' +import { getState } from '@etherealengine/hyperflux' +import { EditorState } from '../services/EditorServices' + +export async function saveProjectResources() { + const project = getState(EditorState).projectName! + + await Engine.instance.api.service('project-resources').create({ project }) +} diff --git a/packages/server-core/src/media/static-resource/project-resource.service.ts b/packages/server-core/src/media/static-resource/project-resource.service.ts index 71df7bc9f83..4fe12f06786 100644 --- a/packages/server-core/src/media/static-resource/project-resource.service.ts +++ b/packages/server-core/src/media/static-resource/project-resource.service.ts @@ -1,3 +1,28 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + import { Application } from '@feathersjs/koa' import { exec } from 'child_process' From a163be68332acaacae8611c90aa4b188356814d6 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Wed, 20 Dec 2023 09:58:04 -0800 Subject: [PATCH 06/21] licensing --- .../src/functions/saveProjectResources.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/editor/src/functions/saveProjectResources.ts b/packages/editor/src/functions/saveProjectResources.ts index 9e511470428..7aa65569912 100644 --- a/packages/editor/src/functions/saveProjectResources.ts +++ b/packages/editor/src/functions/saveProjectResources.ts @@ -1,3 +1,28 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' import { getState } from '@etherealengine/hyperflux' import { EditorState } from '../services/EditorServices' From 79acce9567b033c81800db5834d48d831a69078a Mon Sep 17 00:00:00 2001 From: David Gordon Date: Wed, 20 Dec 2023 15:53:26 -0800 Subject: [PATCH 07/21] switch from .sql to .json resources file --- .../project-resource.service.ts | 68 +++++-------------- .../src/projects/project/project-helper.ts | 36 +++++----- 2 files changed, 34 insertions(+), 70 deletions(-) diff --git a/packages/server-core/src/media/static-resource/project-resource.service.ts b/packages/server-core/src/media/static-resource/project-resource.service.ts index 4fe12f06786..9e548f40f08 100644 --- a/packages/server-core/src/media/static-resource/project-resource.service.ts +++ b/packages/server-core/src/media/static-resource/project-resource.service.ts @@ -23,11 +23,10 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ +import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' import { Application } from '@feathersjs/koa' -import { exec } from 'child_process' -import mysqldump from 'mysqldump' -import { promisify } from 'util' +import { getStorageProvider } from '../storageprovider/storageprovider' export const projectResourcesPath = 'project-resources' @@ -41,57 +40,24 @@ declare module '@etherealengine/common/declarations' { } } -async function createProjectResource({ project }: CreateProjectResourceParams) { - const tableName = `project_${project.replaceAll('-', '_')}` - const user = process.env.MYSQL_USER - const password = process.env.MYSQL_PASSWORD - const host = process.env.MYSQL_HOST - - const cmdPrefix = `mysql -h ${host} -u ${user} -p"${password}"` - - const dropTableIfExistsCmd = `${cmdPrefix} -e 'USE etherealengine; DROP TABLE IF EXISTS ${tableName};'` - const createProjectResourceTableCmd = `${cmdPrefix} -e 'USE etherealengine; CREATE TABLE ${tableName} AS SELECT * FROM \`static-resource\` WHERE project = "${project}";'` - const dropProjectResourceTableCmd = `${cmdPrefix} -e "USE etherealengine; DROP TABLE ${tableName};"` - - const execPromise = promisify(exec) - - function executeCmd(command) { - return execPromise(command) - .then(({ stdout }) => { - console.log(`stdout: ${stdout}`) - }) - .catch((error) => { - console.error(`Error: ${error}`) - throw error // Rethrow the error for upstream catch handling - }) +const createProjectResource = + (app: Application) => + async ({ project }: CreateProjectResourceParams) => { + const resources = await app.service(staticResourcePath).find({ + query: { project } + }) + const data: StaticResourceType[] = resources.data + const storageProvider = getStorageProvider() + + await storageProvider.putObject({ + Body: Buffer.from(JSON.stringify(data)), + ContentType: 'application/json', + Key: `projects/${project}/resources.json` + }) } - await executeCmd(dropTableIfExistsCmd) - - await executeCmd(createProjectResourceTableCmd) - - const connectionConfig = { - host: 'localhost', - user: user!, - password: password!, - database: 'etherealengine' - } - - const dumpToFile = `../projects/projects/${project}/resources.sql` - - await mysqldump({ - connection: connectionConfig, - dump: { - tables: [tableName] - }, - dumpToFile - }) - - await executeCmd(dropProjectResourceTableCmd) -} - export default (app: Application): void => { app.use(projectResourcesPath, { - create: createProjectResource + create: createProjectResource(app) }) } diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index 0c4f2dc126b..d33dd63fd1d 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -45,7 +45,7 @@ import { processFileName } from '@etherealengine/common/src/utils/processFileNam import { AssetLoader } from '@etherealengine/engine/src/assets/classes/AssetLoader' import { AssetClass } from '@etherealengine/engine/src/assets/enum/AssetClass' import { apiJobPath } from '@etherealengine/engine/src/schemas/cluster/api-job.schema' -import { staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' +import { staticResourcePath, StaticResourceType } from '@etherealengine/engine/src/schemas/media/static-resource.schema' import { ProjectBuilderTagsType } from '@etherealengine/engine/src/schemas/projects/project-builder-tags.schema' import { ProjectCheckSourceDestinationMatchType } from '@etherealengine/engine/src/schemas/projects/project-check-source-destination-match.schema' import { ProjectCheckUnfetchedCommitType } from '@etherealengine/engine/src/schemas/projects/project-check-unfetched-commit.schema' @@ -1658,7 +1658,7 @@ export const uploadLocalProjectToProvider = async ( // upload new files to storage provider const projectRootPath = path.resolve(projectsRootFolder, projectName) - const resourceDBPath = path.join(projectRootPath, 'resources.sql') + const resourceDBPath = path.join(projectRootPath, 'resources.json') const hasResourceDB = fs.existsSync(resourceDBPath) const files = getFilesRecursive(projectRootPath) @@ -1682,23 +1682,21 @@ export const uploadLocalProjectToProvider = async ( if (hasResourceDB) { //if we have a resources.sql file, use it to populate static-resource table - const promiseExec = promisify(exec) - const tableName = `project_${projectName.replaceAll('-', '_')}` - const user = process.env.MYSQL_USER - const password = process.env.MYSQL_PASSWORD - const host = process.env.MYSQL_HOST - - const cmdPrefix = `mysql -h ${host} -u ${user} -p"${password}" etherealengine` - - const runResourceCmd = `${cmdPrefix} < ${resourceDBPath}` - const copyIntoStaticResourcesCmd = `${cmdPrefix} -e 'CREATE TABLE IF NOT EXISTS \`${tableName}\` LIKE \`static-resource\`; INSERT IGNORE INTO \`static-resource\` SELECT * FROM \`${tableName}\`;'` - const dropTableCmd = `${cmdPrefix} -e "DROP TABLE ${tableName};"` - - await promiseExec(runResourceCmd) - await promiseExec(copyIntoStaticResourcesCmd) - await promiseExec(dropTableCmd) - - console.log(`Finished populating static-resource table for project ${projectName}`) + const manifest: StaticResourceType[] = JSON.parse(fs.readFileSync(resourceDBPath).toString()) + + for (const item of manifest) { + const key = `projects/${projectName}${item.key}` + const url = getCachedURL(key, cacheDomain) + await app.service(staticResourcePath).create({ + key, + project: projectName, + hash: item.hash, + url, + mimeType: item.mimeType, + tags: item.tags + }) + logger.info(`Uploaded static resource ${key} from resources.json`) + } } for (const file of filtered) { From ff2a8ac976e18465b46e7951360f4024bfa693d4 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Wed, 20 Dec 2023 15:59:04 -0800 Subject: [PATCH 08/21] re-async _fetchDevLocalProjects --- packages/server-core/src/projects/project/project.class.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/server-core/src/projects/project/project.class.ts b/packages/server-core/src/projects/project/project.class.ts index 0338b6f4820..0747bbdfc6f 100644 --- a/packages/server-core/src/projects/project/project.class.ts +++ b/packages/server-core/src/projects/project/project.class.ts @@ -144,10 +144,12 @@ export class ProjectService dirent.isDirectory()) .map((dirent) => dirent.name) + const promises: Promise[] = [] + for (const projectName of locallyInstalledProjects) { if (!data.find((e) => e.name === projectName)) { try { - await this._seedProject(projectName) + promises.push(this._seedProject(projectName)) } catch (e) { logger.error(e) } @@ -159,7 +161,7 @@ export class ProjectService Date: Wed, 20 Dec 2023 16:50:40 -0800 Subject: [PATCH 09/21] remove pagination fully restore fetch projects --- .../project-resource.service.ts | 23 ++++++++++++++----- .../src/projects/project/project.class.ts | 2 ++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/server-core/src/media/static-resource/project-resource.service.ts b/packages/server-core/src/media/static-resource/project-resource.service.ts index 9e548f40f08..89dfabc0aa5 100644 --- a/packages/server-core/src/media/static-resource/project-resource.service.ts +++ b/packages/server-core/src/media/static-resource/project-resource.service.ts @@ -26,8 +26,13 @@ Ethereal Engine. All Rights Reserved. import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' import { Application } from '@feathersjs/koa' +import { isDev } from '@etherealengine/common/src/config' import { getStorageProvider } from '../storageprovider/storageprovider' +import fs from 'fs' +import path from 'path' +import { projectsRootFolder } from '../file-browser/file-browser.class' + export const projectResourcesPath = 'project-resources' export type CreateProjectResourceParams = { @@ -43,17 +48,23 @@ declare module '@etherealengine/common/declarations' { const createProjectResource = (app: Application) => async ({ project }: CreateProjectResourceParams) => { - const resources = await app.service(staticResourcePath).find({ - query: { project } + const resources: StaticResourceType[] = await app.service(staticResourcePath).find({ + query: { project }, + paginate: false }) - const data: StaticResourceType[] = resources.data const storageProvider = getStorageProvider() - + const key = `projects/${project}/resources.json` await storageProvider.putObject({ - Body: Buffer.from(JSON.stringify(data)), + Body: Buffer.from(JSON.stringify(resources)), ContentType: 'application/json', - Key: `projects/${project}/resources.json` + Key: key }) + if (!!isDev) { + const filePath = path.resolve(projectsRootFolder, key) + const dirName = path.dirname(filePath) + fs.mkdirSync(dirName, { recursive: true }) + fs.writeFileSync(filePath, JSON.stringify(resources)) + } } export default (app: Application): void => { diff --git a/packages/server-core/src/projects/project/project.class.ts b/packages/server-core/src/projects/project/project.class.ts index 0747bbdfc6f..8f9ba92e2f3 100644 --- a/packages/server-core/src/projects/project/project.class.ts +++ b/packages/server-core/src/projects/project/project.class.ts @@ -164,6 +164,8 @@ export class ProjectService Date: Wed, 20 Dec 2023 16:54:06 -0800 Subject: [PATCH 10/21] wipe URL from manifest files fully import all resource fields --- .../src/media/static-resource/project-resource.service.ts | 4 ++++ .../server-core/src/projects/project/project-helper.ts | 7 ++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/server-core/src/media/static-resource/project-resource.service.ts b/packages/server-core/src/media/static-resource/project-resource.service.ts index 89dfabc0aa5..400cbde465e 100644 --- a/packages/server-core/src/media/static-resource/project-resource.service.ts +++ b/packages/server-core/src/media/static-resource/project-resource.service.ts @@ -52,6 +52,10 @@ const createProjectResource = query: { project }, paginate: false }) + //wipe URLs from resources + for (const resource of resources) { + resource.url = '' + } const storageProvider = getStorageProvider() const key = `projects/${project}/resources.json` await storageProvider.putObject({ diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index d33dd63fd1d..502ced0bc1d 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -1688,12 +1688,9 @@ export const uploadLocalProjectToProvider = async ( const key = `projects/${projectName}${item.key}` const url = getCachedURL(key, cacheDomain) await app.service(staticResourcePath).create({ + ...item, key, - project: projectName, - hash: item.hash, - url, - mimeType: item.mimeType, - tags: item.tags + url }) logger.info(`Uploaded static resource ${key} from resources.json`) } From b0dd65492af70dd5681ea69290e9c509c9169c9a Mon Sep 17 00:00:00 2001 From: David Gordon Date: Wed, 20 Dec 2023 17:14:30 -0800 Subject: [PATCH 11/21] add check for existing id --- .../src/projects/project/project-helper.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index 502ced0bc1d..6b0698e3037 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -1665,7 +1665,7 @@ export const uploadLocalProjectToProvider = async ( const filtered = files.filter((file) => !file.includes(`projects/${projectName}/.git/`)) const results = [] as (string | null)[] const resourceKey = (key, hash) => `${key}#${hash}` - const currentProjectResources = await app + const { existingContentSet, existingIdSet } = await app .service(staticResourcePath) .find({ query: { @@ -1673,11 +1673,13 @@ export const uploadLocalProjectToProvider = async ( } }) .then((res) => { - const set = new Set() + const existingContentSet = new Set() + const existingIdSet = new Set() for (const item of res.data) { - set.add(resourceKey(item.key, item.hash)) + existingContentSet.add(resourceKey(item.key, item.hash)) + existingIdSet.add(item.id) } - return set + return { existingContentSet, existingIdSet } }) if (hasResourceDB) { @@ -1685,6 +1687,7 @@ export const uploadLocalProjectToProvider = async ( const manifest: StaticResourceType[] = JSON.parse(fs.readFileSync(resourceDBPath).toString()) for (const item of manifest) { + if (existingIdSet.has(item.id)) continue const key = `projects/${projectName}${item.key}` const url = getCachedURL(key, cacheDomain) await app.service(staticResourcePath).create({ @@ -1723,7 +1726,7 @@ export const uploadLocalProjectToProvider = async ( const thisFileClass = AssetLoader.getAssetClass(file) if (filePathRelative.startsWith('/assets/') && staticResourceClasses.includes(thisFileClass)) { const hash = createStaticResourceHash(fileResult, { mimeType: contentType, assetURL: key }) - if (currentProjectResources.has(resourceKey(key, hash))) { + if (existingContentSet.has(resourceKey(key, hash))) { logger.info(`Skipping upload of static resource of class ${thisFileClass}: "${key}"`) } else { await app.service(staticResourcePath).create({ From 46599b8dd15b47819337c6552d05d834d05dd6b3 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Thu, 21 Dec 2023 10:33:18 -0800 Subject: [PATCH 12/21] fix key corruption add localization add "remove tag" button remove null fields when serializing resources.json correctly handle non-resource files in file properties panel --- packages/client-core/i18n/en/editor.json | 6 +- .../components/assets/FilePropertiesPanel.tsx | 120 +++++++++++------- .../project-resource.service.ts | 5 + .../src/projects/project/project-helper.ts | 6 +- 4 files changed, 88 insertions(+), 49 deletions(-) diff --git a/packages/client-core/i18n/en/editor.json b/packages/client-core/i18n/en/editor.json index d6a85e55ff8..9fd2dde8776 100755 --- a/packages/client-core/i18n/en/editor.json +++ b/packages/client-core/i18n/en/editor.json @@ -953,7 +953,11 @@ "name": "Name:", "type": "Type:", "size": "Size:", - "url": "URL:" + "url": "URL:", + "attribution": "Attribution:", + "licensing": "Licensing:", + "tag": "Tag:", + "addTag": "Add Tag" } } }, diff --git a/packages/editor/src/components/assets/FilePropertiesPanel.tsx b/packages/editor/src/components/assets/FilePropertiesPanel.tsx index 271c0db07ed..f8eafb3315d 100644 --- a/packages/editor/src/components/assets/FilePropertiesPanel.tsx +++ b/packages/editor/src/components/assets/FilePropertiesPanel.tsx @@ -34,7 +34,6 @@ import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' import { FileType } from './FileBrowserContentPanel' import styles from './styles.module.scss' -import { FileBrowserService } from '@etherealengine/client-core/src/common/services/FileBrowserService' import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' import { Button } from '../inputs/Button' @@ -61,38 +60,39 @@ export const FilePropertiesPanel = (props: { const onSaveChanges = useCallback(() => { if (isModified.value) { - if (modifiableProperties.name.value != fileProperties.value!.name) { - FileBrowserService.moveContent( - fileProperties.value!.name, - modifiableProperties.name.value, - fileProperties.value!.path, - fileProperties.value!.path - ) - } - if (tags.value.length > 0) { - Engine.instance.api.service(staticResourcePath).patch(id.value, { - tags: tags.value - }) - } + const key = fileProperties.value!.key + Engine.instance.api.service(staticResourcePath).patch(resourceProperties.id.value, { + key, + tags: resourceProperties.tags.value, + licensing: resourceProperties.licensing.value, + attribution: resourceProperties.attribution.value + }) isModified.set(false) } }, []) - const staticResource = useHookstate(() => - Engine.instance.api.service(staticResourcePath).find({ + const staticResource = useHookstate(async () => { + return await Engine.instance.api.service(staticResourcePath).find({ query: { key: fileProperties.value!.key } }) - ) - - const tags: State = useHookstate([]) - const id: State = useHookstate('') + }) + const resourceProperties = useHookstate({ + tags: [] as string[], + id: '', + attribution: '', + licensing: '' + }) useEffect(() => { - if (!staticResource.promised) { - const resources = JSON.parse(JSON.stringify(staticResource.data.value[0])) as StaticResourceType - resources && tags.set(resources.tags) - resources && id.set(resources.id) + if (!staticResource.promised && staticResource.value.data.length > 0) { + const resources = JSON.parse(JSON.stringify(staticResource.value.data[0])) as StaticResourceType + if (resources) { + resourceProperties.tags.set(resources.tags ?? []) + resourceProperties.id.set(resources.id) + resourceProperties.attribution.set(resources.attribution ?? '') + resourceProperties.licensing.set(resources.licensing ?? '') + } } }, [staticResource]) @@ -111,6 +111,7 @@ export const FilePropertiesPanel = (props: { label={t('editor:layout.filebrowser.fileProperties.name')} onChange={onChange(modifiableProperties.name)} value={modifiableProperties.name.value} + disabled={true} /> @@ -122,26 +123,57 @@ export const FilePropertiesPanel = (props: { {modifiableProperties.size.value} - - {tags.value.map((tag, index) => ( - - ))} - {isModified.value && ( - + {resourceProperties.id.value && ( + <> +
+ + + +
+ {(resourceProperties.tags.value ?? []).map((tag, index) => ( +
+ + +
+ ))} +
+ {isModified.value && ( + + )} + )} diff --git a/packages/server-core/src/media/static-resource/project-resource.service.ts b/packages/server-core/src/media/static-resource/project-resource.service.ts index 400cbde465e..b098ae1bd25 100644 --- a/packages/server-core/src/media/static-resource/project-resource.service.ts +++ b/packages/server-core/src/media/static-resource/project-resource.service.ts @@ -54,6 +54,11 @@ const createProjectResource = }) //wipe URLs from resources for (const resource of resources) { + for (const field of Object.keys(resource)) { + if (resource[field] === null) { + delete resource[field] + } + } resource.url = '' } const storageProvider = getStorageProvider() diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index 6b0698e3037..8247a3e303e 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -1688,14 +1688,12 @@ export const uploadLocalProjectToProvider = async ( for (const item of manifest) { if (existingIdSet.has(item.id)) continue - const key = `projects/${projectName}${item.key}` - const url = getCachedURL(key, cacheDomain) + const url = getCachedURL(item.key, cacheDomain) await app.service(staticResourcePath).create({ ...item, - key, url }) - logger.info(`Uploaded static resource ${key} from resources.json`) + logger.info(`Uploaded static resource ${item.key} from resources.json`) } } From f6d36e1d2360e3c284c5c0d3847d892164eba4a8 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Thu, 21 Dec 2023 10:59:42 -0800 Subject: [PATCH 13/21] close file properties on save changes automatically regenerate resources.json whenever a file is changed or uploaded --- .../components/assets/FileBrowserContentPanel.tsx | 2 -- .../src/components/assets/FilePropertiesPanel.tsx | 11 ++++++++--- packages/editor/src/functions/assetFunctions.ts | 12 ++++++++---- .../editor/src/functions/saveProjectResources.ts | 6 +----- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/packages/editor/src/components/assets/FileBrowserContentPanel.tsx b/packages/editor/src/components/assets/FileBrowserContentPanel.tsx index 255f78442e1..a13b1075503 100644 --- a/packages/editor/src/components/assets/FileBrowserContentPanel.tsx +++ b/packages/editor/src/components/assets/FileBrowserContentPanel.tsx @@ -69,7 +69,6 @@ import { archiverPath } from '@etherealengine/engine/src/schemas/media/archiver. import { fileBrowserUploadPath } from '@etherealengine/engine/src/schemas/media/file-browser-upload.schema' import { SupportedFileTypes } from '../../constants/AssetTypes' import { inputFileWithAddToScene } from '../../functions/assetFunctions' -import { saveProjectResources } from '../../functions/saveProjectResources' import { bytesToSize, unique } from '../../functions/utils' import { Button } from '../inputs/Button' import StringInput from '../inputs/StringInput' @@ -533,7 +532,6 @@ const FileBrowserContentPanel: React.FC = (props) variant="body2" /> )} -
diff --git a/packages/editor/src/components/assets/FilePropertiesPanel.tsx b/packages/editor/src/components/assets/FilePropertiesPanel.tsx index f8eafb3315d..84d42b6a072 100644 --- a/packages/editor/src/components/assets/FilePropertiesPanel.tsx +++ b/packages/editor/src/components/assets/FilePropertiesPanel.tsx @@ -35,6 +35,7 @@ import { FileType } from './FileBrowserContentPanel' import styles from './styles.module.scss' import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' +import { saveProjectResources } from '../../functions/saveProjectResources' import { Button } from '../inputs/Button' export const FilePropertiesPanel = (props: { @@ -58,16 +59,18 @@ export const FilePropertiesPanel = (props: { } }, []) - const onSaveChanges = useCallback(() => { - if (isModified.value) { + const onSaveChanges = useCallback(async () => { + if (isModified.value && resourceProperties.value.id) { const key = fileProperties.value!.key - Engine.instance.api.service(staticResourcePath).patch(resourceProperties.id.value, { + await Engine.instance.api.service(staticResourcePath).patch(resourceProperties.id.value, { key, tags: resourceProperties.tags.value, licensing: resourceProperties.licensing.value, attribution: resourceProperties.attribution.value }) + await saveProjectResources(resourceProperties.project.value) isModified.set(false) + openProperties.set(false) } }, []) @@ -81,6 +84,7 @@ export const FilePropertiesPanel = (props: { const resourceProperties = useHookstate({ tags: [] as string[], id: '', + project: '', attribution: '', licensing: '' }) @@ -92,6 +96,7 @@ export const FilePropertiesPanel = (props: { resourceProperties.id.set(resources.id) resourceProperties.attribution.set(resources.attribution ?? '') resourceProperties.licensing.set(resources.licensing ?? '') + resourceProperties.project.set(resources.project ?? '') } } }, [staticResource]) diff --git a/packages/editor/src/functions/assetFunctions.ts b/packages/editor/src/functions/assetFunctions.ts index 88e3afee58e..389dfa429db 100644 --- a/packages/editor/src/functions/assetFunctions.ts +++ b/packages/editor/src/functions/assetFunctions.ts @@ -38,7 +38,6 @@ import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' import { assetLibraryPath } from '@etherealengine/engine/src/schemas/assets/asset-library.schema' import { fileBrowserUploadPath } from '@etherealengine/engine/src/schemas/media/file-browser-upload.schema' import { fileBrowserPath } from '@etherealengine/engine/src/schemas/media/file-browser.schema' -import { addMediaNode } from './addMediaNode' const logger = multiLogger.child({ component: 'editor:assetFunctions' }) @@ -85,9 +84,9 @@ export const inputFileWithAddToScene = async ({ logger.info('zip files extracted') ) - if (projectName) { - uploadedURLs.forEach((url) => addMediaNode(url)) - } + // if (projectName) { + // uploadedURLs.forEach((url) => addMediaNode(url)) + // } resolve(null) } @@ -107,6 +106,11 @@ export const uploadProjectFiles = (projectName: string, files: File[], isAsset = ) } + const uploadPromises = [...promises] + Promise.all(uploadPromises).then(() => + Engine.instance.api.service('project-resources').create({ project: projectName }) + ) + return { cancel: () => promises.forEach((promise) => promise.cancel()), promises: promises.map((promise) => promise.promise) diff --git a/packages/editor/src/functions/saveProjectResources.ts b/packages/editor/src/functions/saveProjectResources.ts index 7aa65569912..b40af07a338 100644 --- a/packages/editor/src/functions/saveProjectResources.ts +++ b/packages/editor/src/functions/saveProjectResources.ts @@ -24,11 +24,7 @@ Ethereal Engine. All Rights Reserved. */ import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' -import { getState } from '@etherealengine/hyperflux' -import { EditorState } from '../services/EditorServices' - -export async function saveProjectResources() { - const project = getState(EditorState).projectName! +export async function saveProjectResources(project: string) { await Engine.instance.api.service('project-resources').create({ project }) } From 4bfa807005597bafc6399b92232ba8ee6eccfb45 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Thu, 21 Dec 2023 14:59:10 -0800 Subject: [PATCH 14/21] undo hardcoded visible back button --- .../editor/src/components/assets/FileBrowserContentPanel.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/editor/src/components/assets/FileBrowserContentPanel.tsx b/packages/editor/src/components/assets/FileBrowserContentPanel.tsx index a13b1075503..976f1527eb6 100644 --- a/packages/editor/src/components/assets/FileBrowserContentPanel.tsx +++ b/packages/editor/src/components/assets/FileBrowserContentPanel.tsx @@ -316,7 +316,7 @@ const FileBrowserContentPanel: React.FC = (props) const showUploadAndDownloadButtons = selectedDirectory.value.slice(1).startsWith('projects/') && !['projects', 'projects/'].includes(selectedDirectory.value.slice(1)) - const showBackButton = true //selectedDirectory.value !== originalPath + const showBackButton = selectedDirectory.value !== originalPath const handleDownloadProject = async () => { const url = selectedDirectory.value From b7b7a57621f8a1f84584a3f607d7644eb002027f Mon Sep 17 00:00:00 2001 From: David Gordon Date: Thu, 21 Dec 2023 15:10:27 -0800 Subject: [PATCH 15/21] increase pagination count in existing asset query --- .../src/projects/project/project-helper.ts | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index 8247a3e303e..03b6282da55 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -1665,23 +1665,21 @@ export const uploadLocalProjectToProvider = async ( const filtered = files.filter((file) => !file.includes(`projects/${projectName}/.git/`)) const results = [] as (string | null)[] const resourceKey = (key, hash) => `${key}#${hash}` - const { existingContentSet, existingIdSet } = await app - .service(staticResourcePath) - .find({ - query: { - project: projectName - } - }) - .then((res) => { - const existingContentSet = new Set() - const existingIdSet = new Set() - for (const item of res.data) { - existingContentSet.add(resourceKey(item.key, item.hash)) - existingIdSet.add(item.id) - } - return { existingContentSet, existingIdSet } - }) - + const existingResources = await app.service(staticResourcePath).find({ + query: { + project: projectName + }, + paginate: { + default: Number.MAX_SAFE_INTEGER, + max: Number.MAX_SAFE_INTEGER + } + }) + const existingContentSet = new Set() + const existingIdSet = new Set() + for (const item of existingResources.data) { + existingContentSet.add(resourceKey(item.key, item.hash)) + existingIdSet.add(item.id) + } if (hasResourceDB) { //if we have a resources.sql file, use it to populate static-resource table const manifest: StaticResourceType[] = JSON.parse(fs.readFileSync(resourceDBPath).toString()) From f7ff511a0e349fe7752c48338618e35e6aad9c57 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Thu, 21 Dec 2023 15:29:44 -0800 Subject: [PATCH 16/21] fix eslint errors --- .../src/media/static-resource/project-resource.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server-core/src/media/static-resource/project-resource.service.ts b/packages/server-core/src/media/static-resource/project-resource.service.ts index b098ae1bd25..0995ab9f3d7 100644 --- a/packages/server-core/src/media/static-resource/project-resource.service.ts +++ b/packages/server-core/src/media/static-resource/project-resource.service.ts @@ -68,7 +68,7 @@ const createProjectResource = ContentType: 'application/json', Key: key }) - if (!!isDev) { + if (isDev !== false) { const filePath = path.resolve(projectsRootFolder, key) const dirName = path.dirname(filePath) fs.mkdirSync(dirName, { recursive: true }) From d08b878bab10ba2e6788619c342812ce9881c5d9 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Fri, 22 Dec 2023 11:37:07 -0800 Subject: [PATCH 17/21] use key instead of id to detect duplicate resources --- .../src/projects/project/project-helper.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index 03b6282da55..2845be79edb 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -1675,17 +1675,17 @@ export const uploadLocalProjectToProvider = async ( } }) const existingContentSet = new Set() - const existingIdSet = new Set() + const existingKeySet = new Set() for (const item of existingResources.data) { existingContentSet.add(resourceKey(item.key, item.hash)) - existingIdSet.add(item.id) + existingKeySet.add(item.key) } if (hasResourceDB) { //if we have a resources.sql file, use it to populate static-resource table const manifest: StaticResourceType[] = JSON.parse(fs.readFileSync(resourceDBPath).toString()) for (const item of manifest) { - if (existingIdSet.has(item.id)) continue + if (existingKeySet.has(item.id)) continue const url = getCachedURL(item.key, cacheDomain) await app.service(staticResourcePath).create({ ...item, @@ -1724,7 +1724,16 @@ export const uploadLocalProjectToProvider = async ( const hash = createStaticResourceHash(fileResult, { mimeType: contentType, assetURL: key }) if (existingContentSet.has(resourceKey(key, hash))) { logger.info(`Skipping upload of static resource of class ${thisFileClass}: "${key}"`) - } else { + } else if (existingKeySet.has(key)) { + logger.info(`Updating static resource of class ${thisFileClass}: "${key}"`) + await app.service(staticResourcePath).patch(null, { + hash, + url, + mimeType: contentType, + tags: [thisFileClass] + }) + } + { await app.service(staticResourcePath).create({ key: `projects/${projectName}${filePathRelative}`, project: projectName, From e88d255289216671cfff5860f286a1e5a8fd9f69 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Fri, 22 Dec 2023 11:39:22 -0800 Subject: [PATCH 18/21] remove pagination on project-helper --- .../server-core/src/projects/project/project-helper.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index 2845be79edb..ffc25b8aa90 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -1643,7 +1643,7 @@ export const deleteProjectFilesInStorageProvider = async (projectName: string, s */ export const uploadLocalProjectToProvider = async ( app: Application, - projectName, + projectName: string, remove = true, storageProviderName?: string ) => { @@ -1669,14 +1669,11 @@ export const uploadLocalProjectToProvider = async ( query: { project: projectName }, - paginate: { - default: Number.MAX_SAFE_INTEGER, - max: Number.MAX_SAFE_INTEGER - } + paginate: false }) const existingContentSet = new Set() const existingKeySet = new Set() - for (const item of existingResources.data) { + for (const item of existingResources) { existingContentSet.add(resourceKey(item.key, item.hash)) existingKeySet.add(item.key) } From f9ca92c7a8daf345a0d7f4662381a4c4cfc8a79d Mon Sep 17 00:00:00 2001 From: David Gordon Date: Fri, 22 Dec 2023 14:33:51 -0800 Subject: [PATCH 19/21] fix duplicate resources generated on npm run dev locally --- .../editor/src/components/assets/FilePropertiesPanel.tsx | 2 ++ packages/server-core/src/projects/project/project-helper.ts | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/editor/src/components/assets/FilePropertiesPanel.tsx b/packages/editor/src/components/assets/FilePropertiesPanel.tsx index 84d42b6a072..2ce1b146b54 100644 --- a/packages/editor/src/components/assets/FilePropertiesPanel.tsx +++ b/packages/editor/src/components/assets/FilePropertiesPanel.tsx @@ -34,6 +34,7 @@ import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' import { FileType } from './FileBrowserContentPanel' import styles from './styles.module.scss' +import { logger } from '@etherealengine/client-core/src/user/services/AuthService' import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' import { saveProjectResources } from '../../functions/saveProjectResources' import { Button } from '../inputs/Button' @@ -90,6 +91,7 @@ export const FilePropertiesPanel = (props: { }) useEffect(() => { if (!staticResource.promised && staticResource.value.data.length > 0) { + if (staticResource.value.data.length > 1) logger.warn('Multiple resources with same key found') const resources = JSON.parse(JSON.stringify(staticResource.value.data[0])) as StaticResourceType if (resources) { resourceProperties.tags.set(resources.tags ?? []) diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index ffc25b8aa90..4916f56a0d8 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -1682,7 +1682,10 @@ export const uploadLocalProjectToProvider = async ( const manifest: StaticResourceType[] = JSON.parse(fs.readFileSync(resourceDBPath).toString()) for (const item of manifest) { - if (existingKeySet.has(item.id)) continue + if (existingKeySet.has(item.key)) { + logger.info(`Skipping upload of static resource: "${item.key}"`) + continue + } const url = getCachedURL(item.key, cacheDomain) await app.service(staticResourcePath).create({ ...item, From a603f2c07b8169209fb71c5ee7c0b17dabe23d45 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Fri, 22 Dec 2023 14:38:20 -0800 Subject: [PATCH 20/21] use useFind in FilePropertiesPanel --- .../components/assets/FilePropertiesPanel.tsx | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/editor/src/components/assets/FilePropertiesPanel.tsx b/packages/editor/src/components/assets/FilePropertiesPanel.tsx index 2ce1b146b54..21ff57504ff 100644 --- a/packages/editor/src/components/assets/FilePropertiesPanel.tsx +++ b/packages/editor/src/components/assets/FilePropertiesPanel.tsx @@ -35,6 +35,7 @@ import { FileType } from './FileBrowserContentPanel' import styles from './styles.module.scss' import { logger } from '@etherealengine/client-core/src/user/services/AuthService' +import { useFind } from '@etherealengine/engine/src/common/functions/FeathersHooks' import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' import { saveProjectResources } from '../../functions/saveProjectResources' import { Button } from '../inputs/Button' @@ -75,13 +76,12 @@ export const FilePropertiesPanel = (props: { } }, []) - const staticResource = useHookstate(async () => { - return await Engine.instance.api.service(staticResourcePath).find({ - query: { - key: fileProperties.value!.key - } - }) + const staticResource = useFind(staticResourcePath, { + query: { + key: fileProperties.value!.key + } }) + const resourceProperties = useHookstate({ tags: [] as string[], id: '', @@ -90,9 +90,9 @@ export const FilePropertiesPanel = (props: { licensing: '' }) useEffect(() => { - if (!staticResource.promised && staticResource.value.data.length > 0) { - if (staticResource.value.data.length > 1) logger.warn('Multiple resources with same key found') - const resources = JSON.parse(JSON.stringify(staticResource.value.data[0])) as StaticResourceType + if (staticResource.data.length > 0) { + if (staticResource.data.length > 1) logger.warn('Multiple resources with same key found') + const resources = JSON.parse(JSON.stringify(staticResource.data[0])) as StaticResourceType if (resources) { resourceProperties.tags.set(resources.tags ?? []) resourceProperties.id.set(resources.id) From b95b3f6d540db11d09bebbdfaadf1e58a8fd1398 Mon Sep 17 00:00:00 2001 From: David Gordon Date: Fri, 22 Dec 2023 19:58:47 -0800 Subject: [PATCH 21/21] remove mysqldump from server package.json --- packages/server-core/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server-core/package.json b/packages/server-core/package.json index f940838a806..f2ffcebff4c 100755 --- a/packages/server-core/package.json +++ b/packages/server-core/package.json @@ -101,7 +101,6 @@ "multer": "1.4.5-lts.1", "mysql": "2.18.1", "mysql2": "3.2.0", - "mysqldump": "3.2.0", "nanoid": "3.3.4", "node-fetch": "2.6.9", "nodemailer-smtp-transport": "^2.7.4",