diff --git a/packages/client-core/i18n/en/editor.json b/packages/client-core/i18n/en/editor.json index bb87cd6b724..3eceac50206 100755 --- a/packages/client-core/i18n/en/editor.json +++ b/packages/client-core/i18n/en/editor.json @@ -950,7 +950,11 @@ "name": "Name:", "type": "Type:", "size": "Size:", - "url": "URL:" + "url": "URL:", + "attribution": "Attribution:", + "licensing": "Licensing:", + "tag": "Tag:", + "addTag": "Add Tag" } } }, diff --git a/packages/editor/src/components/assets/FileBrowserContentPanel.tsx b/packages/editor/src/components/assets/FileBrowserContentPanel.tsx index f3675f8b060..976f1527eb6 100644 --- a/packages/editor/src/components/assets/FileBrowserContentPanel.tsx +++ b/packages/editor/src/components/assets/FileBrowserContentPanel.tsx @@ -59,9 +59,6 @@ import ViewInArIcon from '@mui/icons-material/ViewInAr' import VolumeUpIcon from '@mui/icons-material/VolumeUp' import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' -import Dialog from '@etherealengine/ui/src/primitives/mui/Dialog' -import DialogTitle from '@etherealengine/ui/src/primitives/mui/DialogTitle' -import Grid from '@etherealengine/ui/src/primitives/mui/Grid' import Typography from '@etherealengine/ui/src/primitives/mui/Typography' import { Breadcrumbs, Link, PopoverPosition, TablePagination } from '@mui/material' @@ -80,6 +77,7 @@ import { AssetSelectionChangePropsType } from './AssetsPreviewPanel' import CompressionPanel from './CompressionPanel' import { FileBrowserItem } from './FileBrowserGrid' import { FileDataType } from './FileDataType' +import { FilePropertiesPanel } from './FilePropertiesPanel' import ImageConvertPanel from './ImageConvertPanel' import styles from './styles.module.scss' @@ -558,37 +556,7 @@ const FileBrowserContentPanel: React.FC = (props) )} {openProperties.value && fileProperties.value && ( - openProperties.set(false)} - classes={{ paper: styles.paperDialog }} - > - - {`${fileProperties.value.name} ${fileProperties.value.type == 'folder' ? 'folder' : 'file'} Properties`} - - - - - {t('editor:layout.filebrowser.fileProperties.name')} - - - {t('editor:layout.filebrowser.fileProperties.type')} - - - {t('editor:layout.filebrowser.fileProperties.size')} - - - {t('editor:layout.filebrowser.fileProperties.url')} - - - - {fileProperties.value.name} - {fileProperties.value.type} - {fileProperties.value.size} - {fileProperties.value.url} - - - + )} + fileProperties: State +}) => { + const { openProperties, fileProperties } = props + const { t } = useTranslation() + if (!fileProperties.value) return null + + const modifiableProperties: State = useHookstate( + JSON.parse(JSON.stringify(fileProperties.get(NO_PROXY))) as FileType + ) + + const isModified = useHookstate(false) + + const onChange = useCallback((state: State) => { + isModified.set(true) + return (e) => { + state.set(e.target.value) + } + }, []) + + const onSaveChanges = useCallback(async () => { + if (isModified.value && resourceProperties.value.id) { + const key = fileProperties.value!.key + await Engine.instance.api.service(staticResourcePath).patch(resourceProperties.id.value, { + key, + tags: resourceProperties.tags.value, + licensing: resourceProperties.licensing.value, + attribution: resourceProperties.attribution.value + }) + await saveProjectResources(resourceProperties.project.value) + isModified.set(false) + openProperties.set(false) + } + }, []) + + const staticResource = useFind(staticResourcePath, { + query: { + key: fileProperties.value!.key + } + }) + + const resourceProperties = useHookstate({ + tags: [] as string[], + id: '', + project: '', + attribution: '', + licensing: '' + }) + useEffect(() => { + if (staticResource.data.length > 0) { + if (staticResource.data.length > 1) logger.warn('Multiple resources with same key found') + const resources = JSON.parse(JSON.stringify(staticResource.data[0])) as StaticResourceType + if (resources) { + resourceProperties.tags.set(resources.tags ?? []) + resourceProperties.id.set(resources.id) + resourceProperties.attribution.set(resources.attribution ?? '') + resourceProperties.licensing.set(resources.licensing ?? '') + resourceProperties.project.set(resources.project ?? '') + } + } + }, [staticResource]) + + return ( + openProperties.set(false)} + classes={{ paper: styles.paperDialog }} + > + + {`${fileProperties.value.name} ${fileProperties.value.type == 'folder' ? 'folder' : 'file'} Properties`} + +
+ + + + {t('editor:layout.filebrowser.fileProperties.type')} + {t('editor:layout.filebrowser.fileProperties.size')} + + + {modifiableProperties.type.value} + {modifiableProperties.size.value} + + + {resourceProperties.id.value && ( + <> +
+ + + +
+ {(resourceProperties.tags.value ?? []).map((tag, index) => ( +
+ + +
+ ))} +
+ {isModified.value && ( + + )} + + )} + +
+ ) +} diff --git a/packages/editor/src/functions/assetFunctions.ts b/packages/editor/src/functions/assetFunctions.ts index 88e3afee58e..389dfa429db 100644 --- a/packages/editor/src/functions/assetFunctions.ts +++ b/packages/editor/src/functions/assetFunctions.ts @@ -38,7 +38,6 @@ import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' import { assetLibraryPath } from '@etherealengine/engine/src/schemas/assets/asset-library.schema' import { fileBrowserUploadPath } from '@etherealengine/engine/src/schemas/media/file-browser-upload.schema' import { fileBrowserPath } from '@etherealengine/engine/src/schemas/media/file-browser.schema' -import { addMediaNode } from './addMediaNode' const logger = multiLogger.child({ component: 'editor:assetFunctions' }) @@ -85,9 +84,9 @@ export const inputFileWithAddToScene = async ({ logger.info('zip files extracted') ) - if (projectName) { - uploadedURLs.forEach((url) => addMediaNode(url)) - } + // if (projectName) { + // uploadedURLs.forEach((url) => addMediaNode(url)) + // } resolve(null) } @@ -107,6 +106,11 @@ export const uploadProjectFiles = (projectName: string, files: File[], isAsset = ) } + const uploadPromises = [...promises] + Promise.all(uploadPromises).then(() => + Engine.instance.api.service('project-resources').create({ project: projectName }) + ) + return { cancel: () => promises.forEach((promise) => promise.cancel()), promises: promises.map((promise) => promise.promise) diff --git a/packages/editor/src/functions/saveProjectResources.ts b/packages/editor/src/functions/saveProjectResources.ts new file mode 100644 index 00000000000..b40af07a338 --- /dev/null +++ b/packages/editor/src/functions/saveProjectResources.ts @@ -0,0 +1,30 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + +import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' + +export async function saveProjectResources(project: string) { + await Engine.instance.api.service('project-resources').create({ project }) +} diff --git a/packages/engine/src/schemas/media/static-resource.schema.ts b/packages/engine/src/schemas/media/static-resource.schema.ts index 7366924e18a..a47bed35355 100755 --- a/packages/engine/src/schemas/media/static-resource.schema.ts +++ b/packages/engine/src/schemas/media/static-resource.schema.ts @@ -91,7 +91,7 @@ export const staticResourceQueryProperties = Type.Pick(staticResourceSchema, [ 'driver', 'attribution', 'licensing', - // 'tags', + 'tags', 'url' // 'stats' ]) @@ -103,6 +103,9 @@ export const staticResourceQuerySchema = Type.Intersect( }, mimeType: { $like: Type.String() + }, + tags: { + $like: Type.String() } }), // Add additional query properties here diff --git a/packages/server-core/src/media/services.ts b/packages/server-core/src/media/services.ts index a1618f087b1..9097ad9b6da 100755 --- a/packages/server-core/src/media/services.ts +++ b/packages/server-core/src/media/services.ts @@ -28,7 +28,17 @@ import FileBrowser from './file-browser/file-browser' import OEmbed from './oembed/oembed' import Archiver from './recursive-archiver/archiver' import StaticResourceFilters from './static-resource-filters/static-resource-filters' +import ProjectResource from './static-resource/project-resource.service' import StaticResource from './static-resource/static-resource' import Upload from './upload-asset/upload-asset.service' -export default [StaticResource, StaticResourceFilters, FileBrowser, FileBrowserUpload, OEmbed, Upload, Archiver] +export default [ + ProjectResource, + StaticResource, + StaticResourceFilters, + FileBrowser, + FileBrowserUpload, + OEmbed, + Upload, + Archiver +] diff --git a/packages/server-core/src/media/static-resource/project-resource.service.ts b/packages/server-core/src/media/static-resource/project-resource.service.ts new file mode 100644 index 00000000000..0995ab9f3d7 --- /dev/null +++ b/packages/server-core/src/media/static-resource/project-resource.service.ts @@ -0,0 +1,83 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + +import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' +import { Application } from '@feathersjs/koa' + +import { isDev } from '@etherealengine/common/src/config' +import { getStorageProvider } from '../storageprovider/storageprovider' + +import fs from 'fs' +import path from 'path' +import { projectsRootFolder } from '../file-browser/file-browser.class' + +export const projectResourcesPath = 'project-resources' + +export type CreateProjectResourceParams = { + project: string +} + +declare module '@etherealengine/common/declarations' { + interface ServiceTypes { + [projectResourcesPath]: any + } +} + +const createProjectResource = + (app: Application) => + async ({ project }: CreateProjectResourceParams) => { + const resources: StaticResourceType[] = await app.service(staticResourcePath).find({ + query: { project }, + paginate: false + }) + //wipe URLs from resources + for (const resource of resources) { + for (const field of Object.keys(resource)) { + if (resource[field] === null) { + delete resource[field] + } + } + resource.url = '' + } + const storageProvider = getStorageProvider() + const key = `projects/${project}/resources.json` + await storageProvider.putObject({ + Body: Buffer.from(JSON.stringify(resources)), + ContentType: 'application/json', + Key: key + }) + if (isDev !== false) { + const filePath = path.resolve(projectsRootFolder, key) + const dirName = path.dirname(filePath) + fs.mkdirSync(dirName, { recursive: true }) + fs.writeFileSync(filePath, JSON.stringify(resources)) + } + } + +export default (app: Application): void => { + app.use(projectResourcesPath, { + create: createProjectResource(app) + }) +} diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index ebae102aa1d..4916f56a0d8 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -42,7 +42,10 @@ import fs from 'fs' import { PUBLIC_SIGNED_REGEX } from '@etherealengine/common/src/constants/GitHubConstants' import { ProjectPackageJsonType } from '@etherealengine/common/src/interfaces/ProjectPackageJsonType' import { processFileName } from '@etherealengine/common/src/utils/processFileName' +import { AssetLoader } from '@etherealengine/engine/src/assets/classes/AssetLoader' +import { AssetClass } from '@etherealengine/engine/src/assets/enum/AssetClass' import { apiJobPath } from '@etherealengine/engine/src/schemas/cluster/api-job.schema' +import { staticResourcePath, StaticResourceType } from '@etherealengine/engine/src/schemas/media/static-resource.schema' import { ProjectBuilderTagsType } from '@etherealengine/engine/src/schemas/projects/project-builder-tags.schema' import { ProjectCheckSourceDestinationMatchType } from '@etherealengine/engine/src/schemas/projects/project-check-source-destination-match.schema' import { ProjectCheckUnfetchedCommitType } from '@etherealengine/engine/src/schemas/projects/project-check-unfetched-commit.schema' @@ -64,10 +67,12 @@ import { v4 } from 'uuid' import { Application } from '../../../declarations' import config from '../../appconfig' import { getPodsData } from '../../cluster/pods/pods-helper' +import { projectResourcesPath } from '../../media/static-resource/project-resource.service' import { getCacheDomain } from '../../media/storageprovider/getCacheDomain' import { getCachedURL } from '../../media/storageprovider/getCachedURL' import { getStorageProvider } from '../../media/storageprovider/storageprovider' import { getFileKeysRecursive } from '../../media/storageprovider/storageProviderUtils' +import { createStaticResourceHash } from '../../media/upload-asset/upload-asset.service' import logger from '../../ServerLogger' import { ServerState } from '../../ServerState' import { BUILDER_CHART_REGEX } from '../../setting/helm-setting/helm-setting' @@ -1638,7 +1643,7 @@ export const deleteProjectFilesInStorageProvider = async (projectName: string, s */ export const uploadLocalProjectToProvider = async ( app: Application, - projectName, + projectName: string, remove = true, storageProviderName?: string ) => { @@ -1653,27 +1658,103 @@ export const uploadLocalProjectToProvider = async ( // upload new files to storage provider const projectRootPath = path.resolve(projectsRootFolder, projectName) + const resourceDBPath = path.join(projectRootPath, 'resources.json') + const hasResourceDB = fs.existsSync(resourceDBPath) + const files = getFilesRecursive(projectRootPath) const filtered = files.filter((file) => !file.includes(`projects/${projectName}/.git/`)) const results = [] as (string | null)[] + const resourceKey = (key, hash) => `${key}#${hash}` + const existingResources = await app.service(staticResourcePath).find({ + query: { + project: projectName + }, + paginate: false + }) + const existingContentSet = new Set() + const existingKeySet = new Set() + for (const item of existingResources) { + existingContentSet.add(resourceKey(item.key, item.hash)) + existingKeySet.add(item.key) + } + if (hasResourceDB) { + //if we have a resources.sql file, use it to populate static-resource table + const manifest: StaticResourceType[] = JSON.parse(fs.readFileSync(resourceDBPath).toString()) + + for (const item of manifest) { + if (existingKeySet.has(item.key)) { + logger.info(`Skipping upload of static resource: "${item.key}"`) + continue + } + const url = getCachedURL(item.key, cacheDomain) + await app.service(staticResourcePath).create({ + ...item, + url + }) + logger.info(`Uploaded static resource ${item.key} from resources.json`) + } + } + for (const file of filtered) { try { const fileResult = fs.readFileSync(file) const filePathRelative = processFileName(file.slice(projectRootPath.length)) + const contentType = getContentType(file) + const key = `projects/${projectName}${filePathRelative}` + const url = getCachedURL(key, getCacheDomain(storageProvider)) await storageProvider.putObject( { Body: fileResult, - ContentType: getContentType(file), - Key: `projects/${projectName}${filePathRelative}` + ContentType: contentType, + Key: key }, { isDirectory: false } ) + if (!hasResourceDB) { + //otherwise, upload the files into static resources individually + const staticResourceClasses = [ + AssetClass.Audio, + AssetClass.Image, + AssetClass.Model, + AssetClass.Video, + AssetClass.Volumetric + ] + const thisFileClass = AssetLoader.getAssetClass(file) + if (filePathRelative.startsWith('/assets/') && staticResourceClasses.includes(thisFileClass)) { + const hash = createStaticResourceHash(fileResult, { mimeType: contentType, assetURL: key }) + if (existingContentSet.has(resourceKey(key, hash))) { + logger.info(`Skipping upload of static resource of class ${thisFileClass}: "${key}"`) + } else if (existingKeySet.has(key)) { + logger.info(`Updating static resource of class ${thisFileClass}: "${key}"`) + await app.service(staticResourcePath).patch(null, { + hash, + url, + mimeType: contentType, + tags: [thisFileClass] + }) + } + { + await app.service(staticResourcePath).create({ + key: `projects/${projectName}${filePathRelative}`, + project: projectName, + hash, + url, + mimeType: contentType, + tags: [thisFileClass] + }) + logger.info(`Uploaded static resource of class ${thisFileClass}: "${key}"`) + } + } + } results.push(getCachedURL(`projects/${projectName}${filePathRelative}`, cacheDomain)) } catch (e) { logger.error(e) results.push(null) } } + if (!hasResourceDB) { + await app.service(projectResourcesPath).create({ project: projectName }) + } logger.info(`uploadLocalProjectToProvider for project "${projectName}" ended at "${new Date()}".`) return results.filter((success) => !!success) as string[] } diff --git a/packages/server-core/src/projects/project/project.class.ts b/packages/server-core/src/projects/project/project.class.ts index 92aba121b2d..8f9ba92e2f3 100644 --- a/packages/server-core/src/projects/project/project.class.ts +++ b/packages/server-core/src/projects/project/project.class.ts @@ -165,6 +165,7 @@ export class ProjectService