Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Simplify Static Resources #9061

Merged
merged 4 commits into from
Oct 18, 2023
Merged

Simplify Static Resources #9061

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fd87665
simplify static resource parsing of scenes and uploads
HexaField Oct 16, 2023
9aefcef
Merge branch 'dev' into simplify-static-resources
HexaField Oct 16, 2023
3c71568
add download button in file browser input
HexaField Oct 16, 2023
6bff7f9
fix
HexaField Oct 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 0 additions & 1 deletion .env.local.default
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ LOCAL_STORAGE_PROVIDER_PORT=8642
GOOGLE_ANALYTICS_TRACKING_ID=
HUB_ENDPOINT=https://etherealengine.io
INSTANCESERVER_UNREACHABLE_TIMEOUT_SECONDS=10
CLONE_STATIC_RESOURCES=false


MATCHMAKER_EMULATION_MODE=true
Expand Down
6 changes: 3 additions & 3 deletions packages/editor/src/components/inputs/ArrayInputGroup.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ All portions of the code written by the Ethereal Engine team are Copyright 漏 20
Ethereal Engine. All Rights Reserved.
*/

import Icon from '@etherealengine/ui/src/primitives/mui/Icon'
import AddIcon from '@mui/icons-material/Add'
import DeleteIcon from '@mui/icons-material/Delete'
import IconButton from '@mui/material/IconButton'
import React from 'react'
import styles from './ArrayInputGroup.module.scss'
Expand Down Expand Up @@ -105,7 +105,7 @@ const ArrayInputGroup = ({
padding: 0
}}
>
<AddIcon sx={{ color: 'primary.contrastText' }} />
<AddIcon sx={{ color: 'var(--textColor)' }} />
</IconButton>
</InputGroup>
{values &&
Expand All @@ -125,7 +125,7 @@ const ArrayInputGroup = ({
}}
onClick={() => deleteInput(index + 1)}
>
<DeleteIcon sx={{ color: 'primary.contrastText' }} />
<Icon type="Delete" style={{ color: 'var(--textColor)' }} />
</IconButton>
</InputGroup>
))}
Expand Down
44 changes: 41 additions & 3 deletions packages/editor/src/components/inputs/FileBrowserInput.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,12 @@ All portions of the code written by the Ethereal Engine team are Copyright 漏 20
Ethereal Engine. All Rights Reserved.
*/

import IconButton from '@mui/material/IconButton'
import React from 'react'
import { useDrop } from 'react-dnd'

import config from '@etherealengine/common/src/config'
import Icon from '@etherealengine/ui/src/primitives/mui/Icon'
import { ItemTypes } from '../../constants/AssetTypes'
import useUpload from '../assets/useUpload'
import { ControlledStringInput, StringInputProps } from './StringInput'
Expand Down Expand Up @@ -54,6 +57,32 @@ export function FileBrowserInput({
}
const onUpload = useUpload(uploadOptions)

// todo fix for invalid URLs
const assetIsExternal = value && !value?.includes(config.client.fileServer)
const uploadExternalAsset = () => {
onUpload([
{
isFile: true,
name: value?.split('/').pop(),
file: async (onSuccess, onFail) => {
try {
const asset = await fetch(value!)
const blob = await asset.blob()
const file = new File([blob], value!.split('/').pop()!)
onSuccess(file)
} catch (error) {
if (onFail) onFail(error)
else throw error
}
}
} as Partial<FileSystemFileEntry>
] as any).then((assets) => {
if (assets) {
onChange?.(assets[0])
}
})
}

const [{ canDrop, isOver }, dropRef] = useDrop({
accept: [...acceptDropItems, ItemTypes.File],
async drop(item: any, monitor) {
Expand All @@ -73,9 +102,7 @@ export function FileBrowserInput({

onUpload(entries).then((assets) => {
if (assets) {
for (let index = 0; index < assets.length; index++) {
onChange?.(assets[index])
}
onChange?.(assets[0])
}
})
}
Expand All @@ -96,6 +123,17 @@ export function FileBrowserInput({
canDrop={isOver && canDrop}
{...rest}
/>
{assetIsExternal && (
<IconButton
disableRipple
style={{
padding: 0
}}
onClick={uploadExternalAsset}
>
<Icon type="Download" style={{ color: 'var(--textColor)' }} />
</IconButton>
)}
</>
)
}
Expand Down
2 changes: 0 additions & 2 deletions packages/server-core/src/appconfig.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,6 @@ const server = {
corsServerPort: process.env.CORS_SERVER_PORT!,
storageProvider: process.env.STORAGE_PROVIDER!,
storageProviderExternalEndpoint: process.env.STORAGE_PROVIDER_EXTERNAL_ENDPOINT!,
cloneProjectStaticResources:
typeof process.env.CLONE_STATIC_RESOURCES === 'undefined' ? true : process.env.CLONE_STATIC_RESOURCES === 'true',
gaTrackingId: process.env.GOOGLE_ANALYTICS_TRACKING_ID!,
hub: {
endpoint: process.env.HUB_ENDPOINT!
Expand Down
209 changes: 1 addition & 208 deletions packages/server-core/src/media/static-resource/static-resource-helper.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,10 @@ Ethereal Engine. All Rights Reserved.

import appRootPath from 'app-root-path'
import assert from 'assert'
import fs from 'fs'
import path from 'path'

import { destroyEngine } from '@etherealengine/engine/src/ecs/classes/Engine'

import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema'
import { Paginated } from '@feathersjs/feathers'
import { Application } from '../../../declarations'
import { mockFetch, restoreFetch } from '../../../tests/util/mockFetch'
import { createFeathersKoaApp } from '../../createApp'
import { getCachedURL } from '../storageprovider/getCachedURL'
import { getStorageProvider } from '../storageprovider/storageprovider'
import { addAssetFromProject, downloadResourceAndMetadata } from './static-resource-helper'
import { downloadResourceAndMetadata } from './static-resource-helper'

describe('static-resource-helper', () => {
before(() => {
Expand Down Expand Up @@ -91,201 +82,3 @@ describe('static-resource-helper', () => {
})
})
})

const testProject = 'test-project'

describe('audio-upload', () => {
let app: Application

before(async () => {
app = createFeathersKoaApp()
await app.setup()
const url = 'https://test.com/projects/default-project/assets/test.mp3'
const url2 = getCachedURL('/projects/default-project/assets/test.mp3', getStorageProvider().cacheDomain)
mockFetch({
[url]: {
contentType: 'audio/mpeg',
response: fs.readFileSync(
path.join(appRootPath.path, '/packages/projects/default-project/assets/SampleAudio.mp3')
)
},
[url2]: {
contentType: 'audio/mpeg',
response: fs.readFileSync(
path.join(appRootPath.path, '/packages/projects/default-project/assets/SampleAudio.mp3')
)
}
})
})

afterEach(async () => {
const storageProvider = getStorageProvider()
if (await storageProvider.doesExist('test.mp3', 'static-resources/test-project/'))
await storageProvider.deleteResources(['static-resources/test-project/test.mp3'])

const existingResource = (await app.service(staticResourcePath).find({
query: {
mimeType: 'audio/mpeg'
},
paginate: false
})) as StaticResourceType[]
await Promise.all(existingResource.map((resource) => app.service(staticResourcePath).remove(resource.id)))
})

after(() => {
restoreFetch()
return destroyEngine()
})

describe('addAssetFromProject', () => {
it('should download audio asset as a new static resource from external url when forced to download', async () => {
const storageProvider = getStorageProvider()
const url = 'https://test.com/projects/default-project/assets/test.mp3'

const staticResource = await addAssetFromProject(app, url, testProject, true)

assert(staticResource.id)
assert.equal(staticResource.url, getCachedURL(staticResource.key!, storageProvider.cacheDomain))
assert.equal(staticResource.key, 'static-resources/test-project/test.mp3')
assert.equal(staticResource.mimeType, 'audio/mpeg')
assert.equal(staticResource.project, testProject)

assert(await storageProvider.doesExist('test.mp3', 'static-resources/test-project/'))

const file = await storageProvider.getObject(staticResource.key!)
assert.equal(file.ContentType, 'audio/mpeg')
})

it('should link audio asset as a new static resource from external url when not forced to download', async () => {
const storageProvider = getStorageProvider()
const url = 'https://test.com/projects/default-project/assets/test.mp3'

const staticResource = await addAssetFromProject(app, url, testProject, false)

assert(staticResource.id)
assert.equal(staticResource.url, 'https://test.com/projects/default-project/assets/test.mp3')
assert.equal(staticResource.key, 'https://test.com/projects/default-project/assets/test.mp3')
assert.equal(staticResource.mimeType, 'audio/mpeg')
assert.equal(staticResource.project, testProject)

const fileExists = await storageProvider.doesExist('test.mp3', 'static-resources/test-project/')
assert(!fileExists)
})

it('should download audio asset as a new static resource from another project', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const staticResource = await addAssetFromProject(app, url, testProject, true)

assert.equal(staticResource.key, 'static-resources/test-project/test.mp3')
assert.equal(staticResource.url, getCachedURL(staticResource.key!, storageProvider.cacheDomain))
assert.equal(staticResource.mimeType, 'audio/mpeg')
assert.equal(staticResource.project, testProject)

assert(await storageProvider.doesExist('test.mp3', 'static-resources/test-project/'))

const file = await storageProvider.getObject(staticResource.key!)
assert.equal(file.ContentType, 'audio/mpeg')
})

it('should link audio asset as a new static resource from another project', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const staticResource = await addAssetFromProject(app, url, testProject, false)

assert.equal(staticResource.key, url)
assert.equal(staticResource.url, url)
assert.equal(staticResource.mimeType, 'audio/mpeg')
assert.equal(staticResource.project, testProject)

// should not exist under static resources
const fileExists = await storageProvider.doesExist('test.mp3', 'static-resources/test-project/')
assert(!fileExists)
})

it('should link audio asset as a new static resource from url if from the same project', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const staticResource = await addAssetFromProject(app, url, 'default-project', false)

assert.equal(staticResource.key, 'projects/default-project/assets/test.mp3')
assert.equal(staticResource.url, url)
assert.equal(staticResource.mimeType, 'audio/mpeg')
assert.equal(staticResource.project, 'default-project')

// should not exist under static resources
const fileExists = await storageProvider.doesExist('test.mp3', 'static-resources/test-project/')
assert(!fileExists)
})

it('should return existing static resource with the same hash and project', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const response = await addAssetFromProject(app, url, 'default-project', false)
const response2 = await addAssetFromProject(app, url, 'default-project', false)

const staticResources = (await app.service(staticResourcePath).find({
query: {
url
}
})) as Paginated<StaticResourceType>

assert.equal(staticResources.data.length, 1)
assert.equal(response.id, response2.id)
assert.equal(response.url, response2.url)
assert.equal(response.key, response2.key)
})

it('should return new static resource with the same hash exists in another project when forcing download', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const response = await addAssetFromProject(app, url, 'default-project', true)
const response2 = await addAssetFromProject(app, url, 'test-project', true)

const staticResources = (await app.service(staticResourcePath).find({
query: {
url: response.url
}
})) as Paginated<StaticResourceType>
assert.equal(staticResources.data.length, 1)

const staticResources2 = (await app.service(staticResourcePath).find({
query: {
url: response2.url
}
})) as Paginated<StaticResourceType>
assert.equal(staticResources2.data.length, 1)

assert.notEqual(response.id, response2.id)
assert.notEqual(response.url, response2.url)
assert.notEqual(response.key, response2.key)
assert.equal(response.hash, response2.hash)
})

it('should different static resource with the same url and hash if it exists in another project when not forcing download', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const response = await addAssetFromProject(app, url, 'default-project', false)
const response2 = await addAssetFromProject(app, url, 'test-project', false)

const staticResources = (await app.service(staticResourcePath).find({
query: {
url: response.url
}
})) as Paginated<StaticResourceType>
assert.equal(staticResources.data.length, 2)

assert(response.id !== response2.id)
assert.equal(response.url, response2.url)
// key is different as it is a different project
assert.notEqual(response.key, response2.key)
assert.equal(response.hash, response2.hash)
})
})
})