From 31df4b7de87509d0b5d46856d9c662d32191d9bc Mon Sep 17 00:00:00 2001 From: Josh Field Date: Fri, 22 Dec 2023 13:30:43 +1100 Subject: [PATCH] Loading and mocap fixes (#9485) * loading and mocap fixes * scene unload * fix tests --- .../src/components/World/EngineHooks.tsx | 29 ++++++++--------- .../NetworkInstanceProvisioning.tsx | 6 +--- packages/client/src/pages/capture/capture.tsx | 32 ++++++++++--------- .../engine/src/common/AppLoadingService.ts | 1 + .../engine/src/ecs/functions/EntityTree.ts | 15 ++++++--- .../interaction/systems/GrabbableSystem.tsx | 5 ++- .../engine/src/mocap/MotionCaptureSystem.ts | 5 ++- packages/engine/src/mocap/poseToInput.ts | 2 ++ .../functions/WorldNetworkAction.ts | 3 +- .../state/EntityNetworkState.test.ts | 19 ++--------- .../networking/state/EntityNetworkState.tsx | 31 ++++++++---------- .../src/recording/ECSRecordingSystem.ts | 2 +- packages/ui/src/pages/Capture/index.tsx | 29 ++++++++++++++--- 13 files changed, 94 insertions(+), 85 deletions(-) diff --git a/packages/client-core/src/components/World/EngineHooks.tsx b/packages/client-core/src/components/World/EngineHooks.tsx index 328975b2325..0d46b712ce4 100755 --- a/packages/client-core/src/components/World/EngineHooks.tsx +++ b/packages/client-core/src/components/World/EngineHooks.tsx @@ -36,25 +36,21 @@ import { AppLoadingState, AppLoadingStates } from '@etherealengine/engine/src/co import multiLogger from '@etherealengine/engine/src/common/functions/logger' import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine' import { EngineActions, EngineState } from '@etherealengine/engine/src/ecs/classes/EngineState' -import { getComponent, removeComponent } from '@etherealengine/engine/src/ecs/functions/ComponentFunctions' +import { getComponent } from '@etherealengine/engine/src/ecs/functions/ComponentFunctions' import { NetworkState, addNetwork } from '@etherealengine/engine/src/networking/NetworkState' import { Network, NetworkTopics, createNetwork } from '@etherealengine/engine/src/networking/classes/Network' import { NetworkPeerFunctions } from '@etherealengine/engine/src/networking/functions/NetworkPeerFunctions' import { spawnLocalAvatarInWorld } from '@etherealengine/engine/src/networking/functions/receiveJoinWorld' import { PortalComponent, PortalState } from '@etherealengine/engine/src/scene/components/PortalComponent' import { UUIDComponent } from '@etherealengine/engine/src/scene/components/UUIDComponent' -import { addOutgoingTopicIfNecessary, dispatchAction, getMutableState, getState } from '@etherealengine/hyperflux' +import { addOutgoingTopicIfNecessary, dispatchAction, getMutableState } from '@etherealengine/hyperflux' import { loadEngineInjection } from '@etherealengine/projects/loadEngineInjection' import { AvatarState } from '@etherealengine/engine/src/avatar/state/AvatarNetworkState' -import { FollowCameraComponent } from '@etherealengine/engine/src/camera/components/FollowCameraComponent' -import { TargetCameraRotationComponent } from '@etherealengine/engine/src/camera/components/TargetCameraRotationComponent' import { UndefinedEntity } from '@etherealengine/engine/src/ecs/classes/Entity' -import { removeEntity } from '@etherealengine/engine/src/ecs/functions/EntityFunctions' import { WorldNetworkAction } from '@etherealengine/engine/src/networking/functions/WorldNetworkAction' import { LinkState } from '@etherealengine/engine/src/scene/components/LinkComponent' import { InstanceID } from '@etherealengine/engine/src/schemas/networking/instance.schema' -import { ComputedTransformComponent } from '@etherealengine/engine/src/transform/components/ComputedTransformComponent' import { RouterState } from '../../common/services/RouterService' import { LocationState } from '../../social/services/LocationService' import { SocketWebRTCClientNetwork } from '../../transports/SocketWebRTCClientFunctions' @@ -128,24 +124,27 @@ export const useLocationSpawnAvatarWithDespawn = () => { export const despawnSelfAvatar = () => { const clientEntity = Engine.instance.localClientEntity + console.log('despawnSelfAvatar', clientEntity) if (!clientEntity) return - const peersCountForUser = - getState(NetworkState).networks[getState(NetworkState).hostIds.world!].users[Engine.instance.userID]?.length + const network = NetworkState.worldNetwork + + const peersCountForUser = network?.users?.[Engine.instance.userID]?.length // if we are the last peer in the world for this user, destroy the object if (!peersCountForUser || peersCountForUser === 1) { dispatchAction(WorldNetworkAction.destroyObject({ entityUUID: getComponent(clientEntity, UUIDComponent) })) } - const cameraEntity = Engine.instance.cameraEntity - if (!cameraEntity) return + /** @todo this logic should be handled by the camera system */ + // const cameraEntity = Engine.instance.cameraEntity + // if (!cameraEntity) return - const cameraComputed = getComponent(cameraEntity, ComputedTransformComponent) - removeEntity(cameraComputed.referenceEntity) - removeComponent(cameraEntity, ComputedTransformComponent) - removeComponent(cameraEntity, FollowCameraComponent) - removeComponent(cameraEntity, TargetCameraRotationComponent) + // const cameraComputed = getComponent(cameraEntity, ComputedTransformComponent) + // removeEntity(cameraComputed.referenceEntity) + // removeComponent(cameraEntity, ComputedTransformComponent) + // removeComponent(cameraEntity, FollowCameraComponent) + // removeComponent(cameraEntity, TargetCameraRotationComponent) } export const useLinkTeleport = () => { diff --git a/packages/client-core/src/networking/NetworkInstanceProvisioning.tsx b/packages/client-core/src/networking/NetworkInstanceProvisioning.tsx index f0eb5311d68..e88030ce39c 100644 --- a/packages/client-core/src/networking/NetworkInstanceProvisioning.tsx +++ b/packages/client-core/src/networking/NetworkInstanceProvisioning.tsx @@ -37,7 +37,6 @@ import { } from '@etherealengine/client-core/src/common/services/MediaInstanceConnectionService' import { ChannelService, ChannelState } from '@etherealengine/client-core/src/social/services/ChannelService' import { LocationState } from '@etherealengine/client-core/src/social/services/LocationService' -import { EngineState } from '@etherealengine/engine/src/ecs/classes/EngineState' import { NetworkState } from '@etherealengine/engine/src/networking/NetworkState' import { getMutableState, none, useHookstate } from '@etherealengine/hyperflux' @@ -56,7 +55,6 @@ import MessagesMenu from '../user/components/UserMenu/menus/MessagesMenu' export const WorldInstanceProvisioning = () => { const locationState = useHookstate(getMutableState(LocationState)) const isUserBanned = locationState.currentLocation.selfUserBanned.value - const engineState = useHookstate(getMutableState(EngineState)) const worldNetwork = NetworkState.worldNetwork const worldNetworkState = useWorldNetwork() @@ -69,8 +67,6 @@ export const WorldInstanceProvisioning = () => { // Once we have the location, provision the instance server useEffect(() => { - if (!engineState.sceneLoaded.value || locationInstances.keys.length) return - const currentLocation = locationState.currentLocation.location const hasJoined = !!worldNetwork @@ -106,7 +102,7 @@ export const WorldInstanceProvisioning = () => { ) } } - }, [engineState.sceneLoaded, locationState.currentLocation.location, locationInstances.keys]) + }, [locationState.currentLocation.location]) // Populate the URL with the room code and instance id useEffect(() => { diff --git a/packages/client/src/pages/capture/capture.tsx b/packages/client/src/pages/capture/capture.tsx index 1516eb667e8..8760317b19d 100755 --- a/packages/client/src/pages/capture/capture.tsx +++ b/packages/client/src/pages/capture/capture.tsx @@ -23,26 +23,23 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import React from 'react' +import React, { useEffect } from 'react' import { useParams } from 'react-router-dom' import { NotificationService } from '@etherealengine/client-core/src/common/services/NotificationService' -import { - useLoadEngineWithScene, - useOfflineNetwork, - useOnlineNetwork -} from '@etherealengine/client-core/src/components/World/EngineHooks' -import { useLoadLocation, useLoadScene } from '@etherealengine/client-core/src/components/World/LoadLocationScene' +import { useOfflineNetwork, useOnlineNetwork } from '@etherealengine/client-core/src/components/World/EngineHooks' import { useRemoveEngineCanvas } from '@etherealengine/client-core/src/hooks/useRemoveEngineCanvas' import { AuthService } from '@etherealengine/client-core/src/user/services/AuthService' import { PresentationSystemGroup } from '@etherealengine/engine/src/ecs/functions/EngineFunctions' import { defineSystem } from '@etherealengine/engine/src/ecs/functions/SystemFunctions' import { ECSRecordingActions } from '@etherealengine/engine/src/recording/ECSRecordingSystem' -import { defineActionQueue } from '@etherealengine/hyperflux' +import { defineActionQueue, getMutableState, useHookstate } from '@etherealengine/hyperflux' import CaptureUI from '@etherealengine/ui/src/pages/Capture' +import { LocationService, LocationState } from '@etherealengine/client-core/src/social/services/LocationService' import '@etherealengine/client-core/src/world/ClientNetworkModule' import '@etherealengine/engine/src/EngineModule' +import { AppLoadingState, AppLoadingStates } from '@etherealengine/engine/src/common/AppLoadingService' const ecsRecordingErrorActionQueue = defineActionQueue(ECSRecordingActions.error.matches) @@ -57,6 +54,7 @@ const NotifyRecordingErrorSystem = defineSystem({ }) export const CaptureLocation = () => { + const locationState = useHookstate(getMutableState(LocationState)) useRemoveEngineCanvas() const params = useParams() @@ -64,13 +62,17 @@ export const CaptureLocation = () => { const locationName = params?.locationName as string | undefined const offline = !locationName - useLoadEngineWithScene({ spectate: true }) - - if (offline) { - useLoadScene({ projectName: 'default-project', sceneName: 'default' }) - } else { - useLoadLocation({ locationName: params.locationName! }) - } + useEffect(() => { + if (locationName) LocationState.setLocationName(locationName) + getMutableState(AppLoadingState).merge({ + state: AppLoadingStates.SUCCESS, + loaded: true + }) + }, []) + + useEffect(() => { + if (locationState.locationName.value) LocationService.getLocationByName(locationState.locationName.value) + }, [locationState.locationName.value]) if (offline) { useOfflineNetwork() diff --git a/packages/engine/src/common/AppLoadingService.ts b/packages/engine/src/common/AppLoadingService.ts index aee7b7f7af7..a53313122de 100755 --- a/packages/engine/src/common/AppLoadingService.ts +++ b/packages/engine/src/common/AppLoadingService.ts @@ -34,6 +34,7 @@ export const AppLoadingStates = { type AppLoadingStatesType = (typeof AppLoadingStates)[keyof typeof AppLoadingStates] +/** @deprecated @todo replace with scene loading state directly */ export const AppLoadingState = defineState({ name: 'AppLoadingState', initial: () => ({ diff --git a/packages/engine/src/ecs/functions/EntityTree.ts b/packages/engine/src/ecs/functions/EntityTree.ts index 834f14657cb..dd85ee7e3aa 100644 --- a/packages/engine/src/ecs/functions/EntityTree.ts +++ b/packages/engine/src/ecs/functions/EntityTree.ts @@ -34,6 +34,7 @@ import { getComponent, getMutableComponent, getOptionalComponent, + getOptionalMutableComponent, hasComponent, removeComponent, setComponent @@ -76,10 +77,14 @@ export const EntityTreeComponent = defineComponent({ // If a previous parentEntity, remove this entity from its children if (currentParentEntity && currentParentEntity !== json.parentEntity) { - const oldParent = getMutableComponent(currentParentEntity, EntityTreeComponent) - const parentChildIndex = oldParent.children.value.findIndex((child) => child === entity) - const children = oldParent.children.get(NO_PROXY) - oldParent.children.set([...children.slice(0, parentChildIndex), ...children.slice(parentChildIndex + 1)]) + if (entityExists(currentParentEntity)) { + const oldParent = getOptionalMutableComponent(currentParentEntity, EntityTreeComponent) + if (oldParent) { + const parentChildIndex = oldParent.children.value.findIndex((child) => child === entity) + const children = oldParent.children.get(NO_PROXY) + oldParent.children.set([...children.slice(0, parentChildIndex), ...children.slice(parentChildIndex + 1)]) + } + } } // set new data @@ -92,7 +97,7 @@ export const EntityTreeComponent = defineComponent({ if (matchesEntityUUID.test(json?.uuid) && !hasComponent(entity, UUIDComponent)) setComponent(entity, UUIDComponent, json.uuid) - if (parentEntity) { + if (parentEntity && entityExists(parentEntity)) { if (!hasComponent(parentEntity, EntityTreeComponent)) setComponent(parentEntity, EntityTreeComponent) const parentState = getMutableComponent(parentEntity, EntityTreeComponent) diff --git a/packages/engine/src/interaction/systems/GrabbableSystem.tsx b/packages/engine/src/interaction/systems/GrabbableSystem.tsx index 0f169781595..6a011926c61 100644 --- a/packages/engine/src/interaction/systems/GrabbableSystem.tsx +++ b/packages/engine/src/interaction/systems/GrabbableSystem.tsx @@ -197,12 +197,11 @@ export function transferAuthorityOfObjectReceptor( // since grabbables are all client authoritative, we don't need to recompute this for all users export function grabbableQueryAll(grabbableEntity: Entity) { - const grabberComponent = getComponent(grabbableEntity, GrabbedComponent) - const grabbedComponent = getComponent(grabbableEntity, GrabbedComponent) + if (!grabbedComponent) return const attachmentPoint = grabbedComponent.attachmentPoint - const target = getHandTarget(grabberComponent.grabberEntity, attachmentPoint ?? 'right')! + const target = getHandTarget(grabbedComponent.grabberEntity, attachmentPoint ?? 'right')! const rigidbodyComponent = getComponent(grabbableEntity, RigidBodyComponent) diff --git a/packages/engine/src/mocap/MotionCaptureSystem.ts b/packages/engine/src/mocap/MotionCaptureSystem.ts index 9c8a0de9202..aef71882111 100644 --- a/packages/engine/src/mocap/MotionCaptureSystem.ts +++ b/packages/engine/src/mocap/MotionCaptureSystem.ts @@ -105,8 +105,10 @@ const execute = () => { // for now, it is unnecessary to compute anything on the server if (!isClient) return const network = NetworkState.worldNetwork + if (!network) return + for (const [peerID, mocapData] of timeSeriesMocapData) { - if (!network?.peers?.[peerID] || timeSeriesMocapLastSeen.get(peerID)! < Date.now() - 1000) { + if (!network.peers[peerID] || timeSeriesMocapLastSeen.get(peerID)! < Date.now() - 1000) { timeSeriesMocapData.delete(peerID) timeSeriesMocapLastSeen.delete(peerID) } @@ -126,6 +128,7 @@ const execute = () => { for (const entity of motionCaptureQuery()) { const peers = Object.keys(network.peers).find((peerID: PeerID) => timeSeriesMocapData.has(peerID)) const rigComponent = getComponent(entity, AvatarRigComponent) + if (!rigComponent.normalizedRig) continue const worldHipsParent = rigComponent.normalizedRig.hips.node.parent if (!peers) { removeComponent(entity, MotionCaptureRigComponent) diff --git a/packages/engine/src/mocap/poseToInput.ts b/packages/engine/src/mocap/poseToInput.ts index fa9f58ebef0..d03f0e57cd3 100644 --- a/packages/engine/src/mocap/poseToInput.ts +++ b/packages/engine/src/mocap/poseToInput.ts @@ -41,6 +41,8 @@ let poseHoldTimer = 0 export const evaluatePose = (entity: Entity) => { const rig = getComponent(entity, AvatarRigComponent).normalizedRig + if (!rig) return + const deltaSeconds = getState(EngineState).deltaSeconds const pose = getMutableComponent(entity, MotionCapturePoseComponent) if (!MotionCaptureRigComponent.solvingLowerBody[entity]) return 'none' diff --git a/packages/engine/src/networking/functions/WorldNetworkAction.ts b/packages/engine/src/networking/functions/WorldNetworkAction.ts index dd173f4d5ef..a8f292f424c 100644 --- a/packages/engine/src/networking/functions/WorldNetworkAction.ts +++ b/packages/engine/src/networking/functions/WorldNetworkAction.ts @@ -82,6 +82,7 @@ export class WorldNetworkAction { ownerId: matchesUserId, networkId: matchesNetworkId, newAuthority: matchesPeerID, - $topic: NetworkTopics.world + $topic: NetworkTopics.world, + $cache: true }) } diff --git a/packages/engine/src/networking/state/EntityNetworkState.test.ts b/packages/engine/src/networking/state/EntityNetworkState.test.ts index fd6e5188a8f..bc1a8f2859f 100644 --- a/packages/engine/src/networking/state/EntityNetworkState.test.ts +++ b/packages/engine/src/networking/state/EntityNetworkState.test.ts @@ -49,11 +49,7 @@ import { NetworkObjectComponent, NetworkObjectOwnedTag } from '../components/Net import { NetworkPeerFunctions } from '../functions/NetworkPeerFunctions' import { WorldNetworkAction } from '../functions/WorldNetworkAction' import { NetworkState } from '../NetworkState' -import { - EntityNetworkState, - receiveRequestAuthorityOverObject, - receiveTransferAuthorityOfObject -} from './EntityNetworkState' +import { EntityNetworkState, receiveRequestAuthorityOverObject } from './EntityNetworkState' describe('EntityNetworkState', () => { beforeEach(async () => { @@ -277,10 +273,6 @@ describe('EntityNetworkState', () => { assert.equal(getComponent(networkObjectEntitiesBefore[0], NetworkObjectComponent).authorityPeerID, peerID) assert.equal(hasComponent(networkObjectEntitiesBefore[0], NetworkObjectOwnedTag), true) - const transferAuthorityOfObjectQueue = ActionFunctions.defineActionQueue( - WorldNetworkAction.transferAuthorityOfObject.matches - ) - receiveRequestAuthorityOverObject( WorldNetworkAction.requestAuthorityOverObject({ $from: userId, @@ -293,7 +285,7 @@ describe('EntityNetworkState', () => { ActionFunctions.applyIncomingActions() - for (const action of transferAuthorityOfObjectQueue()) receiveTransferAuthorityOfObject(action) + await act(() => receiveActions(EntityNetworkState)) const networkObjectEntitiesAfter = networkObjectQuery() const networkObjectOwnedEntitiesAfter = networkObjectOwnedQuery() @@ -351,10 +343,6 @@ describe('EntityNetworkState', () => { assert.equal(getComponent(networkObjectEntitiesBefore[0], NetworkObjectComponent).authorityPeerID, peerID) assert.equal(hasComponent(networkObjectEntitiesBefore[0], NetworkObjectOwnedTag), false) - const transferAuthorityOfObjectQueue = ActionFunctions.defineActionQueue( - WorldNetworkAction.transferAuthorityOfObject.matches - ) - receiveRequestAuthorityOverObject( WorldNetworkAction.requestAuthorityOverObject({ $from: userId, // from user @@ -366,9 +354,8 @@ describe('EntityNetworkState', () => { ) applyIncomingActions() - await act(() => receiveActions(EntityNetworkState)) - for (const action of transferAuthorityOfObjectQueue()) receiveTransferAuthorityOfObject(action) + await act(() => receiveActions(EntityNetworkState)) const networkObjectEntitiesAfter = networkObjectQuery() const networkObjectOwnedEntitiesAfter = networkObjectOwnedQuery() diff --git a/packages/engine/src/networking/state/EntityNetworkState.tsx b/packages/engine/src/networking/state/EntityNetworkState.tsx index 01990689c1b..994c81a34fe 100644 --- a/packages/engine/src/networking/state/EntityNetworkState.tsx +++ b/packages/engine/src/networking/state/EntityNetworkState.tsx @@ -85,7 +85,6 @@ export const EntityNetworkState = defineState({ }) } ], - [ WorldNetworkAction.destroyObject, (state, action: typeof WorldNetworkAction.destroyObject.matches._TYPE) => { @@ -106,10 +105,23 @@ export const EntityNetworkState = defineState({ if (!entity) return removeEntity(entity) } + ], + [ + WorldNetworkAction.transferAuthorityOfObject, + (state, action: typeof WorldNetworkAction.transferAuthorityOfObject.matches._TYPE) => { + const entity = NetworkObjectComponent.getNetworkObject(action.ownerId, action.networkId) + if (!entity) return + getMutableComponent(entity, NetworkObjectComponent).authorityPeerID.set(action.newAuthority) + } ] ] }) +/** + * Only the transfer needs to be event sourced + * @param action + * @returns + */ export const receiveRequestAuthorityOverObject = ( action: typeof WorldNetworkAction.requestAuthorityOverObject.matches._TYPE ) => { @@ -135,29 +147,12 @@ export const receiveRequestAuthorityOverObject = ( ) } -export const receiveTransferAuthorityOfObject = ( - action: typeof WorldNetworkAction.transferAuthorityOfObject.matches._TYPE -) => { - // Authority request can only be processed by owner - if (action.$from !== action.ownerId) return - - const entity = NetworkObjectComponent.getNetworkObject(action.ownerId, action.networkId) - if (!entity) - return console.log( - `Warning - tried to get entity belonging to ${action.ownerId} with ID ${action.networkId}, but it doesn't exist` - ) - - getMutableComponent(entity, NetworkObjectComponent).authorityPeerID.set(action.newAuthority) -} - const requestAuthorityOverObjectQueue = defineActionQueue(WorldNetworkAction.requestAuthorityOverObject.matches) -const transferAuthorityOfObjectQueue = defineActionQueue(WorldNetworkAction.transferAuthorityOfObject.matches) const execute = () => { receiveActions(EntityNetworkState) for (const action of requestAuthorityOverObjectQueue()) receiveRequestAuthorityOverObject(action) - for (const action of transferAuthorityOfObjectQueue()) receiveTransferAuthorityOfObject(action) } export const EntityNetworkStateSystem = defineSystem({ diff --git a/packages/engine/src/recording/ECSRecordingSystem.ts b/packages/engine/src/recording/ECSRecordingSystem.ts index 86701b50650..3a109141c42 100644 --- a/packages/engine/src/recording/ECSRecordingSystem.ts +++ b/packages/engine/src/recording/ECSRecordingSystem.ts @@ -651,7 +651,7 @@ export const onStartPlayback = async (action: ReturnType { resizeCanvas() - videoRef.current!.play() - canvasCtxRef.current = canvasRef.current!.getContext('2d')! + if (videoRef.current) videoRef.current.play() + if (canvasRef.current) canvasCtxRef.current = canvasRef.current.getContext('2d')! }) }, [videoRef.current]) @@ -537,6 +540,7 @@ export const PlaybackControls = (props: { durationSeconds: number }) => { const PlaybackMode = () => { const recordingID = useHookstate(getMutableState(PlaybackState).recordingID) + const locationState = useHookstate(getMutableState(LocationState)) const recording = useGet(recordingPath, recordingID.value!) @@ -544,14 +548,29 @@ const PlaybackMode = () => { recording.refetch() }, []) + /** + * Load scene in, and auto-unload upon recording stop + * @todo - wait until scene has loaded to start playback + */ + useEffect(() => { + const scenePath = locationState.currentLocation.location.sceneId.value + if (!scenePath) return + const cleanup = SceneServices.setCurrentScene(scenePath) + return () => { + cleanup() + // hack + getMutableState(EngineState).sceneLoaded.set(false) + } + }, [locationState]) + + useLocationSpawnAvatarWithDespawn() + const ActiveRecording = () => { const data = recording.data! const startTime = new Date(data.createdAt).getTime() const endTime = new Date(data.updatedAt).getTime() const durationSeconds = (endTime - startTime) / 1000 - useLocationSpawnAvatarWithDespawn() - // get all video resources, paired with motion capture data if it exists const videoPlaybackPairs = data.resources.reduce( (acc, r) => { @@ -604,7 +623,7 @@ const PlaybackMode = () => { const CapturePageState = defineState({ name: 'CapturePageState', initial: { - mode: 'playback' as 'playback' | 'capture' + mode: 'capture' as 'playback' | 'capture' }, onCreate: () => { syncStateWithLocalStorage(CapturePageState, ['mode'])