fix: restore VR support. Fix rotation / position camera bugs

Author: zardoy

renderer/viewer/lib/worldrendererCommon.ts

@@ -43,6 +43,7 @@ export const defaultWorldRendererConfig = {
   starfield: true,
   addChunksBatchWaitTime: 200,
   vrSupport: true,
+  vrPageGameRendering: true,
   renderEntities: true,
   fov: 75,
   fetchPlayerSkins: true,

renderer/viewer/three/cameraShake.ts

@@ -1,4 +1,5 @@
 import * as THREE from 'three'
+import { WorldRendererThree } from './worldrendererThree'
 
 export class CameraShake {
   private rollAngle = 0
@@ -8,7 +9,7 @@ export class CameraShake {
   private basePitch = 0
   private baseYaw = 0
 
-  constructor (public camera: THREE.Camera, public onRenderCallbacks: Array<() => void>) {
+  constructor (public worldRenderer: WorldRendererThree, public onRenderCallbacks: Array<() => void>) {
     onRenderCallbacks.push(() => {
       this.update()
     })
@@ -62,14 +63,21 @@ export class CameraShake {
       }
     }
 
-    // Create rotation quaternions
-    const pitchQuat = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(1, 0, 0), this.basePitch)
-    const yawQuat = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), this.baseYaw)
-    const rollQuat = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 0, 1), THREE.MathUtils.degToRad(this.rollAngle))
+    const camera = this.worldRenderer.cameraGroupVr || this.worldRenderer.camera
 
-    // Combine rotations in the correct order: pitch -> yaw -> roll
-    const finalQuat = yawQuat.multiply(pitchQuat).multiply(rollQuat)
-    this.camera.setRotationFromQuaternion(finalQuat)
+    if (this.worldRenderer.cameraGroupVr) {
+      // For VR camera, only apply yaw rotation
+      const yawQuat = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), this.baseYaw)
+      camera.setRotationFromQuaternion(yawQuat)
+    } else {
+      // For regular camera, apply all rotations
+      const pitchQuat = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(1, 0, 0), this.basePitch)
+      const yawQuat = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), this.baseYaw)
+      const rollQuat = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 0, 1), THREE.MathUtils.degToRad(this.rollAngle))
+      // Combine rotations in the correct order: pitch -> yaw -> roll
+      const finalQuat = yawQuat.multiply(pitchQuat).multiply(rollQuat)
+      camera.setRotationFromQuaternion(finalQuat)
+    }
   }
 
   private easeOut (t: number): number {

renderer/viewer/three/documentRenderer.ts

@@ -3,6 +3,7 @@ import Stats from 'stats.js'
 import StatsGl from 'stats-gl'
 import * as tween from '@tweenjs/tween.js'
 import { GraphicsBackendConfig, GraphicsInitOptions } from '../../../src/appViewer'
+import { WorldRendererConfig } from '../lib/worldrendererCommon'
 
 export class DocumentRenderer {
   readonly canvas = document.createElement('canvas')
@@ -23,6 +24,7 @@ export class DocumentRenderer {
   droppedFpsPercentage: number
   config: GraphicsBackendConfig
   onRender = [] as Array<(sizeChanged: boolean) => void>
+  inWorldRenderingConfig: WorldRendererConfig | undefined
 
   constructor (initOptions: GraphicsInitOptions) {
     this.config = initOptions.config
@@ -94,7 +96,7 @@ export class DocumentRenderer {
       if (this.disconnected) return
       this.animationFrameId = requestAnimationFrame(animate)
 
-      if (this.paused) return
+      if (this.paused || (this.renderer.xr.isPresenting && !this.inWorldRenderingConfig?.vrPageGameRendering)) return
 
       // Handle FPS limiting
       if (this.config.fpsLimit) {
@@ -117,18 +119,7 @@ export class DocumentRenderer {
         sizeChanged = true
       }
 
-      this.preRender()
-      this.stats.markStart()
-      tween.update()
-      if (!window.freezeRender) {
-        this.render(sizeChanged)
-      }
-      for (const fn of this.onRender) {
-        fn(sizeChanged)
-      }
-      this.renderedFps++
-      this.stats.markEnd()
-      this.postRender()
+      this.frameRender(sizeChanged)
 
       // Update stats visibility each frame
       if (this.config.statsVisible !== undefined) {
@@ -139,6 +130,21 @@ export class DocumentRenderer {
     animate()
   }
 
+  frameRender (sizeChanged: boolean) {
+    this.preRender()
+    this.stats.markStart()
+    tween.update()
+    if (!window.freezeRender) {
+      this.render(sizeChanged)
+    }
+    for (const fn of this.onRender) {
+      fn(sizeChanged)
+    }
+    this.renderedFps++
+    this.stats.markEnd()
+    this.postRender()
+  }
+
   setPaused (paused: boolean) {
     this.paused = paused
   }

renderer/viewer/three/graphicsBackend.ts

@@ -8,6 +8,7 @@ import supportedVersions from '../../../src/supportedVersions.mjs'
 import { WorldRendererThree } from './worldrendererThree'
 import { DocumentRenderer } from './documentRenderer'
 import { PanoramaRenderer } from './panorama'
+import { initVR } from './world/vr'
 
 // https://discourse.threejs.org/t/updates-to-color-management-in-three-js-r152/50791
 THREE.ColorManagement.enabled = false
@@ -87,10 +88,12 @@ const createGraphicsBackend: GraphicsBackendLoader = (initOptions: GraphicsInitO
       panoramaRenderer = null
     }
     worldRenderer = new WorldRendererThree(documentRenderer.renderer, initOptions, displayOptions)
+    void initVR(worldRenderer, documentRenderer)
     await worldRenderer.worldReadyPromise
     documentRenderer.render = (sizeChanged: boolean) => {
       worldRenderer?.render(sizeChanged)
     }
+    documentRenderer.inWorldRenderingConfig = displayOptions.inWorldRenderingConfig
     window.world = worldRenderer
     callModsMethod('worldReady', worldRenderer)
   }

renderer/viewer/three/world/vr.ts

@@ -4,8 +4,9 @@ import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerM
 import { buttonMap as standardButtonsMap } from 'contro-max/build/gamepad'
 import * as THREE from 'three'
 import { WorldRendererThree } from '../worldrendererThree'
+import { DocumentRenderer } from '../documentRenderer'
 
-export async function initVR (worldRenderer: WorldRendererThree) {
+export async function initVR (worldRenderer: WorldRendererThree, documentRenderer: DocumentRenderer) {
   if (!('xr' in navigator) || !worldRenderer.worldRendererConfig.vrSupport) return
   const { renderer } = worldRenderer
 
@@ -26,12 +27,13 @@ export async function initVR (worldRenderer: WorldRendererThree) {
 
   function enableVr () {
     renderer.xr.enabled = true
+    // renderer.xr.setReferenceSpaceType('local-floor')
     worldRenderer.reactiveState.preventEscapeMenu = true
   }
 
   function disableVr () {
     renderer.xr.enabled = false
-    worldRenderer.cameraObjectOverride = undefined
+    worldRenderer.cameraGroupVr = undefined
     worldRenderer.reactiveState.preventEscapeMenu = false
     worldRenderer.scene.remove(user)
     vrButtonContainer.hidden = true
@@ -189,24 +191,21 @@ export async function initVR (worldRenderer: WorldRendererThree) {
     }
 
     // appViewer.backend?.updateCamera(null, yawOffset, 0)
-    worldRenderer.updateCamera(null, bot.entity.yaw, bot.entity.pitch)
+    // worldRenderer.updateCamera(null, bot.entity.yaw, bot.entity.pitch)
 
     // todo restore this logic (need to preserve ability to move camera)
     // const xrCamera = renderer.xr.getCamera()
     // const d = xrCamera.getWorldDirection(new THREE.Vector3())
     // bot.entity.yaw = Math.atan2(-d.x, -d.z)
     // bot.entity.pitch = Math.asin(d.y)
 
-    // todo ?
-    // bot.physics.stepHeight = 1
-
-    worldRenderer.render()
+    documentRenderer.frameRender(false)
   })
   renderer.xr.addEventListener('sessionstart', () => {
-    worldRenderer.cameraObjectOverride = user
+    worldRenderer.cameraGroupVr = user
   })
   renderer.xr.addEventListener('sessionend', () => {
-    worldRenderer.cameraObjectOverride = undefined
+    worldRenderer.cameraGroupVr = undefined
   })
 
   worldRenderer.abortController.signal.addEventListener('abort', disableVr)

renderer/viewer/three/worldrendererThree.ts

@@ -20,7 +20,6 @@ import { armorModel } from './entity/armorModels'
 import { disposeObject } from './threeJsUtils'
 import { CursorBlock } from './world/cursorBlock'
 import { getItemUv } from './appShared'
-import { initVR } from './world/vr'
 import { Entities } from './entities'
 import { ThreeJsSound } from './threeJsSound'
 import { CameraShake } from './cameraShake'
@@ -42,7 +41,7 @@ export class WorldRendererThree extends WorldRendererCommon {
   ambientLight = new THREE.AmbientLight(0xcc_cc_cc)
   directionalLight = new THREE.DirectionalLight(0xff_ff_ff, 0.5)
   entities = new Entities(this)
-  cameraObjectOverride?: THREE.Object3D // for xr
+  cameraGroupVr?: THREE.Object3D
   material = new THREE.MeshLambertMaterial({ vertexColors: true, transparent: true, alphaTest: 0.1 })
   itemsTexture: THREE.Texture
   cursorBlock = new CursorBlock(this)
@@ -91,10 +90,9 @@ export class WorldRendererThree extends WorldRendererCommon {
     this.addDebugOverlay()
     this.resetScene()
     void this.init()
-    void initVR(this)
 
     this.soundSystem = new ThreeJsSound(this)
-    this.cameraShake = new CameraShake(this.camera, this.onRender)
+    this.cameraShake = new CameraShake(this, this.onRender)
     this.media = new ThreeJsMedia(this)
     // this.fountain = new Fountain(this.scene, this.scene, {
     //   position: new THREE.Vector3(0, 10, 0),
@@ -106,6 +104,10 @@ export class WorldRendererThree extends WorldRendererCommon {
     this.worldSwitchActions()
   }
 
+  get cameraObject () {
+    return this.cameraGroupVr || this.camera
+  }
+
   worldSwitchActions () {
     this.onWorldSwitched.push(() => {
       // clear custom blocks
@@ -301,7 +303,7 @@ export class WorldRendererThree extends WorldRendererCommon {
 
   updateViewerPosition (pos: Vec3): void {
     this.viewerPosition = pos
-    const cameraPos = this.camera.position.toArray().map(x => Math.floor(x / 16)) as [number, number, number]
+    const cameraPos = this.cameraObject.position.toArray().map(x => Math.floor(x / 16)) as [number, number, number]
     this.cameraSectionPos = new Vec3(...cameraPos)
     // eslint-disable-next-line guard-for-in
     for (const key in this.sectionObjects) {
@@ -429,10 +431,8 @@ export class WorldRendererThree extends WorldRendererCommon {
   }
 
   setFirstPersonCamera (pos: Vec3 | null, yaw: number, pitch: number) {
-    const cam = this.cameraObjectOverride || this.camera
     const yOffset = this.displayOptions.playerState.getEyeHeight()
 
-    this.camera = cam as THREE.PerspectiveCamera
     this.updateCamera(pos?.offset(0, yOffset, 0) ?? null, yaw, pitch)
     this.media.tryIntersectMedia()
   }
@@ -445,7 +445,11 @@ export class WorldRendererThree extends WorldRendererCommon {
     // }
 
     if (pos) {
-      new tweenJs.Tween(this.camera.position).to({ x: pos.x, y: pos.y, z: pos.z }, 50).start()
+      if (this.renderer.xr.isPresenting) {
+        pos.y -= this.camera.position.y // Fix Y position of camera in world
+      }
+
+      new tweenJs.Tween(this.cameraObject.position).to({ x: pos.x, y: pos.y, z: pos.z }, 50).start()
       // this.freeFlyState.position = pos
     }
     this.cameraShake.setBaseRotation(pitch, yaw)
@@ -467,13 +471,13 @@ export class WorldRendererThree extends WorldRendererCommon {
     this.entities.render()
 
     // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style
-    const cam = this.camera instanceof THREE.Group ? this.camera.children.find(child => child instanceof THREE.PerspectiveCamera) as THREE.PerspectiveCamera : this.camera
+    const cam = this.cameraGroupVr instanceof THREE.Group ? this.cameraGroupVr.children.find(child => child instanceof THREE.PerspectiveCamera) as THREE.PerspectiveCamera : this.camera
     this.renderer.render(this.scene, cam)
 
-    if (this.displayOptions.inWorldRenderingConfig.showHand && !this.playerState.shouldHideHand /*  && !this.freeFlyMode */) {
-      this.holdingBlock.render(this.camera, this.renderer, this.ambientLight, this.directionalLight)
-      this.holdingBlockLeft.render(this.camera, this.renderer, this.ambientLight, this.directionalLight)
-    }
+    // if (this.displayOptions.inWorldRenderingConfig.showHand && !this.playerState.shouldHideHand /*  && !this.freeFlyMode */) {
+    //   this.holdingBlock.render(this.camera, this.renderer, this.ambientLight, this.directionalLight)
+    //   this.holdingBlockLeft.render(this.camera, this.renderer, this.ambientLight, this.directionalLight)
+    // }
 
     for (const fountain of this.fountains) {
       if (this.sectionObjects[fountain.sectionId] && !this.sectionObjects[fountain.sectionId].foutain) {

src/appViewer.ts

@@ -199,7 +199,7 @@ export class AppViewer {
   resetBackend (cleanState = false) {
     this.disconnectBackend(cleanState)
     if (this.backendLoader) {
-      this.loadBackend(this.backendLoader)
+      void this.loadBackend(this.backendLoader)
     }
   }
 

src/optionsGuiScheme.tsx

@@ -488,7 +488,11 @@ export const guiOptionsScheme: {
           </>
         )
       },
-      vrSupport: {}
+      vrSupport: {},
+      vrPageGameRendering: {
+        text: 'Page Game Rendering',
+        tooltip: 'Wether to continue rendering page even when vr is active.',
+      }
     },
   ],
   advanced: [

src/optionsStorage.ts

@@ -104,6 +104,7 @@ const defaultOptions = {
   autoJump: 'auto' as 'auto' | 'always' | 'never',
   autoParkour: false,
   vrSupport: true, // doesn't directly affect the VR mode, should only disable the button which is annoying to android users
+  vrPageGameRendering: false,
   renderDebug: 'basic' as 'none' | 'advanced' | 'basic',
 
   // advanced bot options

src/watchOptions.ts

@@ -80,6 +80,11 @@ export const watchOptionsAfterViewerInit = () => {
     updateFpsLimit(o)
   })
 
+  watchValue(options, o => {
+    appViewer.inWorldRenderingConfig.vrSupport = o.vrSupport
+    appViewer.inWorldRenderingConfig.vrPageGameRendering = o.vrPageGameRendering
+  })
+
   watchValue(options, (o, isChanged) => {
     appViewer.inWorldRenderingConfig.clipWorldBelowY = o.clipWorldBelowY
     appViewer.inWorldRenderingConfig.extraBlockRenderers = !o.disableSignsMapsSupport