Skip to content
This repository has been archived by the owner on Jun 16, 2023. It is now read-only.

Commit

Permalink
improved & simplified depth occlusion feathering in AR PBR Example, a…
Browse files Browse the repository at this point in the history
…dded ARFeatheredDepthMaskGenerator
  • Loading branch information
rezaali committed May 16, 2023
1 parent 7040130 commit 105fc3d
Show file tree
Hide file tree
Showing 8 changed files with 144 additions and 53 deletions.
10 changes: 1 addition & 9 deletions Example/Assets/ARPBRRenderer/Pipelines/Post/Shaders.metal
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,5 @@ fragment float4 postFragment
float4 virtualSample = virtualTex.sample(s, in.uv);
virtualSample.rgb += mix(0.0, grainSample.rgb * grainSample.a, virtualSample.a * uniforms.grainAmount);

float4 finalColor = mix(realSample, virtualSample, virtualSample.a * depthMask);
return finalColor;

// return float4(depthMask, realSample.g, virtualSample.b, 1.0);
return mix(realSample, virtualSample, virtualSample.a * (1.0 - depthMask));
}





2 changes: 1 addition & 1 deletion Example/Assets/Shared/Pipelines/DepthMask/Shaders.metal
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ kernel void depthMaskUpdate
{
const float realDepth = realDepthTexture.read(gid);
const float virtualDepth = virtualDepthTexture.read(gid);
depthMaskTexture.write(step(realDepth, virtualDepth), gid);
depthMaskTexture.write(step(virtualDepth, realDepth), gid);
}
4 changes: 4 additions & 0 deletions Example/Example.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
E17A55BD28A6C9C6008F1DDB /* IcosahedronGenerator.c in Sources */ = {isa = PBXBuildFile; fileRef = E1CC099F2489797300B92285 /* IcosahedronGenerator.c */; };
E18A1EDB291EFB1D00A71453 /* PBRStandardMaterialRendererView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E18A1EDA291EFB1D00A71453 /* PBRStandardMaterialRendererView.swift */; };
E18FD5962A12B00F001376D8 /* ARDepthMaskGenerator.swift in Sources */ = {isa = PBXBuildFile; fileRef = E18FD5952A12B00F001376D8 /* ARDepthMaskGenerator.swift */; };
E18FD5982A12FE16001376D8 /* ARFeatheredDepthMaskGenerator.swift in Sources */ = {isa = PBXBuildFile; fileRef = E18FD5972A12FE16001376D8 /* ARFeatheredDepthMaskGenerator.swift */; };
E19D4C3429F9C00500DF993D /* ARBloomRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = E19D4C3329F9C00500DF993D /* ARBloomRenderer.swift */; };
E19D4C3629F9C02A00DF993D /* ARBloomRendererView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E19D4C3529F9C02A00DF993D /* ARBloomRendererView.swift */; };
E19D4C4929F9CA3F00DF993D /* ARSessionPublisher.swift in Sources */ = {isa = PBXBuildFile; fileRef = E19D4C4829F9CA3F00DF993D /* ARSessionPublisher.swift */; };
Expand Down Expand Up @@ -195,6 +196,7 @@
E182809B24C52061000810D1 /* ExtrudedTextRenderer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExtrudedTextRenderer.swift; sourceTree = "<group>"; };
E18A1EDA291EFB1D00A71453 /* PBRStandardMaterialRendererView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PBRStandardMaterialRendererView.swift; sourceTree = "<group>"; };
E18FD5952A12B00F001376D8 /* ARDepthMaskGenerator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ARDepthMaskGenerator.swift; sourceTree = "<group>"; };
E18FD5972A12FE16001376D8 /* ARFeatheredDepthMaskGenerator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ARFeatheredDepthMaskGenerator.swift; sourceTree = "<group>"; };
E1919A32248D9BC3008FD392 /* CubemapRenderer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CubemapRenderer.swift; sourceTree = "<group>"; };
E192EFCD24857EB600C7EE74 /* LiveCodeRenderer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LiveCodeRenderer.swift; sourceTree = "<group>"; };
E192F01F2485B16C00C7EE74 /* MatcapRenderer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MatcapRenderer.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -700,6 +702,7 @@
E19D4C4A29F9CF3100DF993D /* ARPlaneMesh.swift */,
E1ED89B12A017E3400DC085F /* ARDepthUpscaler.swift */,
E18FD5952A12B00F001376D8 /* ARDepthMaskGenerator.swift */,
E18FD5972A12FE16001376D8 /* ARFeatheredDepthMaskGenerator.swift */,
);
path = AR;
sourceTree = "<group>";
Expand Down Expand Up @@ -880,6 +883,7 @@
E11325CD28AEADAA0049CACC /* SuperShapesRenderer.swift in Sources */,
E1E767D028A6D41B005D75AB /* TextRendererView.swift in Sources */,
E1E7680128A70000005D75AB /* TextureComputeRendererView.swift in Sources */,
E18FD5982A12FE16001376D8 /* ARFeatheredDepthMaskGenerator.swift in Sources */,
E175243829F8734D00AD1A8F /* ARPBRRendererView.swift in Sources */,
E1E767FC28A6FCA9005D75AB /* FXAARenderer.swift in Sources */,
E1CB1F2029E656A6006FBDE8 /* ARBackgroundDepthRenderer.swift in Sources */,
Expand Down
48 changes: 14 additions & 34 deletions Example/Example/Renderers/AR/ARPBRRenderer.swift
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
#if os(iOS)
import ARKit
import Metal
import MetalPerformanceShaders
import MetalKit
import MetalPerformanceShaders

import Forge
import Satin
Expand Down Expand Up @@ -350,13 +350,8 @@ class ARPBRRenderer: BaseRenderer, MaterialDelegate {
lazy var camera = ARPerspectiveCamera(session: session, mtkView: mtkView, near: 0.01, far: 100.0)
lazy var renderer = Satin.Renderer(context: context)

var _updateTextures = true
var depthMaskTexture: MTLTexture?
var scaledDepthMaskTexture: MTLTexture?

var blurFilter: MPSImageGaussianBlur!
var scaleFilter: MPSImageBilinearScale!
var backgroundRenderer: ARBackgroundDepthRenderer!
var featheredDepthMaskGenerator: ARFeatheredDepthMaskGenerator!

lazy var postMaterial: PostMaterial = {
let material = PostMaterial(pipelinesURL: pipelinesURL)
Expand All @@ -365,7 +360,6 @@ class ARPBRRenderer: BaseRenderer, MaterialDelegate {
return material
}()

lazy var depthMaskGenerator = ARDepthMaskGenerator(device: device, width: Int(mtkView.drawableSize.width), height: Int(mtkView.drawableSize.height))
lazy var postProcessor = PostProcessor(context: Context(device, 1, colorPixelFormat), material: postMaterial)

lazy var startTime = getTime()
Expand All @@ -392,10 +386,6 @@ class ARPBRRenderer: BaseRenderer, MaterialDelegate {
override func setup() {
model.material.delegate = self

scaleFilter = MPSImageBilinearScale(device: device)
blurFilter = MPSImageGaussianBlur(device: device, sigma: 8)
blurFilter.edgeMode = .clamp

renderer.setClearColor(.zero)
renderer.depthStoreAction = .store

Expand All @@ -406,20 +396,22 @@ class ARPBRRenderer: BaseRenderer, MaterialDelegate {
mtkView: mtkView,
near: camera.near,
far: camera.far,
upscaleDepth: false,
upscaleDepth: true,
usePlaneDepth: false,
useMeshDepth: false
)

featheredDepthMaskGenerator = ARFeatheredDepthMaskGenerator(
device: device,
pixelFormat: .r8Unorm,
textureScale: 3,
blurSigma: 4
)
}

override func update() {
let time = getTime() - startTime
model.orientation = simd_quatf(angle: Float(time), axis: Satin.worldUpDirection)

if _updateTextures {
scaledDepthMaskTexture = createTexture("Scaled Depth Mask Texture", .r16Float, 3)
_updateTextures = false
}
}

override func draw(_ view: MTKView, _ commandBuffer: MTLCommandBuffer) {
Expand All @@ -443,33 +435,21 @@ class ARPBRRenderer: BaseRenderer, MaterialDelegate {

// Compare depth

var featheredDepthMaskTexture: MTLTexture?
if let realDepthTexture = backgroundRenderer.depthTexture,
let virtualDepthTexture = renderer.depthTexture
{
depthMaskTexture = depthMaskGenerator.encode(
featheredDepthMaskTexture = featheredDepthMaskGenerator.encode(
commandBuffer: commandBuffer,
realDepthTexture: realDepthTexture,
virtualDepthTexture: virtualDepthTexture
)

if let depthMaskTexture = depthMaskTexture, var scaledDepthMaskTexture = scaledDepthMaskTexture {
scaleFilter.encode(
commandBuffer: commandBuffer,
sourceTexture: depthMaskTexture,
destinationTexture: scaledDepthMaskTexture
)

blurFilter.encode(
commandBuffer: commandBuffer,
inPlaceTexture: &scaledDepthMaskTexture
)
}
}

// Post
postMaterial.backgroundTexture = backgroundRenderer.colorTexture
postMaterial.contentTexture = renderer.colorTexture
postMaterial.depthMaskTexture = scaledDepthMaskTexture
postMaterial.depthMaskTexture = featheredDepthMaskTexture
postMaterial.grainTexture = session.currentFrame?.cameraGrainTexture
postMaterial.grainIntensity = session.currentFrame?.cameraGrainIntensity ?? 0
postMaterial.time = Float(getTime() - startTime)
Expand All @@ -484,7 +464,7 @@ class ARPBRRenderer: BaseRenderer, MaterialDelegate {
renderer.resize(size)
backgroundRenderer.resize(size)
postProcessor.resize(size)
depthMaskGenerator.resize(size)
featheredDepthMaskGenerator.resize(size)
}

override func cleanup() {
Expand Down
8 changes: 5 additions & 3 deletions Example/Example/Utilities/AR/ARDepthMaskGenerator.swift
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,12 @@ public class ARDepthMaskGenerator {
}

private var compute: DepthMaskComputeSystem
private var pixelFormat: MTLPixelFormat

public init(device: MTLDevice, width: Int, height: Int) {
public init(device: MTLDevice, width: Int, height: Int, pixelFormat: MTLPixelFormat = .r16Float) {
self.pixelFormat = pixelFormat
let textureDescriptor: MTLTextureDescriptor = .texture2DDescriptor(
pixelFormat: .r16Float,
pixelFormat: pixelFormat,
width: width,
height: height,
mipmapped: false
Expand All @@ -61,7 +63,7 @@ public class ARDepthMaskGenerator {

public func resize(_ size: (width: Float, height: Float)) {
let textureDescriptor: MTLTextureDescriptor = .texture2DDescriptor(
pixelFormat: .r16Float,
pixelFormat: pixelFormat,
width: Int(size.width),
height: Int(size.height),
mipmapped: false
Expand Down
99 changes: 99 additions & 0 deletions Example/Example/Utilities/AR/ARFeatheredDepthMaskGenerator.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
//
// ARDepthMaskStencilGenerator.swift
// Example
//
// Created by Reza Ali on 5/15/23.
// Copyright © 2023 Hi-Rez. All rights reserved.
//

import Foundation
import Metal
import MetalPerformanceShaders

class ARFeatheredDepthMaskGenerator {
private var device: MTLDevice
private var pixelFormat: MTLPixelFormat
private var textureScale: Int

private var depthMaskGenerator: ARDepthMaskGenerator
private var scaleFilter: MPSImageBilinearScale
private var blurFilter: MPSImageGaussianBlur

private var _updateTextures = true
private var texture: MTLTexture?
private var scaledTexture: MTLTexture?

public init(device: MTLDevice, pixelFormat: MTLPixelFormat, textureScale: Int, blurSigma: Float) {
self.device = device
self.pixelFormat = pixelFormat
self.textureScale = textureScale

depthMaskGenerator = ARDepthMaskGenerator(device: device, width: 1, height: 1, pixelFormat: pixelFormat)

scaleFilter = MPSImageBilinearScale(device: device)
blurFilter = MPSImageGaussianBlur(device: device, sigma: blurSigma)
blurFilter.edgeMode = .clamp
}

public func encode(commandBuffer: MTLCommandBuffer, realDepthTexture: MTLTexture, virtualDepthTexture: MTLTexture) -> MTLTexture? {
if _updateTextures {
texture = createTexture(
label: "Feathered Depth Mask Texture",
pixelFormat: pixelFormat,
width: realDepthTexture.width,
height: realDepthTexture.height,
textureScale: textureScale
)

scaledTexture = createTexture(
label: "Feathered Scaled Depth Mask Texture",
pixelFormat: pixelFormat,
width: realDepthTexture.width,
height: realDepthTexture.height,
textureScale: textureScale
)
_updateTextures = false
}

if let depthMaskTexture = depthMaskGenerator.encode(
commandBuffer: commandBuffer,
realDepthTexture: realDepthTexture,
virtualDepthTexture: virtualDepthTexture
), let texture = texture, let scaledTexture = scaledTexture {
scaleFilter.encode(
commandBuffer: commandBuffer,
sourceTexture: depthMaskTexture,
destinationTexture: scaledTexture
)

blurFilter.encode(
commandBuffer: commandBuffer,
sourceTexture: scaledTexture,
destinationTexture: texture
)
}

return texture
}

public func resize(_ size: (width: Float, height: Float)) {
depthMaskGenerator.resize(size)
_updateTextures = true
}

private func createTexture(label: String, pixelFormat: MTLPixelFormat, width: Int, height: Int, textureScale: Int) -> MTLTexture? {
guard width > 0, height > 0 else { return nil }
let descriptor = MTLTextureDescriptor()
descriptor.pixelFormat = pixelFormat
descriptor.width = width / textureScale
descriptor.height = height / textureScale
descriptor.sampleCount = 1
descriptor.textureType = .type2D
descriptor.usage = [.renderTarget, .shaderRead, .shaderWrite]
descriptor.storageMode = .private
descriptor.resourceOptions = .storageModePrivate
guard let texture = device.makeTexture(descriptor: descriptor) else { return nil }
texture.label = label
return texture
}
}
1 change: 1 addition & 0 deletions Example/Example/Utilities/Shadows/MeshShadowRenderer.swift
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import Foundation
import Metal
import MetalPerformanceShaders
import Satin
import SatinCore

class MeshShadowRenderer {
public var texture: MTLTexture? {
Expand Down
25 changes: 19 additions & 6 deletions Example/Example/Utilities/Shadows/ObjectShadowRenderer.swift
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ class ObjectShadowRenderer {
private var processor: ShadowPostProcessor

public private(set) var texture: MTLTexture?
public private(set) var renderTexture: MTLTexture?
private var _updateTexture = true

private var blurFilter: MPSImageGaussianBlur?
Expand Down Expand Up @@ -129,7 +130,7 @@ class ObjectShadowRenderer {
renderer = Satin.Renderer(context: context)
renderer.label = "Object Shadow Renderer"

processor = ShadowPostProcessor(context: Context(context.device, 1, .rgba32Float))
processor = ShadowPostProcessor(context: Context(context.device, 1, .rgba16Float))
camera = OrthographicCamera()

renderer.setClearColor(.zero)
Expand All @@ -149,7 +150,6 @@ class ObjectShadowRenderer {
let renderables = getRenderables(object, true, false)
materialCache.removeAll(keepingCapacity: true)


for var renderable in renderables {
if let object = renderable as? Object {
materialCache[object] = renderable.material
Expand All @@ -168,19 +168,25 @@ class ObjectShadowRenderer {
camera: camera
)

if var texture = texture {
if let renderTexture = renderTexture,
var texture = texture
{
let srpd = MTLRenderPassDescriptor()
srpd.renderTargetWidth = resolution
srpd.renderTargetHeight = resolution
srpd.colorAttachments[0].texture = texture
srpd.colorAttachments[0].texture = renderTexture
processor.colorTexture = renderer.colorTexture
processor.depthTexture = renderer.depthTexture
processor.mesh.material?.set("Near Far", [camera.near, camera.far])
processor.mesh.material?.set("Color", color)
processor.draw(renderPassDescriptor: srpd, commandBuffer: commandBuffer)

if let blurFilter = blurFilter {
blurFilter.encode(commandBuffer: commandBuffer, inPlaceTexture: &texture)
blurFilter.encode(
commandBuffer: commandBuffer,
sourceTexture: renderTexture,
destinationTexture: texture
)
}

if let material = catcher.material as? BasicTextureMaterial {
Expand Down Expand Up @@ -231,7 +237,14 @@ class ObjectShadowRenderer {
device: context.device,
width: resolution,
height: resolution,
pixelFormat: .rgba32Float
pixelFormat: .rgba16Float
)

renderTexture = createTexture(
device: context.device,
width: resolution,
height: resolution,
pixelFormat: .rgba16Float
)

_updateTexture = false
Expand Down

0 comments on commit 105fc3d

Please sign in to comment.