diff --git a/framework/Source/BasicOperation.swift b/framework/Source/BasicOperation.swift index e0e288a8..25509e1b 100644 --- a/framework/Source/BasicOperation.swift +++ b/framework/Source/BasicOperation.swift @@ -52,10 +52,6 @@ open class BasicOperation: ImageProcessingOperation { public func newTextureAvailable(_ texture: Texture, fromSourceIndex: UInt) { let _ = textureInputSemaphore.wait(timeout:DispatchTime.distantFuture) - defer { - textureInputSemaphore.signal() - } - inputTextures[fromSourceIndex] = texture if (UInt(inputTextures.count) >= maximumInputs) || activatePassthroughOnNextFrame { @@ -76,10 +72,23 @@ open class BasicOperation: ImageProcessingOperation { uniformSettings["aspectRatio"] = firstInputTexture.aspectRatio(for: outputRotation) } - guard let commandBuffer = sharedMetalRenderingDevice.commandQueue.makeCommandBuffer() else {return} + guard + let commandBuffer = sharedMetalRenderingDevice.commandQueue.makeCommandBuffer(), + let outputTexture = Texture( + device:sharedMetalRenderingDevice.device, + orientation: .portrait, + width: outputWidth, + height: outputHeight, + timingStyle: firstInputTexture.timingStyle + ) + else { + assertionFailure("CommandBuffer or Texture creation failed") + removeTransientInputs() + textureInputSemaphore.signal() + updateTargetsWithTexture(firstInputTexture) + return + } - let outputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation: .portrait, width: outputWidth, height: outputHeight, timingStyle: firstInputTexture.timingStyle) - guard (!activatePassthroughOnNextFrame) else { // Use this to allow a bootstrap of cyclical processing, like with a low pass filter activatePassthroughOnNextFrame = false // TODO: Render rotated passthrough image here @@ -95,8 +104,21 @@ open class BasicOperation: ImageProcessingOperation { if let alternateRenderingFunction = metalPerformanceShaderPathway, useMetalPerformanceShaders { var rotatedInputTextures: [UInt:Texture] if (firstInputTexture.orientation.rotationNeeded(for:.portrait) != .noRotation) { - let rotationOutputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation: .portrait, width: outputWidth, height: outputHeight) - guard let rotationCommandBuffer = sharedMetalRenderingDevice.commandQueue.makeCommandBuffer() else {return} + guard + let rotationCommandBuffer = sharedMetalRenderingDevice.commandQueue.makeCommandBuffer(), + let rotationOutputTexture = Texture( + device:sharedMetalRenderingDevice.device, + orientation: .portrait, + width: outputWidth, + height: outputHeight + ) + else { + assertionFailure("CommandBuffer or Texture creation failed") + removeTransientInputs() + textureInputSemaphore.signal() + updateTargetsWithTexture(firstInputTexture) + return + } rotationCommandBuffer.renderQuad(pipelineState: sharedMetalRenderingDevice.passthroughRenderState, uniformSettings: uniformSettings, inputTextures: inputTextures, useNormalizedTextureCoordinates: useNormalizedTextureCoordinates, outputTexture: rotationOutputTexture) rotationCommandBuffer.commit() rotatedInputTextures = inputTextures diff --git a/framework/Source/Camera.swift b/framework/Source/Camera.swift index 137f3dce..2a841e0b 100644 --- a/framework/Source/Camera.swift +++ b/framework/Source/Camera.swift @@ -217,8 +217,8 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer outputWidth = bufferWidth outputHeight = bufferHeight } - let outputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation:.portrait, width:outputWidth, height:outputHeight, timingStyle: .videoFrame(timestamp: Timestamp(currentTime))) - + let outputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation:.portrait, width:outputWidth, height:outputHeight, timingStyle: .videoFrame(timestamp: Timestamp(currentTime)))! + convertYUVToRGB(pipelineState:self.yuvConversionRenderPipelineState!, lookupTable:self.yuvLookupTable, luminanceTexture:Texture(orientation: self.orientation ?? self.location.imageOrientation(), texture:luminanceTexture), chrominanceTexture:Texture(orientation: self.orientation ?? self.location.imageOrientation(), texture:chrominanceTexture), diff --git a/framework/Source/ImageGenerator.swift b/framework/Source/ImageGenerator.swift index a5a9c451..7bbbf1df 100644 --- a/framework/Source/ImageGenerator.swift +++ b/framework/Source/ImageGenerator.swift @@ -6,7 +6,7 @@ public class ImageGenerator: ImageSource { public init(size:Size) { self.size = size - internalTexture = Texture(device:sharedMetalRenderingDevice.device, orientation:.portrait, width:Int(size.width), height:Int(size.height), timingStyle:.stillImage) + internalTexture = Texture(device:sharedMetalRenderingDevice.device, orientation:.portrait, width:Int(size.width), height:Int(size.height), timingStyle:.stillImage)! } public func transmitPreviousImage(to target:ImageConsumer, atIndex:UInt) { diff --git a/framework/Source/MovieInput.swift b/framework/Source/MovieInput.swift index 4ff0aea2..a38f7e55 100644 --- a/framework/Source/MovieInput.swift +++ b/framework/Source/MovieInput.swift @@ -175,7 +175,7 @@ public class MovieInput: ImageSource { if let concreteLuminanceTextureRef = luminanceTextureRef, let concreteChrominanceTextureRef = chrominanceTextureRef, let luminanceTexture = CVMetalTextureGetTexture(concreteLuminanceTextureRef), let chrominanceTexture = CVMetalTextureGetTexture(concreteChrominanceTextureRef) { - let outputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation:.portrait, width:bufferWidth, height:bufferHeight, timingStyle:.videoFrame(timestamp:Timestamp(withSampleTime))) + let outputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation:.portrait, width:bufferWidth, height:bufferHeight, timingStyle:.videoFrame(timestamp:Timestamp(withSampleTime)))! convertYUVToRGB(pipelineState:self.yuvConversionRenderPipelineState, lookupTable:self.yuvLookupTable, luminanceTexture:Texture(orientation:.portrait, texture:luminanceTexture), diff --git a/framework/Source/MovieOutput.swift b/framework/Source/MovieOutput.swift index 23ac892e..51dfbae5 100644 --- a/framework/Source/MovieOutput.swift +++ b/framework/Source/MovieOutput.swift @@ -151,7 +151,7 @@ public class MovieOutput: ImageConsumer, AudioEncodingTarget { if (Int(round(self.size.width)) != texture.texture.width) && (Int(round(self.size.height)) != texture.texture.height) { let commandBuffer = sharedMetalRenderingDevice.commandQueue.makeCommandBuffer() - outputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation: .portrait, width: Int(round(self.size.width)), height: Int(round(self.size.height)), timingStyle: texture.timingStyle) + outputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation: .portrait, width: Int(round(self.size.width)), height: Int(round(self.size.height)), timingStyle: texture.timingStyle)! commandBuffer?.renderQuad(pipelineState: renderPipelineState, inputTextures: [0:texture], outputTexture: outputTexture) commandBuffer?.commit() diff --git a/framework/Source/Operations/CBRescaleEffect.swift b/framework/Source/Operations/CBRescaleEffect.swift index 321e4170..b1b4317c 100644 --- a/framework/Source/Operations/CBRescaleEffect.swift +++ b/framework/Source/Operations/CBRescaleEffect.swift @@ -31,14 +31,19 @@ public class CBRescaleEffect: BasicOperation { } let outputSize = calculateTargetSize(from: texture) - let outputTexture = Texture( + guard let outputTexture = Texture( device: sharedMetalRenderingDevice.device, orientation: texture.orientation, pixelFormat: texture.texture.pixelFormat, width: Int(outputSize.width), height: Int(outputSize.height), timingStyle: texture.timingStyle - ) + ) else { + assertionFailure("CommandBuffer or Texture creation failed") + removeTransientInputs() + updateTargetsWithTexture(texture) + return + } inputTextures[0] = texture internalRenderFunction(commandBuffer: commandBuffer, outputTexture: outputTexture) diff --git a/framework/Source/Operations/HistogramEqualization.swift b/framework/Source/Operations/HistogramEqualization.swift index d600d61d..7234a656 100644 --- a/framework/Source/Operations/HistogramEqualization.swift +++ b/framework/Source/Operations/HistogramEqualization.swift @@ -35,14 +35,19 @@ public class HistogramEqualization: ImageProcessingOperation { width: CGFloat(frameTexture.width), height: CGFloat(frameTexture.height) ) - let outputTexture = Texture( + guard let outputTexture = Texture( device: sharedMetalRenderingDevice.device, orientation: texture.orientation, pixelFormat: texture.texture.pixelFormat, width: Int(size.width), height: Int(size.height), timingStyle: texture.timingStyle - ) + ) else { + assertionFailure("CommandBuffer or Texture creation failed") + removeTransientInputs() + updateTargetsWithTexture(texture) + return + } inputTexture = texture renderer.render( @@ -51,6 +56,7 @@ public class HistogramEqualization: ImageProcessingOperation { ) inputTexture = nil + removeTransientInputs() updateTargetsWithTexture(outputTexture) } } diff --git a/framework/Source/PictureOutput.swift b/framework/Source/PictureOutput.swift index fb8a9d9e..770ba40f 100644 --- a/framework/Source/PictureOutput.swift +++ b/framework/Source/PictureOutput.swift @@ -50,9 +50,9 @@ public class PictureOutput: ImageConsumer { storedTexture = texture } - if let imageCallback = imageAvailableCallback { - let cgImageFromBytes = texture.cgImage() - + if let imageCallback = imageAvailableCallback, + let cgImageFromBytes = texture.cgImage() { + // TODO: Let people specify orientations #if canImport(UIKit) let image = UIImage(cgImage:cgImageFromBytes, scale:1.0, orientation:.up) @@ -61,15 +61,15 @@ public class PictureOutput: ImageConsumer { #endif imageCallback(image) - + if onlyCaptureNextFrame { imageAvailableCallback = nil } } - - if let imageCallback = encodedImageAvailableCallback { - let cgImageFromBytes = texture.cgImage() - + + if let imageCallback = encodedImageAvailableCallback, + let cgImageFromBytes = texture.cgImage() { + let imageData:Data #if canImport(UIKit) let image = UIImage(cgImage:cgImageFromBytes, scale:1.0, orientation:.up) diff --git a/framework/Source/Texture.swift b/framework/Source/Texture.swift index a47e2efc..a074607a 100644 --- a/framework/Source/Texture.swift +++ b/framework/Source/Texture.swift @@ -39,7 +39,7 @@ public class Texture { self.timingStyle = timingStyle } - public init(device:MTLDevice, orientation: ImageOrientation, pixelFormat: MTLPixelFormat = .bgra8Unorm, width: Int, height: Int, mipmapped:Bool = false, timingStyle: TextureTimingStyle = .stillImage) { + public init?(device:MTLDevice, orientation: ImageOrientation, pixelFormat: MTLPixelFormat = .bgra8Unorm, width: Int, height: Int, mipmapped:Bool = false, timingStyle: TextureTimingStyle = .stillImage) { let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .bgra8Unorm, width: width, height: height, @@ -47,7 +47,8 @@ public class Texture { textureDescriptor.usage = [.renderTarget, .shaderRead, .shaderWrite] guard let newTexture = sharedMetalRenderingDevice.device.makeTexture(descriptor: textureDescriptor) else { - fatalError("Could not create texture of size: (\(width), \(height))") + assertionFailure("Could not create texture of size: (\(width), \(height))") + return nil } self.orientation = orientation @@ -113,10 +114,25 @@ extension Texture { } extension Texture { - func cgImage() -> CGImage { + func cgImage() -> CGImage? { // Flip and swizzle image - guard let commandBuffer = sharedMetalRenderingDevice.commandQueue.makeCommandBuffer() else { fatalError("Could not create command buffer on image rendering.")} - let outputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation:self.orientation, width:self.texture.width, height:self.texture.height) + guard + let commandBuffer = sharedMetalRenderingDevice.commandQueue.makeCommandBuffer() + else { + assertionFailure("Could not create command buffer on image rendering.") + return nil + } + guard + let outputTexture = Texture( + device:sharedMetalRenderingDevice.device, + orientation: orientation, + width: texture.width, + height: texture.height + ) + else { + assertionFailure("MTLDevice makeTexture failed") + return nil + } commandBuffer.renderQuad(pipelineState:sharedMetalRenderingDevice.colorSwizzleRenderState, uniformSettings:nil, inputTextures:[0:self], useNormalizedTextureCoordinates:true, outputTexture:outputTexture) commandBuffer.commit() commandBuffer.waitUntilCompleted()