Skip to content

Commit

Permalink
trying making record with RealityKit
Browse files Browse the repository at this point in the history 
using old view draw method, screen capture is doable, but too slow for video recording.
  • Loading branch information
@jerryrt
jerryrt committed Jul 3, 2019
1 parent 4fc85a3 commit 5f221a1
Show file tree
Hide file tree
Showing 8 changed files with 93 additions and 37 deletions.
14 changes: 14 additions & 0 deletions ARVideoKit/Extensions/UIImage+VideoBuffer.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,17 @@ extension UIImage
return pixelBuffer
}
}

public extension UIImage {
convenience init?(color: UIColor, size: CGSize = CGSize(width: 1, height: 1)) {
let rect = CGRect(origin: .zero, size: size)
UIGraphicsBeginImageContextWithOptions(rect.size, false, 0.0)
color.setFill()
UIRectFill(rect)
let image = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()

guard let cgImage = image?.cgImage else { return nil }
self.init(cgImage: cgImage)
}
}
20 changes: 20 additions & 0 deletions ARVideoKit/Extensions/UIView+isType.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,24 @@ extension UIView {
return (self is ARSCNView) || (self is ARSKView)
}
}

func imageViaLayerDraw() -> UIImage? {//always black
UIGraphicsBeginImageContextWithOptions(self.bounds.size, self.isOpaque, 0.0)
defer { UIGraphicsEndImageContext() }
if let context = UIGraphicsGetCurrentContext() {
self.layer.render(in: context)
let image = UIGraphicsGetImageFromCurrentImageContext()
return image
}
return nil
}

func imageViaViewDraw() -> UIImage? {//very slow
let renderer = UIGraphicsImageRenderer(size: self.bounds.size)
let capturedImage = renderer.image {
(ctx) in
self.drawHierarchy(in: self.bounds, afterScreenUpdates: true)
}
return capturedImage
}
}
13 changes: 5 additions & 8 deletions ARVideoKit/Rendering/RenderAR.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,14 @@ struct RenderAR {
}
guard let buffer = renderedFrame!.buffer else { return nil }
return buffer
} else if #available(iOS 13.0, *), view is RealityKit.ARView {
} else if #available(iOS 13.0, *), let view = view as? RealityKit.ARView {
guard let size = bufferSize else { return nil }
print("buffer size: \(size)")
//UIScreen.main.bounds.size
var renderedFrame: UIImage?
pixelsQueue.sync {
renderedFrame = renderEngine.snapshot(atTime: self.time, with: size, antialiasingMode: .none)
}
if let _ = renderedFrame {
} else {
renderedFrame = renderEngine.snapshot(atTime: time, with: size, antialiasingMode: .none)
}

renderedFrame = view.imageViaViewDraw()

guard let buffer = renderedFrame!.buffer else { return nil }
return buffer
}
Expand Down
15 changes: 2 additions & 13 deletions ARVideoKit/Sources/RecordAR.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,6 @@ import RealityKit
@objc override public init?(RealityKit: RealityKit.ARView) {
super.init(RealityKit: RealityKit)
view = RealityKit
scnView = SCNView(frame: UIScreen.main.bounds)
setup()
}

Expand Down Expand Up @@ -279,23 +278,13 @@ import RealityKit
gpuLoop.add(to: .main, forMode: .common)

status = .readyToRecord
} else if #available(iOS 13.0, *), let view = view as? RealityKit.ARView {
} else if #available(iOS 13.0, *), let _ = view as? RealityKit.ARView {
guard let mtlDevice = MTLCreateSystemDefaultDevice() else {
logAR.message("ERROR:- This device does not support Metal")
return
}
let material = SCNMaterial()
material.diffuse.contents = view.scene

let plane = SCNPlane(width: view.bounds.width, height: view.bounds.height)
let node = SCNNode(geometry: plane)
node.geometry?.firstMaterial = material
node.position = SCNVector3Make(0, 0, 0)

scnView.scene?.rootNode.addChildNode(node)

//NOT used in actual rendering, just a place holder.
renderEngine = SCNRenderer(device: mtlDevice, options: nil)
renderEngine.scene = scnView.scene

gpuLoop = CADisplayLink(target: WeakProxy(target: self),
selector: #selector(renderFrame))
Expand Down
22 changes: 20 additions & 2 deletions Examples/Swift/ARVideoKit-Example/Storyboards/Base.lproj/Main.storyboard
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,7 @@
</connections>
</button>
<button opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="roundedRect" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="cuZ-iu-01E">
<rect key="frame" x="174.5" y="723" width="65" height="65"/>
<rect key="frame" x="80" y="723" width="65" height="65"/>
<constraints>
<constraint firstAttribute="width" relation="greaterThanOrEqual" constant="65" id="Dry-Oq-0A1"/>
<constraint firstAttribute="height" constant="65" id="xnG-qU-Hd6"/>
Expand All @@ -454,13 +454,30 @@
<action selector="toggleRecord:" destination="Sut-aM-F8k" eventType="touchUpInside" id="hRM-N2-Lwm"/>
</connections>
</button>
<button opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="roundedRect" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="qYO-KD-l0h">
<rect key="frame" x="269" y="723" width="65" height="65"/>
<constraints>
<constraint firstAttribute="height" constant="65" id="XfN-cU-dpT"/>
<constraint firstAttribute="width" relation="greaterThanOrEqual" constant="65" id="h6F-EW-zfA"/>
</constraints>
<fontDescription key="fontDescription" type="system" pointSize="12"/>
<size key="titleShadowOffset" width="0.0" height="1"/>
<state key="normal" title="Screenshot">
<color key="titleShadowColor" white="0.0" alpha="0.65000000000000002" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
</state>
<connections>
<action selector="captureScreenshot:" destination="Sut-aM-F8k" eventType="touchUpInside" id="wKE-HQ-s27"/>
</connections>
</button>
</subviews>
<color key="backgroundColor" xcode11CocoaTouchSystemColor="systemBackgroundColor" cocoaTouchSystemColor="whiteColor"/>
<constraints>
<constraint firstItem="tqL-gq-GOP" firstAttribute="trailing" secondItem="qYO-KD-l0h" secondAttribute="trailing" constant="80" id="1Zn-SV-Qax"/>
<constraint firstItem="tqL-gq-GOP" firstAttribute="bottom" secondItem="cuZ-iu-01E" secondAttribute="bottom" constant="20" id="6DY-uq-WYo"/>
<constraint firstItem="cuZ-iu-01E" firstAttribute="leading" secondItem="tqL-gq-GOP" secondAttribute="leading" constant="80" id="88o-Lb-unv"/>
<constraint firstItem="tqL-gq-GOP" firstAttribute="top" secondItem="lpj-To-9o7" secondAttribute="top" constant="17" id="Awc-mL-Jlq"/>
<constraint firstItem="tqL-gq-GOP" firstAttribute="bottom" secondItem="qYO-KD-l0h" secondAttribute="bottom" constant="20" id="FbE-Pd-rFj"/>
<constraint firstItem="oIy-xw-BxF" firstAttribute="top" secondItem="tqL-gq-GOP" secondAttribute="top" constant="20" id="Izy-BD-Bg4"/>
<constraint firstItem="cuZ-iu-01E" firstAttribute="centerX" secondItem="tqL-gq-GOP" secondAttribute="centerX" id="WEd-QU-MPq"/>
<constraint firstItem="oIy-xw-BxF" firstAttribute="leading" secondItem="tqL-gq-GOP" secondAttribute="leading" constant="20" id="WPh-VR-5wf"/>
<constraint firstItem="lpj-To-9o7" firstAttribute="leading" secondItem="tqL-gq-GOP" secondAttribute="leading" id="ePx-ia-RPn"/>
<constraint firstItem="tqL-gq-GOP" firstAttribute="trailing" relation="greaterThanOrEqual" secondItem="oIy-xw-BxF" secondAttribute="trailing" symbolic="YES" id="fvG-0I-CRS"/>
Expand All @@ -470,6 +487,7 @@
<viewLayoutGuide key="safeArea" id="tqL-gq-GOP"/>
</view>
<connections>
<outlet property="captureBtn" destination="qYO-KD-l0h" id="RiU-v8-cRH"/>
<outlet property="dismissBtn" destination="oIy-xw-BxF" id="ctt-UC-rKC"/>
<outlet property="recordBtn" destination="cuZ-iu-01E" id="JQq-Ij-cXO"/>
<outlet property="rkView" destination="lpj-To-9o7" id="BCk-Nx-97k"/>
Expand Down
26 changes: 22 additions & 4 deletions Examples/Swift/ARVideoKit-Example/View Controllers/RKViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,31 @@ class RKViewController : UIViewController, ARSessionDelegate, RenderARDelegate,
}
}

@IBAction func captureScreenshot(_ sender: UIButton) {
//Photo
if recorder?.status == .readyToRecord {
let image = self.recorder?.photo()
self.recorder?.export(UIImage: image) { saved, status in
if saved {
DispatchQueue.main.sync {
self.exportMessage(success: saved, status: status)
}
} else {
print("failed to save screenshot: \(status)")
}
}
} else {
print("recorder status not ready: \(String(describing: recorder?.status))")
}
}

@IBOutlet var rkView: ARView!
@IBOutlet var recordBtn: UIButton!
@IBOutlet var captureBtn: UIButton!
@IBOutlet var dismissBtn: UIButton!

let recordingQueue = DispatchQueue(label: "recordingThread", attributes: .concurrent)
let caprturingQueue = DispatchQueue(label: "capturingThread", attributes: .concurrent)
let capturingQueue = DispatchQueue(label: "capturingThread", attributes: .concurrent)

var recorder:RecordAR?
let configuration = ARWorldTrackingConfiguration()
Expand Down Expand Up @@ -118,7 +137,8 @@ class RKViewController : UIViewController, ARSessionDelegate, RenderARDelegate,
super.viewDidAppear(animated)

// Run a body tracking configration.
rkView.session.run(configuration)
recorder?.onlyRenderWhileRecording = true
recorder?.prepare(configuration)

}

Expand All @@ -128,8 +148,6 @@ class RKViewController : UIViewController, ARSessionDelegate, RenderARDelegate,
if recorder?.status == .recording {
recorder?.stopAndExport()
}
recorder?.onlyRenderWhileRecording = true
recorder?.prepare(configuration)

// Switch off the orientation lock for UIViewControllers with AR Scenes
recorder?.rest()
Expand Down
10 changes: 5 additions & 5 deletions Examples/Swift/ARVideoKit-Example/View Controllers/SCNViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class SCNViewController: UIViewController, ARSCNViewDelegate, RenderARDelegate,
@IBOutlet var pauseBtn: UIButton!

let recordingQueue = DispatchQueue(label: "recordingThread", attributes: .concurrent)
let caprturingQueue = DispatchQueue(label: "capturingThread", attributes: .concurrent)
let capturingQueue = DispatchQueue(label: "capturingThread", attributes: .concurrent)

var recorder:RecordAR?

Expand Down Expand Up @@ -154,14 +154,14 @@ extension SCNViewController {
self.recorder?.export(UIImage: image) { saved, status in
if saved {
// Inform user photo has exported successfully
self.exportMessage(success: saved, status: status)
DispatchQueue.main.sync {self.exportMessage(success: saved, status: status)}
}
}
}
}else if sender.tag == 1 {
//Live Photo
if recorder?.status == .readyToRecord {
caprturingQueue.async {
capturingQueue.async {
self.recorder?.livePhoto(export: true) { ready, photo, status, saved in
/*
if ready {
Expand All @@ -171,7 +171,7 @@ extension SCNViewController {

if saved {
// Inform user Live Photo has exported successfully
self.exportMessage(success: saved, status: status)
DispatchQueue.main.sync {self.exportMessage(success: saved, status: status)}
}
}
}
Expand All @@ -188,7 +188,7 @@ extension SCNViewController {

if saved {
// Inform user GIF image has exported successfully
self.exportMessage(success: saved, status: status)
DispatchQueue.main.sync {self.exportMessage(success: saved, status: status)}
}
}
}
Expand Down
10 changes: 5 additions & 5 deletions Examples/Swift/ARVideoKit-Example/View Controllers/SKViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class SKViewController: UIViewController, ARSKViewDelegate, RenderARDelegate, Re
@IBOutlet var pauseBtn: UIButton!

let recordingQueue = DispatchQueue(label: "recordingThread")
let caprturingQueue = DispatchQueue(label: "capturingThread", attributes: .concurrent)
let capturingQueue = DispatchQueue(label: "capturingThread", attributes: .concurrent)

var recorder:RecordAR?

Expand Down Expand Up @@ -145,14 +145,14 @@ extension SKViewController {
self.recorder?.export(UIImage: image) { saved, status in
if saved {
// Inform user photo has exported successfully
self.exportMessage(success: saved, status: status)
DispatchQueue.main.sync {self.exportMessage(success: saved, status: status)}
}
}
}
}else if sender.tag == 1 {
//Live Photo
if recorder?.status == .readyToRecord {
caprturingQueue.async {
capturingQueue.async {
self.recorder?.livePhoto(export: true) { ready, photo, status, saved in
/*
if ready {
Expand All @@ -162,7 +162,7 @@ extension SKViewController {

if saved {
// Inform user Live Photo has exported successfully
self.exportMessage(success: saved, status: status)
DispatchQueue.main.sync {self.exportMessage(success: saved, status: status)}
}
}
}
Expand All @@ -179,7 +179,7 @@ extension SKViewController {

if saved {
// Inform user GIF image has exported successfully
self.exportMessage(success: saved, status: status)
DispatchQueue.main.sync {self.exportMessage(success: saved, status: status)}
}
}
}
Expand Down

0 comments on commit 5f221a1

Please sign in to comment.