Skip to content

Commit

Permalink
Video message recording improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@laktyushin
laktyushin committed Jan 14, 2024
1 parent 331f0dd commit 8bcc38c
Show file tree
Hide file tree
Showing 8 changed files with 182 additions and 46 deletions.
15 changes: 8 additions & 7 deletions submodules/Camera/Sources/Camera.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,12 @@ final class CameraDeviceContext {
self.device.resetZoom(neutral: self.exclusive || !self.additional)
}

func invalidate() {
func invalidate(switchAudio: Bool = true) {
guard let session = self.session else {
return
}
self.output.invalidate(for: session)
self.input.invalidate(for: session)
self.output.invalidate(for: session, switchAudio: switchAudio)
self.input.invalidate(for: session, switchAudio: switchAudio)
}

private func maxDimensions(additional: Bool, preferWide: Bool) -> CMVideoDimensions {
Expand Down Expand Up @@ -248,7 +248,8 @@ private final class CameraContext {
mainDeviceContext.output.markPositionChange(position: targetPosition)
} else {
self.configure {
self.mainDeviceContext?.invalidate()
let isRoundVideo = self.initialConfiguration.isRoundVideo
self.mainDeviceContext?.invalidate(switchAudio: !isRoundVideo)

let targetPosition: Camera.Position
if case .back = mainDeviceContext.device.position {
Expand All @@ -260,8 +261,8 @@ private final class CameraContext {
self._positionPromise.set(targetPosition)
self.modeChange = .position

let isRoundVideo = self.initialConfiguration.isRoundVideo
let preferWide = self.initialConfiguration.preferWide || (self.positionValue == .front && isRoundVideo)

let preferWide = self.initialConfiguration.preferWide || isRoundVideo
let preferLowerFramerate = self.initialConfiguration.preferLowerFramerate || isRoundVideo

mainDeviceContext.configure(position: targetPosition, previewView: self.simplePreviewView, audio: self.initialConfiguration.audio, photo: self.initialConfiguration.photo, metadata: self.initialConfiguration.metadata, preferWide: preferWide, preferLowerFramerate: preferLowerFramerate, switchAudio: !isRoundVideo)
Expand Down Expand Up @@ -352,7 +353,7 @@ private final class CameraContext {
self.additionalDeviceContext?.invalidate()
self.additionalDeviceContext = nil

let preferWide = self.initialConfiguration.preferWide || (self.positionValue == .front && self.initialConfiguration.isRoundVideo)
let preferWide = self.initialConfiguration.preferWide || self.initialConfiguration.isRoundVideo
let preferLowerFramerate = self.initialConfiguration.preferLowerFramerate || self.initialConfiguration.isRoundVideo

self.mainDeviceContext = CameraDeviceContext(session: self.session, exclusive: true, additional: false, ciContext: self.ciContext, use32BGRA: self.initialConfiguration.isRoundVideo)
Expand Down
5 changes: 4 additions & 1 deletion submodules/Camera/Sources/CameraInput.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,11 @@ class CameraInput {
}
}

func invalidate(for session: CameraSession) {
func invalidate(for session: CameraSession, switchAudio: Bool = true) {
for input in session.session.inputs {
if !switchAudio && input === self.audioInput {
continue
}
session.session.removeInput(input)
}
}
Expand Down
61 changes: 56 additions & 5 deletions submodules/Camera/Sources/CameraOutput.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ final class CameraOutput: NSObject {
}
}

func invalidate(for session: CameraSession) {
func invalidate(for session: CameraSession, switchAudio: Bool = true) {
if #available(iOS 13.0, *) {
if let previewConnection = self.previewConnection {
if session.session.connections.contains(where: { $0 === previewConnection }) {
Expand All @@ -214,7 +214,7 @@ final class CameraOutput: NSObject {
if session.session.outputs.contains(where: { $0 === self.videoOutput }) {
session.session.removeOutput(self.videoOutput)
}
if session.session.outputs.contains(where: { $0 === self.audioOutput }) {
if switchAudio, session.session.outputs.contains(where: { $0 === self.audioOutput }) {
session.session.removeOutput(self.audioOutput)
}
if session.session.outputs.contains(where: { $0 === self.photoOutput }) {
Expand Down Expand Up @@ -409,6 +409,14 @@ final class CameraOutput: NSObject {
private weak var masterOutput: CameraOutput?

private var lastSampleTimestamp: CMTime?

private var needsCrossfadeTransition = false
private var crossfadeTransitionStart: Double = 0.0

private var needsSwitchSampleOffset = false
private var lastAudioSampleTime: CMTime?
private var videoSwitchSampleTimeOffset: CMTime?

func processVideoRecording(_ sampleBuffer: CMSampleBuffer, fromAdditionalOutput: Bool) {
guard let formatDescriptor = CMSampleBufferGetFormatDescription(sampleBuffer) else {
return
Expand All @@ -417,10 +425,10 @@ final class CameraOutput: NSObject {

if let videoRecorder = self.videoRecorder, videoRecorder.isRecording {
if case .roundVideo = self.currentMode, type == kCMMediaType_Video {
let currentTimestamp = CACurrentMediaTime()
let duration: Double = 0.2
if !self.exclusive {
var transitionFactor: CGFloat = 0.0
let currentTimestamp = CACurrentMediaTime()
let duration: Double = 0.2
if case .front = self.currentPosition {
transitionFactor = 1.0
if self.lastSwitchTimestamp > 0.0, currentTimestamp - self.lastSwitchTimestamp < duration {
Expand All @@ -446,13 +454,51 @@ final class CameraOutput: NSObject {
videoRecorder.appendSampleBuffer(sampleBuffer)
}
} else {
if let processedSampleBuffer = self.processRoundVideoSampleBuffer(sampleBuffer, additional: self.currentPosition == .front, transitionFactor: self.currentPosition == .front ? 1.0 : 0.0) {
var additional = self.currentPosition == .front
var transitionFactor = self.currentPosition == .front ? 1.0 : 0.0
if self.lastSwitchTimestamp > 0.0 {
if self.needsCrossfadeTransition {
self.needsCrossfadeTransition = false
self.crossfadeTransitionStart = currentTimestamp + 0.03
self.needsSwitchSampleOffset = true
}
if self.crossfadeTransitionStart > 0.0, currentTimestamp - self.crossfadeTransitionStart < duration {
if case .front = self.currentPosition {
transitionFactor = max(0.0, (currentTimestamp - self.crossfadeTransitionStart) / duration)
} else {
transitionFactor = 1.0 - max(0.0, (currentTimestamp - self.crossfadeTransitionStart) / duration)
}
} else if currentTimestamp - self.lastSwitchTimestamp < 0.05 {
additional = !additional
transitionFactor = 1.0 - transitionFactor
self.needsCrossfadeTransition = true
}
}
if let processedSampleBuffer = self.processRoundVideoSampleBuffer(sampleBuffer, additional: additional, transitionFactor: transitionFactor) {
videoRecorder.appendSampleBuffer(processedSampleBuffer)
} else {
videoRecorder.appendSampleBuffer(sampleBuffer)
}
}
} else {
if type == kCMMediaType_Audio {
if self.needsSwitchSampleOffset {
self.needsSwitchSampleOffset = false

if let lastAudioSampleTime = self.lastAudioSampleTime {
let videoSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
let offset = videoSampleTime - lastAudioSampleTime
if let current = self.videoSwitchSampleTimeOffset {
self.videoSwitchSampleTimeOffset = current + offset
} else {
self.videoSwitchSampleTimeOffset = offset
}
self.lastAudioSampleTime = nil
}
}

self.lastAudioSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + CMSampleBufferGetDuration(sampleBuffer)
}
videoRecorder.appendSampleBuffer(sampleBuffer)
}
}
Expand Down Expand Up @@ -494,6 +540,11 @@ final class CameraOutput: NSObject {
var sampleTimingInfo: CMSampleTimingInfo = .invalid
CMSampleBufferGetSampleTimingInfo(sampleBuffer, at: 0, timingInfoOut: &sampleTimingInfo)

if let videoSwitchSampleTimeOffset = self.videoSwitchSampleTimeOffset {
sampleTimingInfo.decodeTimeStamp = sampleTimingInfo.decodeTimeStamp - videoSwitchSampleTimeOffset
sampleTimingInfo.presentationTimeStamp = sampleTimingInfo.presentationTimeStamp - videoSwitchSampleTimeOffset
}

var newSampleBuffer: CMSampleBuffer?
status = CMSampleBufferCreateForImageBuffer(
allocator: kCFAllocatorDefault,
Expand Down
40 changes: 31 additions & 9 deletions submodules/Camera/Sources/CameraPreviewView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,29 @@ private extension UIInterfaceOrientation {
}
}

private class SimpleCapturePreviewLayer: AVCaptureVideoPreviewLayer {
public var didEnterHierarchy: (() -> Void)?
public var didExitHierarchy: (() -> Void)?

override open func action(forKey event: String) -> CAAction? {
if event == kCAOnOrderIn {
self.didEnterHierarchy?()
} else if event == kCAOnOrderOut {
self.didExitHierarchy?()
}
return nullAction
}

override public init(layer: Any) {
super.init(layer: layer)
}

required public init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
}


public class CameraSimplePreviewView: UIView {
func updateOrientation() {
guard self.videoPreviewLayer.connection?.isVideoOrientationSupported == true else {
Expand Down Expand Up @@ -72,11 +95,16 @@ public class CameraSimplePreviewView: UIView {
private var previewingDisposable: Disposable?
private let placeholderView = UIImageView()

public init(frame: CGRect, main: Bool) {
public init(frame: CGRect, main: Bool, roundVideo: Bool = false) {
super.init(frame: frame)

self.videoPreviewLayer.videoGravity = main ? .resizeAspectFill : .resizeAspect
self.placeholderView.contentMode = main ? .scaleAspectFill : .scaleAspectFit
if roundVideo {
self.videoPreviewLayer.videoGravity = .resizeAspectFill
self.placeholderView.contentMode = .scaleAspectFill
} else {
self.videoPreviewLayer.videoGravity = main ? .resizeAspectFill : .resizeAspect
self.placeholderView.contentMode = main ? .scaleAspectFill : .scaleAspectFit
}

self.addSubview(self.placeholderView)
}
Expand Down Expand Up @@ -567,35 +595,29 @@ public class CameraPreviewView: MTKView {
var scaleX: CGFloat
var scaleY: CGFloat

// Rotate the layer into screen orientation.
switch UIDevice.current.orientation {
case .portraitUpsideDown:
rotation = 180
scaleX = videoPreviewRect.width / captureDeviceResolution.width
scaleY = videoPreviewRect.height / captureDeviceResolution.height

case .landscapeLeft:
rotation = 90
scaleX = videoPreviewRect.height / captureDeviceResolution.width
scaleY = scaleX

case .landscapeRight:
rotation = -90
scaleX = videoPreviewRect.height / captureDeviceResolution.width
scaleY = scaleX

default:
rotation = 0
scaleX = videoPreviewRect.width / captureDeviceResolution.width
scaleY = videoPreviewRect.height / captureDeviceResolution.height
}

// Scale and mirror the image to ensure upright presentation.
let affineTransform = CGAffineTransform(rotationAngle: radiansForDegrees(rotation))
.scaledBy(x: scaleX, y: -scaleY)
overlayLayer.setAffineTransform(affineTransform)

// Cover entire screen UI.
let rootLayerBounds = self.bounds
overlayLayer.position = CGPoint(x: rootLayerBounds.midX, y: rootLayerBounds.midY)
}
Expand Down
4 changes: 2 additions & 2 deletions submodules/Camera/Sources/VideoRecorder.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ private final class VideoRecorderImpl {
}

if failed {
print("error")
print("append video error")
return
}

Expand Down Expand Up @@ -256,7 +256,7 @@ private final class VideoRecorderImpl {
}

if failed {
print("error")
print("append audio error")
return
}

Expand Down
20 changes: 15 additions & 5 deletions .../TelegramUI/Components/Settings/WallpaperGridScreen/Sources/ThemeGridControllerNode.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,10 @@ final class ThemeGridControllerNode: ASDisplayNode {
if let strongSelf = self, !strongSelf.currentState.editing {
let entries = previousEntries.with { $0 }
if let entries = entries, !entries.isEmpty {
let wallpapers = entries.map { $0.wallpaper }.filter { !$0.isColorOrGradient }
var wallpapers = entries.map { $0.wallpaper }
if case .peer = mode {
wallpapers = wallpapers.filter { !$0.isColorOrGradient }
}

var options = WallpaperPresentationOptions()
if wallpaper == strongSelf.presentationData.chatWallpaper, let settings = wallpaper.settings {
Expand Down Expand Up @@ -575,7 +578,14 @@ final class ThemeGridControllerNode: ASDisplayNode {
transition.updateFrame(node: strongSelf.bottomBackgroundNode, frame: CGRect(origin: CGPoint(x: 0.0, y: gridLayout.contentSize.height), size: CGSize(width: layout.size.width, height: 500.0)))
transition.updateFrame(node: strongSelf.bottomSeparatorNode, frame: CGRect(origin: CGPoint(x: 0.0, y: gridLayout.contentSize.height), size: CGSize(width: layout.size.width, height: UIScreenPixel)))

let params = ListViewItemLayoutParams(width: layout.size.width, leftInset: layout.safeInsets.left, rightInset: layout.safeInsets.right, availableHeight: layout.size.height)
let sideInset = max(16.0, floor((layout.size.width - 674.0) / 2.0))
var listInsets = layout.safeInsets
if layout.size.width >= 375.0 {
listInsets.left = sideInset
listInsets.right = sideInset
}

let params = ListViewItemLayoutParams(width: layout.size.width, leftInset: listInsets.left, rightInset: listInsets.right, availableHeight: layout.size.height)

let makeResetLayout = strongSelf.resetItemNode.asyncLayout()
let makeResetDescriptionLayout = strongSelf.resetDescriptionItemNode.asyncLayout()
Expand All @@ -588,8 +598,8 @@ final class ThemeGridControllerNode: ASDisplayNode {
transition.updateFrame(node: strongSelf.resetItemNode, frame: CGRect(origin: CGPoint(x: 0.0, y: gridLayout.contentSize.height + 35.0), size: resetLayout.contentSize))
transition.updateFrame(node: strongSelf.resetDescriptionItemNode, frame: CGRect(origin: CGPoint(x: 0.0, y: gridLayout.contentSize.height + 35.0 + resetLayout.contentSize.height), size: resetDescriptionLayout.contentSize))

let sideInset = strongSelf.leftOverlayNode.frame.maxX
strongSelf.maskNode.frame = CGRect(origin: CGPoint(x: sideInset, y: strongSelf.separatorNode.frame.minY + UIScreenPixel + 4.0), size: CGSize(width: layout.size.width - sideInset * 2.0, height: gridLayout.contentSize.height + 6.0))
let maskSideInset = strongSelf.leftOverlayNode.frame.maxX
strongSelf.maskNode.frame = CGRect(origin: CGPoint(x: maskSideInset, y: strongSelf.separatorNode.frame.minY + UIScreenPixel + 4.0), size: CGSize(width: layout.size.width - sideInset * 2.0, height: gridLayout.contentSize.height + 6.0))
}
}
}
Expand Down Expand Up @@ -934,7 +944,7 @@ final class ThemeGridControllerNode: ASDisplayNode {
let (resetDescriptionLayout, _) = makeResetDescriptionLayout(self.resetDescriptionItem, params, ItemListNeighbors(top: .none, bottom: .none))

if !isChannel {
insets.bottom += buttonHeight + 35.0 + resetDescriptionLayout.contentSize.height + 32.0
listInsets.bottom += buttonHeight + 35.0 + resetDescriptionLayout.contentSize.height + 32.0
}

self.gridNode.frame = CGRect(x: 0.0, y: 0.0, width: layout.size.width, height: layout.size.height)
Expand Down
Loading

0 comments on commit 8bcc38c

Please sign in to comment.