Add VoiceChromePresenter

NuguAgents/Sources/CapabilityAgents/Interaction/InteractionControlManageable.swift

@@ -22,8 +22,13 @@ import Foundation
 
 /// <#Description#>
 public protocol InteractionControlManageable: class {
-    /// <#Description#>
-    var delegate: InteractionControlDelegate? { get set }
+    /// Adds a delegate to be notified of `InteractionControlManageable` state changes.
+    /// - Parameter delegate: The object to add.
+    func add(delegate: InteractionControlDelegate)
+
+    /// Removes a delegate from `InteractionControlManageable`.
+    /// - Parameter delegate: The object to remove.
+    func remove(delegate: InteractionControlDelegate)
 
     /// <#Description#>
     /// - Parameters:

NuguAgents/Sources/CapabilityAgents/Interaction/InteractionControlManager.swift

@@ -20,6 +20,8 @@
 
 import Foundation
 
+import NuguUtils
+
 import RxSwift
 
 public class InteractionControlManager: InteractionControlManageable {
@@ -29,35 +31,51 @@ public class InteractionControlManager: InteractionControlManageable {
         internalSerialQueueName: "com.sktelecom.romaine.interaction_control"
     )
 
-    public weak var delegate: InteractionControlDelegate?
+    private let delegates = DelegateSet<InteractionControlDelegate>()
 
     private var interactionControls = Set<CapabilityAgentCategory>()
     private var timeoutTimers = [String: Disposable]()
 
     public init() {}
+}
+
+// MARK: - InteractionControlManageable
+
+public extension InteractionControlManager {
+    func add(delegate: InteractionControlDelegate) {
+        delegates.add(delegate)
+    }
 
-    public func start(mode: InteractionControl.Mode, category: CapabilityAgentCategory) {
+    func remove(delegate: InteractionControlDelegate) {
+        delegates.remove(delegate)
+    }
+
+    func start(mode: InteractionControl.Mode, category: CapabilityAgentCategory) {
         log.debug(category)
         interactionDispatchQueue.async { [weak self] in
             guard let self = self, mode == .multiTurn else { return }
 
             self.addTimer(category: category)
             self.interactionControls.insert(category)
             if self.interactionControls.count == 1 {
-                self.delegate?.interactionControlDidChange(isMultiturn: true)
+                self.delegates.notify {
+                    $0.interactionControlDidChange(isMultiturn: true)
+                }
             }
         }
     }
 
-    public func finish(mode: InteractionControl.Mode, category: CapabilityAgentCategory) {
+    func finish(mode: InteractionControl.Mode, category: CapabilityAgentCategory) {
         log.debug(category)
         interactionDispatchQueue.async { [weak self] in
             guard let self = self, mode == .multiTurn else { return }
 
             self.removeTimer(category: category)
             self.interactionControls.remove(category)
             if self.interactionControls.isEmpty {
-                self.delegate?.interactionControlDidChange(isMultiturn: false)
+                self.delegates.notify {
+                    $0.interactionControlDidChange(isMultiturn: false)
+                }
             }
         }
     }
@@ -73,7 +91,9 @@ private extension InteractionControlManager {
                 log.debug("Timer fired. \(category)")
                 self.interactionControls.remove(category)
                 if self.interactionControls.isEmpty {
-                    self.delegate?.interactionControlDidChange(isMultiturn: false)
+                    self.delegates.notify {
+                        $0.interactionControlDidChange(isMultiturn: false)
+                    }
                 }
             })
     }

NuguClientKit.podspec

@@ -26,6 +26,7 @@ Default Instances for Nugu service
   s.dependency 'NuguCore', '~> 0'
   s.dependency 'NuguAgents', '~> 0'
   s.dependency 'KeenSense', '~> 0'
+  s.dependency 'NuguUIKit', '~> 0'
 
   s.dependency 'NattyLog', '~> 1'
 end

NuguClientKit/Sources/Business/DialogStateAggregator.swift

@@ -87,7 +87,7 @@ public class DialogStateAggregator {
         self.sessionManager = sessionManager
         self.focusManager = focusManager
 
-        interactionControlManager.delegate = self
+        interactionControlManager.add(delegate: self)
     }
 }
 

NuguClientKit/Sources/Presenter/VoiceChromePresenter.swift

@@ -0,0 +1,255 @@
+//
+//  VoiceChromePresenter.swift
+//  NuguClientKit
+//
+//  Created by 이민철님/AI Assistant개발Cell on 2020/11/18.
+//  Copyright (c) 2020 SK Telecom Co., Ltd. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Foundation
+import UIKit
+
+import NuguAgents
+import NuguUIKit
+import NuguUtils
+
+/// <#Description#>
+public class VoiceChromePresenter {
+    private let nuguVoiceChrome: NuguVoiceChrome
+    private let viewController: UIViewController
+
+    private var asrState: ASRState = .idle
+    private var isMultiturn: Bool = false
+    private var voiceChromeDismissWorkItem: DispatchWorkItem?
+
+    public weak var delegate: VoiceChromePresenterDelegate?
+    public var isHidden = true
+
+    public init(viewController: UIViewController, nuguVoiceChrome: NuguVoiceChrome, nuguClient: NuguClient) {
+        self.viewController = viewController
+        self.nuguVoiceChrome = nuguVoiceChrome
+
+        nuguClient.dialogStateAggregator.add(delegate: self)
+        nuguClient.asrAgent.add(delegate: self)
+        nuguClient.interactionControlManager.add(delegate: self)
+    }
+}
+
+// MARK: - Public (Voice Chrome)
+
+public extension VoiceChromePresenter {
+    func presentVoiceChrome() throws {
+        guard NetworkReachabilityManager.shared.isReachable else { throw VoiceChromePresenterError.networkUnreachable }
+        log.debug("")
+
+        voiceChromeDismissWorkItem?.cancel()
+        nuguVoiceChrome.removeFromSuperview()
+        nuguVoiceChrome.changeState(state: .listeningPassive)
+
+        try showVoiceChrome()
+    }
+
+    func dismissVoiceChrome() {
+        log.debug("")
+        delegate?.voiceChromeWillHide()
+
+        isHidden = true
+        voiceChromeDismissWorkItem?.cancel()
+
+        UIView.animate(withDuration: 0.3, animations: { [weak self] in
+            guard let self = self else { return }
+            self.nuguVoiceChrome.transform = CGAffineTransform(translationX: 0.0, y: 0)
+        }, completion: { [weak self] _ in
+            self?.nuguVoiceChrome.removeFromSuperview()
+        })
+    }
+}
+
+// MARK: - Private
+
+private extension VoiceChromePresenter {
+    var bottomSafeAreaHeight: CGFloat {
+        if #available(iOS 11.0, *) {
+            return viewController.view?.safeAreaInsets.bottom ?? 0
+        } else {
+            return viewController.bottomLayoutGuide.length
+        }
+    }
+
+    func showVoiceChrome() throws {
+        log.debug("")
+        guard let view = viewController.view else { throw VoiceChromePresenterError.superViewNotExsit }
+        guard isHidden == true else { throw VoiceChromePresenterError.alreadyShown      }
+
+        delegate?.voiceChromeWillShow()
+
+        isHidden = false
+
+        let showAnimation = {
+            UIView.animate(withDuration: 0.3) { [weak self] in
+                guard let self = self else { return }
+                self.nuguVoiceChrome.transform = CGAffineTransform(translationX: 0.0, y: -self.nuguVoiceChrome.bounds.height)
+            }
+        }
+
+        if view.subviews.contains(nuguVoiceChrome) == false {
+            nuguVoiceChrome.frame = CGRect(x: 0, y: view.frame.size.height, width: view.frame.size.width, height: NuguVoiceChrome.recommendedHeight + bottomSafeAreaHeight)
+            view.addSubview(nuguVoiceChrome)
+        }
+        showAnimation()
+    }
+
+    func setChipsButton(actionList: [(text: String, token: String?)], normalList: [(text: String, token: String?)]) {
+        var chipsButtonList = [NuguChipsButton.NuguChipsButtonType]()
+        let actionButtonList = actionList.map { NuguChipsButton.NuguChipsButtonType.action(text: $0.text, token: $0.token) }
+        chipsButtonList.append(contentsOf: actionButtonList)
+        let normalButtonList = normalList.map { NuguChipsButton.NuguChipsButtonType.normal(text: $0.text, token: $0.token) }
+        chipsButtonList.append(contentsOf: normalButtonList)
+        nuguVoiceChrome.setChipsData(chipsData: chipsButtonList)
+    }
+
+    func disableIdleTimer() {
+        guard UIApplication.shared.isIdleTimerDisabled == false else { return }
+        guard delegate?.voiceChromeShouldDisableIdleTimer() != false else { return }
+
+        UIApplication.shared.isIdleTimerDisabled = true
+        log.debug("Disable idle timer")
+    }
+
+    func enableIdleTimer() {
+        guard UIApplication.shared.isIdleTimerDisabled == true else { return }
+        guard isMultiturn == false, asrState == .idle else { return }
+        guard delegate?.voiceChromeShouldEnableIdleTimer() != false else { return }
+
+        UIApplication.shared.isIdleTimerDisabled = false
+        log.debug("Enable idle timer")
+    }
+}
+
+
+// MARK: - DialogStateDelegate
+
+extension VoiceChromePresenter: DialogStateDelegate {
+    public func dialogStateDidChange(_ state: DialogState, isMultiturn: Bool, chips: [ChipsAgentItem.Chip]?, sessionActivated: Bool) {
+        log.debug("\(state) \(isMultiturn), \(chips.debugDescription)")
+        switch state {
+        case .idle:
+            voiceChromeDismissWorkItem = DispatchWorkItem(block: { [weak self] in
+                self?.dismissVoiceChrome()
+            })
+            guard let voiceChromeDismissWorkItem = voiceChromeDismissWorkItem else { break }
+            DispatchQueue.main.async(execute: voiceChromeDismissWorkItem)
+        case .speaking:
+            voiceChromeDismissWorkItem?.cancel()
+            DispatchQueue.main.async { [weak self] in
+                guard let self = self else { return }
+                guard isMultiturn == true else {
+                    self.dismissVoiceChrome()
+                    return
+                }
+                // If voice chrome is not showing or dismissing in speaking state, voice chrome should be presented
+                try? self.showVoiceChrome()
+                self.nuguVoiceChrome.changeState(state: .speaking)
+                if let chips = chips {
+                    let actionList = chips.filter { $0.type == .action }.map { ($0.text, $0.token) }
+                    let normalList = chips.filter { $0.type == .general }.map { ($0.text, $0.token) }
+                    self.setChipsButton(actionList: actionList, normalList: normalList)
+                }
+            }
+        case .listening:
+            voiceChromeDismissWorkItem?.cancel()
+            DispatchQueue.main.async { [weak self] in
+                guard let self = self else { return }
+                // If voice chrome is not showing or dismissing in listening state, voice chrome should be presented
+                try? self.showVoiceChrome()
+                if isMultiturn || sessionActivated {
+                    self.nuguVoiceChrome.changeState(state: .listeningPassive)
+                    self.nuguVoiceChrome.setRecognizedText(text: nil)
+                }
+                if let chips = chips {
+                    let actionList = chips.filter { $0.type == .action }.map { ($0.text, $0.token) }
+                    let normalList = chips.filter { $0.type == .general }.map { ($0.text, $0.token) }
+                    self.setChipsButton(actionList: actionList, normalList: normalList)
+                }
+            }
+        case .recognizing:
+            DispatchQueue.main.async { [weak self] in
+                self?.nuguVoiceChrome.changeState(state: .listeningActive)
+            }
+        case .thinking:
+            DispatchQueue.main.async { [weak self] in
+                self?.nuguVoiceChrome.changeState(state: .processing)
+            }
+        }
+    }
+}
+
+// MARK: - AutomaticSpeechRecognitionDelegate
+
+extension VoiceChromePresenter: ASRAgentDelegate {
+    public func asrAgentDidChange(state: ASRState) {
+        DispatchQueue.main.async { [weak self] in
+            guard let self = self else { return }
+
+            self.asrState = state
+            switch state {
+            case .idle:
+                self.enableIdleTimer()
+            default:
+                self.disableIdleTimer()
+            }
+        }
+    }
+
+    public func asrAgentDidReceive(result: ASRResult, dialogRequestId: String) {
+        switch result {
+        case .complete(let text, _):
+            DispatchQueue.main.async { [weak self] in
+                self?.nuguVoiceChrome.setRecognizedText(text: text)
+            }
+        case .partial(let text, _):
+            DispatchQueue.main.async { [weak self] in
+                self?.nuguVoiceChrome.setRecognizedText(text: text)
+            }
+        case .error(let error, _):
+            DispatchQueue.main.async { [weak self] in
+                switch error {
+                case ASRError.listenFailed:
+                    self?.nuguVoiceChrome.changeState(state: .speakingError)
+                default:
+                    break
+                }
+            }
+        default: break
+        }
+    }
+}
+
+// MARK: - InteractionControlDelegate
+
+extension VoiceChromePresenter: InteractionControlDelegate {
+    public func interactionControlDidChange(isMultiturn: Bool) {
+        DispatchQueue.main.async { [weak self] in
+            guard let self = self else { return }
+
+            self.isMultiturn = isMultiturn
+            if isMultiturn {
+                self.disableIdleTimer()
+            } else {
+                self.enableIdleTimer()
+            }
+        }
+    }
+}

NuguClientKit/Sources/Presenter/VoiceChromePresenterDelegate.swift

@@ -0,0 +1,33 @@
+//
+//  VoiceChromePresenterDelegate.swift
+//  NuguClientKit
+//
+//  Created by 이민철님/AI Assistant개발Cell on 2020/11/25.
+//  Copyright (c) 2020 SK Telecom Co., Ltd. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Foundation
+
+/// A delegate that application can extend to observe `VoiceChromePresenter` changes.
+public protocol VoiceChromePresenterDelegate: class {
+    /// Notifies the `NuguVoiceChrome` to be shown.
+    func voiceChromeWillShow()
+    /// Notifies the `NuguVoiceChrome` to be hidden.
+    func voiceChromeWillHide()
+    /// Determines whether to set `UIApplication.shared.isIdleTimerDisabled` to true.
+    func voiceChromeShouldDisableIdleTimer() -> Bool
+    /// Determines whether to set `UIApplication.shared.isIdleTimerDisabled` to false.
+    func voiceChromeShouldEnableIdleTimer() -> Bool
+}