Add support for audio ops (Spectrogram & MFCC) in Coreml and add trained coreml model

Baby Monitor/Source Files/AppDependencies.swift

@@ -15,14 +15,14 @@ final class AppDependencies {
     private let bag = DisposeBag()
     /// Service for cleaning too many crying events
     private(set) lazy var memoryCleaner: MemoryCleanerProtocol = MemoryCleaner()
-    /// Service for recording audio
-    private(set) lazy var audioRecordService: AudioRecordServiceProtocol? = try? AudioRecordService(recorderFactory: AudioKitRecorderFactory.makeRecorderFactory)
+    /// Service for capturing/recording microphone audio
+    private(set) lazy var audioMicrophoneService: AudioMicrophoneServiceProtocol? = try? AudioMicrophoneService(microphoneFactory: AudioKitMicrophoneFactory.makeMicrophoneFactory)
     /// Service for detecting baby's cry
-    private(set) lazy var cryingDetectionService: CryingDetectionServiceProtocol = CryingDetectionService(microphoneTracker: AKMicrophoneTracker())
+    private(set) lazy var cryingDetectionService: CryingDetectionServiceProtocol = CryingDetectionService(microphoneCapture: audioMicrophoneService)
     /// Service that takes care of appropriate controling: crying detection, audio recording and saving these events to realm database
     private(set) lazy var cryingEventService: CryingEventsServiceProtocol = CryingEventService(
         cryingDetectionService: cryingDetectionService,
-        audioRecordService: audioRecordService,
+        microphoneRecord: audioMicrophoneService,
         activityLogEventsRepository: databaseRepository,
         storageService: storageServerService)
 

Baby Monitor/Source Files/Services/CryingDetection/AudioMicrophoneService.swift

@@ -0,0 +1,119 @@
+//
+//  AudioMicrophoneService.swift
+//  Baby Monitor
+//
+
+import Foundation
+import AudioKit
+import RxSwift
+import RxCocoa
+
+protocol ErrorProducable {
+    var errorObservable: Observable<Error> { get }
+}
+
+protocol AudioMicrophoneRecordServiceProtocol {
+    var directoryDocumentsSavableObservable: Observable<DirectoryDocumentsSavable> { get }
+    var isRecording: Bool { get }
+
+    func stopRecording()
+    func startRecording()
+}
+
+protocol AudioMicrophoneCaptureServiceProtocol {
+    var microphoneBufferReadableObservable: Observable<AVAudioPCMBuffer> { get }
+    var isCapturing: Bool { get }
+
+    func stopCapturing()
+    func startCapturing()
+}
+
+protocol AudioMicrophoneServiceProtocol: AudioMicrophoneRecordServiceProtocol, AudioMicrophoneCaptureServiceProtocol {}
+
+final class AudioMicrophoneService: AudioMicrophoneServiceProtocol, ErrorProducable {
+
+    enum AudioError: Error {
+        case initializationFailure
+        case captureFailure
+        case recordFailure
+        case saveFailure
+    }
+
+    lazy var errorObservable = errorSubject.asObservable()
+    lazy var microphoneBufferReadableObservable = microphoneBufferReadableSubject.asObservable()
+    lazy var directoryDocumentsSavableObservable = directoryDocumentsSavableSubject.asObservable()
+
+    private(set) var isCapturing = false
+    private(set) var isRecording = false
+
+    private var capture: MicrophoneCaptureProtocol
+    private var record: MicrophoneRecordProtocol
+
+    private let errorSubject = PublishSubject<Error>()
+    private let microphoneBufferReadableSubject = PublishSubject<AVAudioPCMBuffer>()
+    private let directoryDocumentsSavableSubject = PublishSubject<DirectoryDocumentsSavable>()
+
+    private let disposeBag = DisposeBag()
+
+    init(microphoneFactory: () throws -> AudioKitMicrophoneProtocol?) throws {
+        guard let audioKitMicrophone = try microphoneFactory() else {
+            throw(AudioMicrophoneService.AudioError.initializationFailure)
+        }
+        capture = audioKitMicrophone.capture
+        record = audioKitMicrophone.record
+        rxSetup()
+    }
+
+    func stopCapturing() {
+        guard isCapturing else {
+            return
+        }
+        capture.stop()
+        isCapturing = false
+    }
+
+    func startCapturing() {
+        guard !isCapturing else {
+            return
+        }
+        do {
+            try capture.start()
+        } catch {
+            return
+        }
+        isCapturing = true
+    }
+
+    func stopRecording() {
+        guard isRecording else {
+            return
+        }
+        record.stop()
+        isRecording = false
+        guard let audioFile = record.audioFile else {
+            errorSubject.onNext(AudioError.recordFailure)
+            return
+        }
+        directoryDocumentsSavableSubject.onNext(audioFile)
+    }
+
+    func startRecording() {
+        guard !isRecording else {
+            return
+        }
+        do {
+            try record.reset()
+            try record.record()
+            isRecording = true
+        } catch {
+            errorSubject.onNext(AudioError.recordFailure)
+        }
+    }
+
+    private func rxSetup() {
+        capture.bufferReadable.subscribe(onNext: { [unowned self] bufferReadable in
+            self.microphoneBufferReadableSubject.onNext(bufferReadable)
+        }).disposed(by: disposeBag)
+    }
+
+}

Baby Monitor/Source Files/Services/CryingDetection/CryingDetectionService.swift

@@ -0,0 +1,72 @@
+//
+//  CryingDetectionService.swift
+//  Baby Monitor
+//
+
+import Foundation
+import CoreML
+import AudioKit
+import RxSwift
+import RxCocoa
+
+protocol CryingDetectionServiceProtocol: Any {
+
+    /// Observable that informs about detection of baby's cry
+    var cryingDetectionObservable: Observable<Bool> { get }
+
+    /// Starts crying detection
+    func startAnalysis()
+    /// Stops crying detection
+    func stopAnalysis()
+}
+
+final class CryingDetectionService: CryingDetectionServiceProtocol {
+
+    lazy var cryingDetectionObservable = cryingDetectionSubject.asObservable()
+
+    private let cryingDetectionSubject = PublishSubject<Bool>()
+
+    private let microphoneCapture: AudioMicrophoneCaptureServiceProtocol?
+    private let disposeBag = DisposeBag()
+    private let audioprocessingModel = audioprocessing()
+    private let crydetectionModel = crydetection()
+
+    init(microphoneCapture: AudioMicrophoneCaptureServiceProtocol?) {
+        self.microphoneCapture = microphoneCapture
+        rxSetup()
+    }
+
+    func startAnalysis() {
+        microphoneCapture?.startCapturing()
+    }
+
+    func stopAnalysis() {
+        microphoneCapture?.stopCapturing()
+    }
+
+    private func rxSetup() {
+        microphoneCapture?.microphoneBufferReadableObservable.subscribe(onNext: { [unowned self] bufferReadable in
+            do {
+                print("RUNNING ML MODEL")
+                let audioProcessingMultiArray = try MLMultiArray(dataPointer: bufferReadable.floatChannelData!.pointee,
+                                                                 shape: [264600],
+                                                                 dataType: .float32,
+                                                                 strides: [1])
+
+                let input = audioprocessingInput(raw_audio__0: audioProcessingMultiArray)
+                let pred = try self.audioprocessingModel.prediction(input: input)
+                let crydetectionMultiArray = try MLMultiArray(shape: [1, 1, 1, 598, 64], dataType: .float32)
+                crydetectionMultiArray.dataPointer.copyMemory(from: pred.Mfcc__0.dataPointer, byteCount: 38272 * 4)
+                let input1 = crydetectionInput(Mfcc__0: crydetectionMultiArray)
+                let pred2 = try self.crydetectionModel.prediction(input: input1)
+                let babyCryingDetected: Bool = pred2.labels_softmax__0[0].compare(pred2.labels_softmax__0[1]) == .orderedAscending
+                self.cryingDetectionSubject.onNext(babyCryingDetected)
+
+                print(pred2.labels_softmax__0)
+
+            } catch {
+                print("ERROR")
+            }
+        }).disposed(by: disposeBag)
+    }
+}