Merge pull request #14 from exPHAT/coreml

Package.swift

@@ -9,7 +9,9 @@ let package = Package(
     targets: [
         .target(name: "SwiftWhisper", dependencies: [.target(name: "whisper_cpp")]),
         .target(name: "whisper_cpp", cSettings: [
-            .define("GGML_USE_ACCELERATE", .when(platforms: [.macOS, .macCatalyst, .iOS]))
+            .define("GGML_USE_ACCELERATE", .when(platforms: [.macOS, .macCatalyst, .iOS])),
+            .define("WHISPER_USE_COREML", .when(platforms: [.macOS, .macCatalyst, .iOS])),
+            .define("WHISPER_COREML_ALLOW_FALLBACK", .when(platforms: [.macOS, .macCatalyst, .iOS]))
         ]),
         .testTarget(name: "WhisperTests", dependencies: [.target(name: "SwiftWhisper")], resources: [.copy("TestResources/")])
     ],

README.md

@@ -72,7 +72,11 @@ protocol WhisperDelegate {
 
 ### Downloading Models :inbox_tray:
 
-You can find the pre-trained models at [here](https://huggingface.co/ggerganov/whisper.cpp) for download.
+You can find the pre-trained models [here](https://huggingface.co/ggerganov/whisper.cpp) for download.
+
+### CoreML Support :brain:
+
+To use CoreML, you'll need to include a CoreML model file with the suffix `-encoder.mlmodelc` under the same name as the whisper model (Example: `tiny.bin` would also sit beside a `tiny-encoder.mlmodelc` file). In addition to the additonal model file, you will also need to use the `Whisper(fromFileURL:)` initializer. You can verify CoreML is active by checking the console output during transcription.
 
 ### Converting audio to 16kHz PCM :wrench:
 
@@ -113,7 +117,7 @@ func convertAudioFileToPCMArray(fileURL: URL, completionHandler: @escaping (Resu
 }
 ```
 
-### Speed boost :rocket:
+### Development speed boost :rocket:
 
 You may find the performance of the transcription slow when compiling your app for the `Debug` build configuration. This is because the compiler doesn't fully optimize SwiftWhisper unless the build configuration is set to `Release`.
 

Sources/whisper_cpp/coreml/whisper-encoder-impl.h

@@ -0,0 +1 @@
+../../../whisper.cpp/coreml/whisper-encoder-impl.h

Sources/whisper_cpp/coreml/whisper-encoder-impl.m

@@ -0,0 +1 @@
+../../../whisper.cpp/coreml/whisper-encoder-impl.m

Sources/whisper_cpp/coreml/whisper-encoder.h

@@ -0,0 +1 @@
+../../../whisper.cpp/coreml/whisper-encoder.h

Sources/whisper_cpp/coreml/whisper-encoder.mm

@@ -0,0 +1 @@
+../../../whisper.cpp/coreml/whisper-encoder.mm

Tests/WhisperTests/TestResources/tiny-encoder.mlmodelc/metadata.json

@@ -0,0 +1,64 @@
+[
+  {
+    "metadataOutputVersion" : "3.0",
+    "storagePrecision" : "Float16",
+    "outputSchema" : [
+      {
+        "hasShapeFlexibility" : "0",
+        "isOptional" : "0",
+        "dataType" : "Float32",
+        "formattedType" : "MultiArray (Float32)",
+        "shortDescription" : "",
+        "shape" : "[]",
+        "name" : "output",
+        "type" : "MultiArray"
+      }
+    ],
+    "modelParameters" : [
+
+    ],
+    "specificationVersion" : 6,
+    "mlProgramOperationTypeHistogram" : {
+      "Linear" : 24,
+      "Matmul" : 8,
+      "Cast" : 2,
+      "Conv" : 2,
+      "Softmax" : 4,
+      "Add" : 9,
+      "LayerNorm" : 9,
+      "Mul" : 8,
+      "Transpose" : 17,
+      "Gelu" : 6,
+      "Reshape" : 16
+    },
+    "computePrecision" : "Mixed (Float16, Float32, Int32)",
+    "isUpdatable" : "0",
+    "availability" : {
+      "macOS" : "12.0",
+      "tvOS" : "15.0",
+      "watchOS" : "8.0",
+      "iOS" : "15.0",
+      "macCatalyst" : "15.0"
+    },
+    "modelType" : {
+      "name" : "MLModelType_mlProgram"
+    },
+    "userDefinedMetadata" : {
+
+    },
+    "inputSchema" : [
+      {
+        "hasShapeFlexibility" : "0",
+        "isOptional" : "0",
+        "dataType" : "Float32",
+        "formattedType" : "MultiArray (Float32 1 × 80 × 3000)",
+        "shortDescription" : "",
+        "shape" : "[1, 80, 3000]",
+        "name" : "logmel_data",
+        "type" : "MultiArray"
+      }
+    ],
+    "generatedClassName" : "coreml_encoder_tiny",
+    "method" : "predict"
+  }
+]