Fix bundle resource crash on Linux

alfianlosari · Apr 1, 2023 · a869688 · a869688
1 parent 2ade249
commit a869688
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/Sources/GPTEncoder/GPTEncoder.swift b/Sources/GPTEncoder/GPTEncoder.swift
@@ -1,4 +1,9 @@
 import Foundation
+#if os(Linux)
+let bundle = Bundle.module
+#else
+let bundle = GPTEncoderResources.resourceBundle
+#endif
 
 public final class GPTEncoder {
 
@@ -9,13 +14,13 @@ public final class GPTEncoder {
     private let regex = try! NSRegularExpression(pattern: #"\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"#, options: .anchorsMatchLines)
 
     private let bpe: String = {
-        let fileURL = GPTEncoderResources.resourceBundle.url(forResource: "vocab", withExtension: "bpe")!
+        let fileURL = bundle.url(forResource: "vocab", withExtension: "bpe")!
         let bpe = try! Data(contentsOf: fileURL)
         return String(data: bpe, encoding: .utf8)!
     }()
 
     private let encoder: [String: Int] = {
-        let fileURL = GPTEncoderResources.resourceBundle.url(forResource: "encoder", withExtension: "json")!
+        let fileURL = bundle.url(forResource: "encoder", withExtension: "json")!
         let jsonEncoderData = try! Data(contentsOf: fileURL)
         return try! JSONSerialization.jsonObject(with: jsonEncoderData) as! [String: Int]
     }()

diff --git a/Tests/GPTEncoderTests/GPTEncoderTests.swift b/Tests/GPTEncoderTests/GPTEncoderTests.swift
@@ -6,7 +6,7 @@ final class GPTEncoderTests: XCTestCase {
     func testEncodeAndDecode() throws {
         let encoder = GPTEncoder()
 
-        let str = "A helpful rule of thumb is that one token generally corresponds to ~4 characters of text for common English text. This translates to roughly ¾ of a word (so 100 tokens ~= 75 words)."
+        let str = "這個算法真的太棒了"
         let encoded = encoder.encode(text: str)
         print("String: \(str)")
         print("Encoded this string looks like: \(encoded)")