Add testDecodeWithSpecial

danny50610 · Aug 20, 2023 · 0ba62de · 0ba62de
1 parent 06bdc70
commit 0ba62de
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 1 deletion.
diff --git a/phpunit.xml b/phpunit.xml
@@ -9,6 +9,7 @@
   xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/10.3/phpunit.xsd"
   cacheDirectory=".phpunit.cache"
   backupStaticProperties="false"
+  displayDetailsOnTestsThatTriggerWarnings="true"
 >
   <coverage/>
   <testsuites>

diff --git a/src/Encoding.php b/src/Encoding.php
@@ -280,7 +280,7 @@ public function decode(array $tokens): string
     {
         $result = '';
         foreach ($tokens as $token) {
-            $out = $this->decodeMergeableRanks[$token];
+            $out = $this->decodeMergeableRanks[$token] ?? null;
             if (is_null($out)) {
                 $out = $this->decodeSpecialTokens[$token];
             }

diff --git a/tests/EncodingTest.php b/tests/EncodingTest.php
@@ -139,5 +139,13 @@ public static function specialDataProvider()
         ];
     }
 
+    public function testDecodeWithSpecial()
+    {
+        $enc = EncodingFactory::createByEncodingName('cl100k_base');
+        $text = $enc->decode([9906, 4435, 100257, 9906, 294, 13184, 13]);
+
+        $this->assertSame('Hello World<|endoftext|>Hello danny.', $text);
+    }
+
     // TODO: test: encodeOrdinary === encode($text, disallowedSpecial: [])
 }