Process non ascii symbols (#6)

Toflar · Mar 18, 2024 · a14fc43 · a14fc43
1 parent b2f8e09
commit a14fc43
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 5 deletions.
diff --git a/src/Levenshtein.php b/src/Levenshtein.php
@@ -6,13 +6,32 @@ class Levenshtein
 {
     public static function distance(string $string1, string $string2, int $insertionCost = 1, $replacementCost = 1, $deletionCost = 1): int
     {
-        $string1 = mb_convert_encoding($string1, 'ASCII', 'utf8');
-        $string2 = mb_convert_encoding($string2, 'ASCII', 'utf8');
+        $map = [];
+        $string1 = self::utf8_to_extended_ascii($string1, $map);
+        $string2 = self::utf8_to_extended_ascii($string2, $map);
 
-        if (false === $string1 || false === $string2) {
-            throw new \InvalidArgumentException('Both, string1 and string2 have to be valid utf-8 strings.');
+        return levenshtein($string1, $string2, $insertionCost, $replacementCost, $deletionCost);
+    }
+
+    private static function utf8_to_extended_ascii($str, &$map)
+    {
+        // find all multibyte characters (cf. utf-8 encoding specs)
+        $matches = [];
+        if (!preg_match_all('/[\xC0-\xF7][\x80-\xBF]+/', $str, $matches)) {
+            return $str;
+        } // plain ascii string
+
+        // update the encoding map with the characters not already met
+        foreach ($matches[0] as $mbc) {
+            if (!isset($map[$mbc])) {
+                if (\count($map) >= 128) {
+                    throw new \InvalidArgumentException('Strings with more than 128 individual unicode characters are not supported.');
+                }
+                $map[$mbc] = \chr(128 + \count($map));
+            }
         }
 
-        return levenshtein($string1, $string2, $insertionCost, $replacementCost, $deletionCost);
+        // finally remap non-ascii characters
+        return strtr($str, $map);
     }
 }
diff --git a/tests/LevenshteinTest.php b/tests/LevenshteinTest.php
@@ -12,7 +12,12 @@ public function testLevenshtein(): void
         $this->assertSame(1, Levenshtein::distance('hello', 'helo'));
         $this->assertSame(2, Levenshtein::distance('hello', 'heo'));
         $this->assertSame(1, Levenshtein::distance('héllo', 'hello'));
+        $this->assertSame(2, Levenshtein::distance('Ñörbärm', 'Üörbarm'));
         $this->assertSame(2, Levenshtein::distance('garçonnière', 'garconniere'));
         $this->assertSame(1, Levenshtein::distance('garçonnière', 'garçonniere'));
+        $this->assertSame(1, Levenshtein::distance('пожар', 'пажар'));
+        $this->assertSame(1, Levenshtein::distance('пожар', 'пожаr'));
+        $this->assertSame(2, Levenshtein::distance('слово', 'слива'));
+        $this->assertNotSame(0, Levenshtein::distance('стул', 'вода'));
     }
 }