Fixed unicode support in the source code

Closes #56, #62
zephir-lang · Apr 21, 2019 · 02a036f · 02a036f
1 parent f5a9c68
commit 02a036f
Show file tree

Hide file tree

Showing 5 changed files with 238 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
+### Fixed
+- Fixed unicode support in the source code
+  [#62](https://github.com/phalcon/php-zephir-parser/issues/62),
+  [#56](https://github.com/phalcon/php-zephir-parser/issues/56)
 
 ## [1.2.0] - 2019-01-14
 ### Added

diff --git a/parser/scanner.c b/parser/scanner.c
@@ -20,7 +20,7 @@
 #include "scanner.h"
 
 // for re2c
-#define YYCTYPE char
+#define YYCTYPE unsigned char
 #define YYCURSOR (s->cursor)
 #define YYLIMIT (s->limit)
 #define YYMARKER (s->marker)

diff --git a/parser/scanner.re b/parser/scanner.re
@@ -18,7 +18,7 @@
 #include "scanner.h"
 
 // for re2c
-#define YYCTYPE char
+#define YYCTYPE unsigned char
 #define YYCURSOR (s->cursor)
 #define YYLIMIT (s->limit)
 #define YYMARKER (s->marker)

diff --git a/tests/unicode/bug56.phpt b/tests/unicode/bug56.phpt
@@ -0,0 +1,112 @@
+--TEST--
+Using Cyrillic characters in the source code
+--SKIPIF--
+<?php include(__DIR__ . '/../skipif.inc'); ?>
+--FILE--
+<?php
+
+$code =<<<ZEP
+namespace Test;
+
+class test {
+
+    public function testUTF8() {
+        return "сдфггхх";
+    }
+}
+ZEP;
+
+var_dump(zephir_parse_file($code, '(eval code)'));
+?>
+--EXPECT--
+array(2) {
+  [0]=>
+  array(5) {
+    ["type"]=>
+    string(9) "namespace"
+    ["name"]=>
+    string(4) "Test"
+    ["file"]=>
+    string(11) "(eval code)"
+    ["line"]=>
+    int(3)
+    ["char"]=>
+    int(5)
+  }
+  [1]=>
+  array(8) {
+    ["type"]=>
+    string(5) "class"
+    ["name"]=>
+    string(4) "test"
+    ["abstract"]=>
+    int(0)
+    ["final"]=>
+    int(0)
+    ["definition"]=>
+    array(4) {
+      ["methods"]=>
+      array(1) {
+        [0]=>
+        array(8) {
+          ["visibility"]=>
+          array(1) {
+            [0]=>
+            string(6) "public"
+          }
+          ["type"]=>
+          string(6) "method"
+          ["name"]=>
+          string(8) "testUTF8"
+          ["statements"]=>
+          array(1) {
+            [0]=>
+            array(5) {
+              ["type"]=>
+              string(6) "return"
+              ["expr"]=>
+              array(5) {
+                ["type"]=>
+                string(6) "string"
+                ["value"]=>
+                string(14) "сдфггхх"
+                ["file"]=>
+                string(11) "(eval code)"
+                ["line"]=>
+                int(6)
+                ["char"]=>
+                int(30)
+              }
+              ["file"]=>
+              string(11) "(eval code)"
+              ["line"]=>
+              int(7)
+              ["char"]=>
+              int(5)
+            }
+          }
+          ["file"]=>
+          string(11) "(eval code)"
+          ["line"]=>
+          int(5)
+          ["last-line"]=>
+          int(8)
+          ["char"]=>
+          int(19)
+        }
+      }
+      ["file"]=>
+      string(11) "(eval code)"
+      ["line"]=>
+      int(3)
+      ["char"]=>
+      int(5)
+    }
+    ["file"]=>
+    string(11) "(eval code)"
+    ["line"]=>
+    int(3)
+    ["char"]=>
+    int(5)
+  }
+}
diff --git a/tests/unicode/bug62.phpt b/tests/unicode/bug62.phpt
@@ -0,0 +1,120 @@
+--TEST--
+Using Chinese characters in the source code
+--SKIPIF--
+<?php include(__DIR__ . '/../skipif.inc'); ?>
+--FILE--
+<?php
+
+$code =<<<ZEP
+namespace Utils;
+
+class Greeting
+{
+
+    public static function say()
+    {
+        echo "中文";
+    }
+
+}
+ZEP;
+
+var_dump(zephir_parse_file($code, '(eval code)'));
+?>
+--EXPECT--
+array(2) {
+  [0]=>
+  array(5) {
+    ["type"]=>
+    string(9) "namespace"
+    ["name"]=>
+    string(5) "Utils"
+    ["file"]=>
+    string(11) "(eval code)"
+    ["line"]=>
+    int(3)
+    ["char"]=>
+    int(5)
+  }
+  [1]=>
+  array(8) {
+    ["type"]=>
+    string(5) "class"
+    ["name"]=>
+    string(8) "Greeting"
+    ["abstract"]=>
+    int(0)
+    ["final"]=>
+    int(0)
+    ["definition"]=>
+    array(4) {
+      ["methods"]=>
+      array(1) {
+        [0]=>
+        array(8) {
+          ["visibility"]=>
+          array(2) {
+            [0]=>
+            string(6) "public"
+            [1]=>
+            string(6) "static"
+          }
+          ["type"]=>
+          string(6) "method"
+          ["name"]=>
+          string(3) "say"
+          ["statements"]=>
+          array(1) {
+            [0]=>
+            array(5) {
+              ["type"]=>
+              string(4) "echo"
+              ["expressions"]=>
+              array(1) {
+                [0]=>
+                array(5) {
+                  ["type"]=>
+                  string(6) "string"
+                  ["value"]=>
+                  string(6) "中文"
+                  ["file"]=>
+                  string(11) "(eval code)"
+                  ["line"]=>
+                  int(8)
+                  ["char"]=>
+                  int(20)
+                }
+              }
+              ["file"]=>
+              string(11) "(eval code)"
+              ["line"]=>
+              int(9)
+              ["char"]=>
+              int(5)
+            }
+          }
+          ["file"]=>
+          string(11) "(eval code)"
+          ["line"]=>
+          int(6)
+          ["last-line"]=>
+          int(11)
+          ["char"]=>
+          int(26)
+        }
+      }
+      ["file"]=>
+      string(11) "(eval code)"
+      ["line"]=>
+      int(3)
+      ["char"]=>
+      int(5)
+    }
+    ["file"]=>
+    string(11) "(eval code)"
+    ["line"]=>
+    int(3)
+    ["char"]=>
+    int(5)
+  }
+}