Skip to content

Commit

Permalink
Merge pull request #65 from phalcon/fix/issue-56-unicode-support
Browse files Browse the repository at this point in the history
Fixed unicode support in the source code
  • Loading branch information
sergeyklay authored Apr 21, 2019
2 parents 2f5032a + 02a036f commit ec5cc41
Show file tree
Hide file tree
Showing 6 changed files with 243 additions and 7 deletions.
6 changes: 5 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).

## [Unreleased]
### Fixed
- Fixed unicode support in the source code
[#62](https://github.com/phalcon/php-zephir-parser/issues/62),
[#56](https://github.com/phalcon/php-zephir-parser/issues/56)

## [1.2.0] - 2019-01-14
### Added
Expand Down Expand Up @@ -114,7 +118,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
- Initial stable release

[Unreleased]: https://github.com/phalcon/php-zephir-parser/compare/v1.2.0...HEAD
[1.2.0]: https://github.com/phalcon/php-zephir-parser/compare/v1.1.2...v1.2.0
[1.2.0]: https://github.com/phalcon/php-zephir-parser/compare/v1.1.4...v1.2.0
[1.1.4]: https://github.com/phalcon/php-zephir-parser/compare/v1.1.3...v1.1.4
[1.1.3]: https://github.com/phalcon/php-zephir-parser/compare/v1.1.2...v1.1.3
[1.1.2]: https://github.com/phalcon/php-zephir-parser/compare/v1.1.1...v1.1.2
Expand Down
2 changes: 1 addition & 1 deletion parser/scanner.c
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
#include "scanner.h"

// for re2c
#define YYCTYPE char
#define YYCTYPE unsigned char
#define YYCURSOR (s->cursor)
#define YYLIMIT (s->limit)
#define YYMARKER (s->marker)
Expand Down
2 changes: 1 addition & 1 deletion parser/scanner.re
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
#include "scanner.h"

// for re2c
#define YYCTYPE char
#define YYCTYPE unsigned char
#define YYCURSOR (s->cursor)
#define YYLIMIT (s->limit)
#define YYMARKER (s->marker)
Expand Down
8 changes: 4 additions & 4 deletions parser/zephir.lemon
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@

/*
/* zephir.lemon
*
* This file is part of the Zephir Parser.
*
* (c) Zephir Team <team@zephir-lang.com>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/

%token_prefix XX_
Expand Down
112 changes: 112 additions & 0 deletions tests/unicode/bug56.phpt
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
--TEST--
Using Cyrillic characters in the source code
--SKIPIF--
<?php include(__DIR__ . '/../skipif.inc'); ?>
--FILE--
<?php

$code =<<<ZEP
namespace Test;
class test {
public function testUTF8() {
return "сдфггхх";
}
}
ZEP;

var_dump(zephir_parse_file($code, '(eval code)'));
?>
--EXPECT--
array(2) {
[0]=>
array(5) {
["type"]=>
string(9) "namespace"
["name"]=>
string(4) "Test"
["file"]=>
string(11) "(eval code)"
["line"]=>
int(3)
["char"]=>
int(5)
}
[1]=>
array(8) {
["type"]=>
string(5) "class"
["name"]=>
string(4) "test"
["abstract"]=>
int(0)
["final"]=>
int(0)
["definition"]=>
array(4) {
["methods"]=>
array(1) {
[0]=>
array(8) {
["visibility"]=>
array(1) {
[0]=>
string(6) "public"
}
["type"]=>
string(6) "method"
["name"]=>
string(8) "testUTF8"
["statements"]=>
array(1) {
[0]=>
array(5) {
["type"]=>
string(6) "return"
["expr"]=>
array(5) {
["type"]=>
string(6) "string"
["value"]=>
string(14) "сдфггхх"
["file"]=>
string(11) "(eval code)"
["line"]=>
int(6)
["char"]=>
int(30)
}
["file"]=>
string(11) "(eval code)"
["line"]=>
int(7)
["char"]=>
int(5)
}
}
["file"]=>
string(11) "(eval code)"
["line"]=>
int(5)
["last-line"]=>
int(8)
["char"]=>
int(19)
}
}
["file"]=>
string(11) "(eval code)"
["line"]=>
int(3)
["char"]=>
int(5)
}
["file"]=>
string(11) "(eval code)"
["line"]=>
int(3)
["char"]=>
int(5)
}
}
120 changes: 120 additions & 0 deletions tests/unicode/bug62.phpt
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
--TEST--
Using Chinese characters in the source code
--SKIPIF--
<?php include(__DIR__ . '/../skipif.inc'); ?>
--FILE--
<?php

$code =<<<ZEP
namespace Utils;
class Greeting
{
public static function say()
{
echo "中文";
}
}
ZEP;

var_dump(zephir_parse_file($code, '(eval code)'));
?>
--EXPECT--
array(2) {
[0]=>
array(5) {
["type"]=>
string(9) "namespace"
["name"]=>
string(5) "Utils"
["file"]=>
string(11) "(eval code)"
["line"]=>
int(3)
["char"]=>
int(5)
}
[1]=>
array(8) {
["type"]=>
string(5) "class"
["name"]=>
string(8) "Greeting"
["abstract"]=>
int(0)
["final"]=>
int(0)
["definition"]=>
array(4) {
["methods"]=>
array(1) {
[0]=>
array(8) {
["visibility"]=>
array(2) {
[0]=>
string(6) "public"
[1]=>
string(6) "static"
}
["type"]=>
string(6) "method"
["name"]=>
string(3) "say"
["statements"]=>
array(1) {
[0]=>
array(5) {
["type"]=>
string(4) "echo"
["expressions"]=>
array(1) {
[0]=>
array(5) {
["type"]=>
string(6) "string"
["value"]=>
string(6) "中文"
["file"]=>
string(11) "(eval code)"
["line"]=>
int(8)
["char"]=>
int(20)
}
}
["file"]=>
string(11) "(eval code)"
["line"]=>
int(9)
["char"]=>
int(5)
}
}
["file"]=>
string(11) "(eval code)"
["line"]=>
int(6)
["last-line"]=>
int(11)
["char"]=>
int(26)
}
}
["file"]=>
string(11) "(eval code)"
["line"]=>
int(3)
["char"]=>
int(5)
}
["file"]=>
string(11) "(eval code)"
["line"]=>
int(3)
["char"]=>
int(5)
}
}

0 comments on commit ec5cc41

Please sign in to comment.