Skip to content

Commit

Permalink
Unicode tests (HaxeFoundation#7009)
Browse files Browse the repository at this point in the history 
* started unicode unit tests

* more unicode tests and api changes

* turn off neko (will not be made unicode compatible)

* flash unicode support

* added polyfill for missing IE String.fromCodePoint

* make ucs2 with utf16 encoding the default for tests compliance

* [typer] allow the String module to have multiple module types

I think taking the closure of e.g. String's `fromCharCode` on JS causes a `_String_Impl` class to be created.

* [eval] disable unicode tests

* [java/cs] Update Bytes code to support the RawNative encoding

Also fix `String.fromCharCode` when the code point is a surrogate pair

* [unicode] move some code from HL to common

* [hxcpp] Some work to prepare for utf16 strings

* [hxcpp] Remove compiler warnings.

* [hxcpp] Allow hxcpp strings to be non-utf8

* [hxcpp] Use the same hash for utf8 and wide-char representations.  Remove assumptions about hxcpp string format from crypto.Sha

* [hxcpp] Do not assume utf8 on hxcpp.  Add optimization for Std.string in the case of passed String

* Use native utf8 encoding for hxcpp. Disable unicode tests for cpp without 'hxcpp_smart_strings' define

* Add hxcpp_smart_strings define to compile

* [cpp] remove unused variables

* Add some unicode indexOf tests

* [php] load the file with polyfills

* [php] Converted String to multibyte

* [php] fixed StringTools.fastCodeAt() for utf8

* [php] bytes io seems fixed

* [php] fixed xml.Parser for unicode

* [php] fixed haxe.JsonParser for unicode

* [php] php strings are binary-safe

* [php] fixed sha224 & sha256 tests

* [hxcpp] Export spcial cpp types as fsUnkown to cppia.  Change definition of cpp.Star to allow null setting.  Add cpp.Native class for some easier access to cpp.Star pointers.  Bump hxcpp_api_version to 400.

* [lua] refactor lib installation methods

* [lua] lib version adjustment

* update luarocks lib name

* [lua] Add Utf8 extern, use it for base String class

* [lua] use native string tools inside Bytes

* [lua] use fast byte decoding for utf8

* [lua] remove slice allocation inside of Bytes

* [lua] fix utf8 in xml

* [lua] fix stack overflow for byte decoding

* [lua] adjust offsets for byte encoding

* [lua] get rid of defunct Utf8 implementation

* [lua] remove hardcoded stack size limit

* [lua] set utf8 handling for related std string methods and tests

* [lua] Std.string checks for a userdata metatable and will use that if present

* [php] fixed JsonParser and xml Parser

* [lua] remove special utf8 handling logic from json/xml parser

* [lua] skip sha tests for now

* add more tests because I hate myself

* fix

* [eval] it passes!

* [eval] cleanup

* [eval] try to get substr/substring right and add some tests

* [eval] fix and test Reflect.compare

* [as3] make fromCodePoint public

* [eval] make (last)indexOf ECMA-compliant with regards to ""

see HaxeFoundation#5271

* [lua] use safe table method for decoding bytes to strings

* [lua] reformat NativeStringTools and remove non-existent charCodeAt function

* [lua] use NativeStringTools for byte management in Process

* [python] get tests to pass
  • Loading branch information
@ncannasse @Simn
ncannasse authored and Simn committed Sep 4, 2018
1 parent 3404dd3 commit b4fa222
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 21 deletions.
42 changes: 42 additions & 0 deletions std/php/Global.hx
View file
Expand Up @@ -957,6 +957,11 @@ extern class Global {
**/
static function mb_check_encoding( str:String = null, ?encoding:String ) : Bool;

/**
@see http://php.net/manual/en/function.mb-split.php
**/
static function mb_split( pattern:String, str:String, ?limit:Int ) : NativeIndexedArray<String>;

/**
@see http://php.net/manual/en/function.mb-strlen.php
**/
Expand All @@ -967,6 +972,43 @@ extern class Global {
**/
static function mb_substr( str:String, start:Int, length:Int = null, ?encoding:String ) : String;

/**
@see http://php.net/manual/en/function.mb-chr.php
(Polyfilled for php 7.0)
**/
static function mb_chr( cp:Int, ?encoding:String ) : String;

/**
@see http://php.net/manual/en/function.mb-ord.php
(Polyfilled for php 7.0)
**/
static function mb_ord( str:String, ?encoding:String ) : Int;

/**
@see http://php.net/manual/en/function.mb-regex-encoding.php
**/
static function mb_regex_encoding( ?encoding:String ) : EitherType<Bool,String>;

/**
@see http://php.net/manual/en/function.mb-strtoupper.php
**/
static function mb_strtoupper( str:String, ?encoding:String ) : String;

/**
@see http://php.net/manual/en/function.mb-strpos.php
**/
static function mb_strpos( haystack:String, needle:String, ?offset:Int, ?encoding:String ) : EitherType<Int,Bool>;

/**
@see http://php.net/manual/en/function.mb-strrpos.php
**/
static function mb_strrpos( haystack:String, needle:String, ?offset:Int, ?encoding:String ) : EitherType<Int,Bool>;

/**
@see http://php.net/manual/en/function.mb-strtolower.php
**/
static function mb_strtolower( str:String, ?encoding:String ) : String;

/**
@see http://php.net/manual/en/function.proc-open.php
**/
Expand Down
7 changes: 4 additions & 3 deletions std/php/_std/haxe/io/Bytes.hx
View file
Expand Up @@ -147,10 +147,11 @@ class Bytes {
setInt32(pos + 4, v.high);
}

public inline function getString( pos : Int, len : Int ) : String {
public inline function getString( pos : Int, len : Int, ?encoding : Encoding ) : String {
if( pos < 0 || len < 0 || pos + len > length ) {
throw Error.OutsideBounds;
} else {
//no need to handle encoding, because PHP strings are binary safe.
return b.getString(pos, len);
}
}
Expand All @@ -177,8 +178,8 @@ class Bytes {
return new Bytes(length, BytesData.alloc(length));
}

public static inline function ofString( s : String ) : Bytes {
return new Bytes(s.length, s);
public static inline function ofString( s : String, ?encoding : Encoding ) : Bytes {
return new Bytes(php.Global.strlen(s), s);
}

public static inline function ofData( b : BytesData ) : Bytes {
Expand Down
34 changes: 16 additions & 18 deletions tests/runci/targets/Lua.hx
View file
Expand Up @@ -23,26 +23,24 @@ class Lua {
}
}

static public function installLuaVersionDependencies(lv:String){
if (lv == "-l5.1"){
if (!commandSucceed("luarocks", ["show", "luabit"])) {
runCommand("luarocks", ["install", "luabitop", "1.0.2-3", "--server=https://luarocks.org/dev"]);
}
}
if (!commandSucceed("luarocks", ["show", "lrexlib-pcre"])) {
runCommand("luarocks", ["install", "lrexlib-pcre", "2.8.0-1", "--server=https://luarocks.org/dev"]);
}
if (!commandSucceed("luarocks", ["show", "luv"])) {
runCommand("luarocks", ["install", "luv", "1.9.1-0", "--server=https://luarocks.org/dev"]);
}
if (!commandSucceed("luarocks", ["show", "luasocket"])) {
runCommand("luarocks", ["install", "luasocket", "3.0rc1-2", "--server=https://luarocks.org/dev"]);
}
if (!commandSucceed("luarocks", ["show", "environ"])) {
runCommand("luarocks", ["install", "environ", "0.1.0-1", "--server=https://luarocks.org/dev"]);
static function installLib(lib : String, version : String, server = "https://luarocks.org/dev"){
var server_arg = '--server=$server';
if (!commandSucceed("luarocks", ["show", lib])) {
runCommand("luarocks", ["install",lib, version, server_arg]);
}
}

static public function installLuaVersionDependencies(lv:String){
if (lv == "-l5.1") installLib("luabitop", "1.0.2-3");

installLib("lrexlib-pcre" , "2.8.0-1");
installLib("luv" , "1.9.1-0");
installLib("luasocket" , "3.0rc1-2");
installLib("environ" , "0.1.0-1");
installLib("luautf8" , "0.1.1-1");

}

static public function run(args:Array<String>) {
getLuaDependencies();
var envpath = Sys.getEnv("HOME") + '/lua_env';
Expand Down Expand Up @@ -70,4 +68,4 @@ class Lua {
runCommand("haxe", ["compile.hxml"]);
}
}
}
}

0 comments on commit b4fa222

Please sign in to comment.