diff --git a/ChangeLog b/ChangeLog index 6641b4262b..39856ab1e0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -10,6 +10,7 @@ bytes from every installed executable. [GH #824] + Parrot_get_cpu_type returns now "unknown" for all unknown cpu types. On non-windows it returned before 4.9.0 a null string. [GH #846] + + Fixed src/hash.c for platforms where size_t is unsigned (e.g. vms) [GH #854] - Build + Improved warnings for clang. [GH #843] + Cleaned wrong UNUSED(arg) macros in all pmc's due to an improved @@ -22,14 +23,15 @@ have been fixed. There are only some remaining platforms errors on darwin/x86. - Libraries - + Fixed Mime/Base64 encode_base64. Split long lines at 76 chars. [GH #826] + + Fixed Mime::Base64: Split long lines at 76 chars. [GH #826] + Support multi-byte codepoints. Use binary encoding, + encoded files are now endian specific [GH #813 + #814], + Added 2nd optional encoding arg to decode_base64(). - Documentation + Cleaned up removed parrot cmdline options -D/-O/-v from --help output and running.pod [GH #838] - Tools + Improved gdb pretty-printing for multi-byte strings - - Tests - - Community 2012-09-18 release 4.8.0 - Core diff --git a/MANIFEST b/MANIFEST index ce860f7987..775ec4d243 100644 --- a/MANIFEST +++ b/MANIFEST @@ -583,6 +583,8 @@ examples/library/getopt_demo.pir [examples] examples/library/md5sum.pir [examples] examples/library/ncurses_life.pir [examples] examples/library/pcre.pir [examples] +examples/library/utf8_base64.pir [examples] +examples/library/utf8_base64.pl [examples] examples/mops/README.pod [examples] examples/mops/mops.c [examples] examples/mops/mops.cs [examples] @@ -1710,6 +1712,7 @@ t/library/iter.t [test] t/library/lwp.t [test] t/library/md5.t [test] t/library/mime_base64.t [test] +t/library/mime_base64u.t [test] t/library/nciutils.t [test] t/library/osutils.t [test] t/library/p6object.t [test] diff --git a/MANIFEST.generated b/MANIFEST.generated index 1c23af4f69..783e8aaa2c 100644 --- a/MANIFEST.generated +++ b/MANIFEST.generated @@ -1,14 +1,14 @@ # See tools/dev/install_files.pl for documentation on the # format of this file. # Please re-sort this file after *EVERY* modification -blib/lib/libparrot.4.8.0.dylib [main]lib +blib/lib/libparrot.4.9.0.dylib [main]lib blib/lib/libparrot.a [main]lib blib/lib/libparrot.dylib [main]lib blib/lib/libparrot.so [main]lib -blib/lib/libparrot.so.4.8.0 [main]lib +blib/lib/libparrot.so.4.9.0 [main]lib compilers/data_json/data_json.pbc [data_json] config/gen/call_list/opengl.in [] -cygparrot-4.8.0.dll [main]bin +cygparrot-4.9.0.dll [main]bin docs/ops/bit.pod [doc] docs/ops/cmp.pod [doc] docs/ops/core.pod [doc] diff --git a/README.pod b/README.pod index b4a0f0495c..7825095b49 100644 --- a/README.pod +++ b/README.pod @@ -4,7 +4,7 @@ =head1 NAME -README.pod - Readme to Parrot, version 4.8.0. +README.pod - Readme to Parrot, version 4.9.0. =head1 DESCRIPTION diff --git a/VERSION b/VERSION index 88f181192c..6ed7776bf3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.8.0 +4.9.0 diff --git a/config/auto/gcc.pm b/config/auto/gcc.pm index d3bc38abdd..54c8046643 100644 --- a/config/auto/gcc.pm +++ b/config/auto/gcc.pm @@ -47,8 +47,10 @@ sub _probe_for_gcc { sub _evaluate_gcc { my ($self, $conf, $gnucref) = @_; - # Set gccversion to undef. This will also trigger any hints-file - # callbacks that depend on knowing whether or not we're using gcc. + # Set empty noonline and gccversion to undef. + # This will also trigger any hints-file callbacks that depend on + # knowing whether or not we're using gcc. + $conf->data->set( noinline => '' ); # This key should always exist unless the program couldn't be run, # which should have been caught by the 'die' above. diff --git a/config/auto/llvm.pm b/config/auto/llvm.pm index cfede4f504..f959508f3a 100644 --- a/config/auto/llvm.pm +++ b/config/auto/llvm.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2009-2011, Parrot Foundation. +# Copyright (C) 2009-2012, Parrot Foundation. =head1 NAME diff --git a/config/auto/thread.pm b/config/auto/thread.pm index ec4bd51183..fe400128c0 100644 --- a/config/auto/thread.pm +++ b/config/auto/thread.pm @@ -6,7 +6,7 @@ config/auto/thread.pm - Thread support =head1 DESCRIPTION -Determining if the system has a Thread support. +Determining if the system has thread support. =cut @@ -22,7 +22,7 @@ use Parrot::Configure::Utils ':auto'; sub _init { my $self = shift; my %data; - $data{description} = q{Does your system has thread}; + $data{description} = q{Does your system have threads}; $data{result} = q{}; return \%data; } diff --git a/config/gen/makefiles/root.in b/config/gen/makefiles/root.in index 3b619b86ff..c2e736414c 100644 --- a/config/gen/makefiles/root.in +++ b/config/gen/makefiles/root.in @@ -596,7 +596,7 @@ MAKE_C = @make_c@ .c$(O) : # suffix rule (limited support) $(CC) $(CFLAGS) @optimize@ $(CC_WARN) -I$(@D) -Isrc/ @cc_o_out@$@ -c $< .c.i : # suffix rule (limited support) - $(CC) -E $(CFLAGS) -I$(@D) -Isrc/ @cc_o_out@$@ -c $< + $(CC) -E $(CFLAGS) -I$(@D) -Isrc/ @cc_o_out@$@ $< #UNLESS(win32):.s$(O) : # suffix rule (limited support) #UNLESS(win32): $(CC) $(CFLAGS) @optimize@ $(CC_WARN) -I$(@D) @cc_o_out@$@ -c $< diff --git a/docs/parrothist.pod b/docs/parrothist.pod index 6dc98d4398..b341a75894 100644 --- a/docs/parrothist.pod +++ b/docs/parrothist.pod @@ -147,5 +147,6 @@ Starred release numbers indicate supported releases. rurban 4.6.0 * 2012-Jul-17 "Wild Parrots of Telegraph Hill" Whiteknight 4.7.0 2012-Aug-22 "Hispaniolan" alvis 4.8.0 2012-Sep-18 "Spix's Macaw" + dukeleto 4.9.0 * 2012-Oct-16 "Proto-Hydra" =cut diff --git a/docs/project/release_manager_guide.pod b/docs/project/release_manager_guide.pod index 55faab9794..ab990adc87 100644 --- a/docs/project/release_manager_guide.pod +++ b/docs/project/release_manager_guide.pod @@ -544,7 +544,6 @@ The calendar of releases is available at the C Google calendar, visible at L. - - Oct 16, 2012 - 4.9.0 - dukeleto - Nov 20, 2012 - 4.10.0 - rurban - Dec 18, 2012 - 4.11.0 - ?? - Jan 15, 2013 - 5.0.0 - dukeleto diff --git a/examples/library/utf8_base64.pir b/examples/library/utf8_base64.pir new file mode 100644 index 0000000000..c07f4e9205 --- /dev/null +++ b/examples/library/utf8_base64.pir @@ -0,0 +1,57 @@ +#!./parrot +# Copyright (C) 2012, Parrot Foundation. + +=head1 NAME + +examples/mime_base64/utf_base64.pir - Conformant MIME::Base64 utf8 handling + +=head1 SYNOPSIS + + % ./parrot examples/mime_base64/utf_base64.pir + +=head1 DESCRIPTION + +Compare conformant coreutils C and F +against ours. +See L + +=cut + +.sub main :main + load_bytecode 'MIME/Base64.pbc' + + .local pmc enc_sub + enc_sub = get_global [ "MIME"; "Base64" ], 'encode_base64' + + .local string result_encode + # GH 814 + result_encode = enc_sub(utf8:"\x{a2}") + say "encode: utf8:\"\\x{a2}\"" + say "expected: wqI=" + print "result: " + say result_encode + + # GH 813 + result_encode = enc_sub(utf8:"\x{203e}") + say "encode: utf8:\"\\x{203e}\"" + say "expected: 4oC+" + print "result: " + say result_encode + +.end + +=head1 AUTHOR + +ronaldxs + +=head1 SEE ALSO + +F, + +=cut + +# Local Variables: +# mode: pir +# fill-column: 100 +# End: +# vim: expandtab shiftwidth=4 ft=pir: diff --git a/examples/library/utf8_base64.pl b/examples/library/utf8_base64.pl new file mode 100644 index 0000000000..f4e12d5c86 --- /dev/null +++ b/examples/library/utf8_base64.pl @@ -0,0 +1,59 @@ +#! perl +# Copyright (C) 2012, Parrot Foundation. + +=head1 NAME + +examples/mime_base64/utf_base64.pl - Conformant MIME::Base64 utf8 handling + +=head1 SYNOPSIS + + % perl examples/mime_base64/utf_base64.pl + +=head1 DESCRIPTION + +Compare conformant coreutils C and F +against parrots. +See L + +Note: Unicode stored as MIME::Base64 is inherently endian-dependent. + +=cut + +use strict; +use warnings; +use MIME::Base64 qw(encode_base64 decode_base64); +use Encode qw(encode); + +my $encoded = encode_base64(encode("UTF-8", "\x{a2}")); +print "encode: utf-8:\"\\x{a2}\" - ", encode("UTF-8", "\x{a2}"), "\n"; +print "expected: wqI=\n"; +print "result: $encoded\n"; +print "decode: ",decode_base64("wqI="),"\n\n"; # 302 242 + +my $encoded = encode_base64(encode("UTF-8", "\x{203e}")); +print "encode: utf-8:\"\\x{203e}\" -> ",encode("UTF-8", "\x{203e}"),"\n"; +print "expected: 4oC+\n"; +print "result: $encoded\n"; # 342 200 276 +print "decode: ",decode_base64("4oC+"),"\n"; + +for ([qq(a2)],[qq(c2a2)],[qw(203e)],[qw(3e 20)],[qw(1000)],[qw(00c7)],[qw(00ff 0000)]){ + $s = pack "H*",@{$_}; + printf "0x%s\t=> %s", join("",@{$_}), encode_base64($s); +} + +=head1 AUTHORS + +ronaldxs, Reini Urban + +=head1 SEE ALSO + +F, + +=cut + +# Local Variables: +# mode: cperl +# cperl-indent-level: 4 +# fill-column: 100 +# End: +# vim: expandtab shiftwidth=4: diff --git a/include/parrot/oplib/core_ops.h b/include/parrot/oplib/core_ops.h index 1912f72b3a..a7d6fc2fb8 100644 --- a/include/parrot/oplib/core_ops.h +++ b/include/parrot/oplib/core_ops.h @@ -19,7 +19,7 @@ #include "parrot/runcore_api.h" PARROT_EXPORT -op_lib_t *Parrot_DynOp_core_4_8_0(PARROT_INTERP, long init); +op_lib_t *Parrot_DynOp_core_4_9_0(PARROT_INTERP, long init); opcode_t * Parrot_end(opcode_t *, PARROT_INTERP); opcode_t * Parrot_noop(opcode_t *, PARROT_INTERP); diff --git a/runtime/parrot/library/MIME/Base64.pir b/runtime/parrot/library/MIME/Base64.pir index dd4567d6bd..712f649a4b 100644 --- a/runtime/parrot/library/MIME/Base64.pir +++ b/runtime/parrot/library/MIME/Base64.pir @@ -25,19 +25,18 @@ is the string to encode. The returned encoded string is broken into lines of no more than 76 characters each. -=item C +Note: Unicode stored as MIME::Base64 is inherently endian-dependent. + +=item C Decode a base64 string by calling the decode_base64() function. -This function takes a single argument which is the string to decode -and returns the decoded data. +This function takes as first argument the string to decode, +as optional second argument the encoding string for the decoded data. +It returns the decoded data. Any character not part of the 65-character base64 subset is silently ignored. Characters occurring after a '=' padding character are never decoded. -If the length of the string to decode, after ignoring non-base64 chars, -is not a multiple of 4 or if padding occurs too early, -then a warning is generated if perl is running under -w. - =back =cut @@ -93,14 +92,19 @@ then a warning is generated if perl is running under -w. six_to_eight = get_global 'six_to_eight' .local int len, len_mod_3 - len = length plain - len_mod_3 = len % 3 + .local pmc bb + # For unicode we cannot use chr/ord. This breaks endianness. + # GH 813 and #814 + len = bytelength plain + bb = new ['ByteBuffer'], len + bb = plain + len_mod_3 = len % 3 # Fill up with with null bytes if len_mod_3 == 0 goto END_1 - plain = concat plain, ascii:"\0" + push bb, 0 if len_mod_3 == 2 goto END_1 - plain = concat plain, ascii:"\0" + push bb, 0 END_1: base64 = '' @@ -117,12 +121,11 @@ then a warning is generated if perl is running under -w. if i >= len goto END_3 # read 3*8 bits - # TODO GH #813 and #814 unicode chars - eight_0 = ord plain, i + eight_0 = bb[i] inc i - eight_1 = ord plain, i + eight_1 = bb[i] inc i - eight_2 = ord plain, i + eight_2 = bb[i] inc i # d[i]>>2; @@ -179,32 +182,37 @@ then a warning is generated if perl is running under -w. .sub decode_base64 .param string base64 + .param string enc :optional + .param int has_enc :opt_flag - .local string plain, base64_cleaned + .local string result, base64_cleaned + .local int enc_num base64_cleaned = '' - plain = '' + if has_enc goto HAS_ENC + enc = 'ascii' + HAS_ENC: - .local pmc eight_to_six + .local pmc eight_to_six, bb eight_to_six = get_global 'eight_to_six' .local int i, len .local int tmp_int_1, tmp_int_2 - .local string s_tmp_1 # Get rid of non-base64 chars len = length base64 i = 0 - START_5: + START_5: + .local string s_tmp_1 if i >= len goto END_5 tmp_int_1 = ord base64, i - inc i - tmp_int_2 = eight_to_six[tmp_int_1] - if tmp_int_2 == -1 goto START_5 - s_tmp_1 = chr tmp_int_1 - base64_cleaned = concat base64_cleaned, s_tmp_1 + inc i + tmp_int_2 = eight_to_six[tmp_int_1] + if tmp_int_2 == -1 goto START_5 + s_tmp_1 = chr tmp_int_1 + base64_cleaned = concat base64_cleaned, s_tmp_1 goto START_5 - END_5: + END_5: .local int len_mod_4 len = length base64_cleaned len_mod_4 = len % 4 @@ -212,27 +220,27 @@ then a warning is generated if perl is running under -w. # make sure that there are dummy bits beyond base64_cleaned = concat base64_cleaned, ascii:"\0\0\0" + bb = new ['ByteBuffer'] .local int eight_0, eight_1, eight_2 .local int six_0, six_1, six_2, six_3 i = 0 - START_2: + START_2: if i >= len goto END_2 # read 4*6 bits tmp_int_1 = ord base64_cleaned, i - six_0 = eight_to_six[tmp_int_1] - inc i + six_0 = eight_to_six[tmp_int_1] + inc i tmp_int_1 = ord base64_cleaned, i - six_1 = eight_to_six[tmp_int_1] - inc i + six_1 = eight_to_six[tmp_int_1] + inc i tmp_int_1 = ord base64_cleaned, i - six_2 = eight_to_six[tmp_int_1] - inc i + six_2 = eight_to_six[tmp_int_1] + inc i tmp_int_1 = ord base64_cleaned, i - six_3 = eight_to_six[tmp_int_1] - inc i - + six_3 = eight_to_six[tmp_int_1] + inc i # (f64[t.charAt(i)]<<2) | (f64[t.charAt(i+1)]>>4) shl tmp_int_1, six_0, 2 @@ -252,25 +260,25 @@ then a warning is generated if perl is running under -w. # write 3*8 bits # output is larger than input - s_tmp_1 = chr eight_0 - plain = concat plain, s_tmp_1 - s_tmp_1 = chr eight_1 - plain = concat plain, s_tmp_1 - s_tmp_1 = chr eight_2 - plain = concat plain, s_tmp_1 - + push bb, eight_0 + push bb, eight_1 + push bb, eight_2 goto START_2 - END_2: + END_2: # cut padded '=' if len_mod_4 == 0 goto END_3 if len_mod_4 == 1 goto END_3 - plain = chopn plain, 1 + len = elements bb + dec len + bb = len if len_mod_4 == 3 goto END_3 - plain = chopn plain, 1 - END_3: + dec len + bb = len - .return( plain ) + END_3: + result = bb.'get_string'(enc) + .return( result ) .end =head1 SEE ALSO diff --git a/src/ops/core_ops.c b/src/ops/core_ops.c index 131193790d..44d135d58f 100644 --- a/src/ops/core_ops.c +++ b/src/ops/core_ops.c @@ -24439,7 +24439,7 @@ op_lib_t core_op_lib = { PARROT_FUNCTION_CORE, /* core_type = PARROT_XX_CORE */ 0, /* flags */ 4, /* major_version */ - 8, /* minor_version */ + 9, /* minor_version */ 0, /* patch_version */ 1125, /* op_count */ core_op_info_table, /* op_info_table */ @@ -24568,7 +24568,7 @@ static void hop_deinit(PARROT_INTERP) hop_buckets = NULL; }PARROT_EXPORT op_lib_t * -Parrot_DynOp_core_4_8_0(PARROT_INTERP, long init) { +Parrot_DynOp_core_4_9_0(PARROT_INTERP, long init) { /* initialize and return op_lib ptr */ if (init == 1) { @@ -24597,7 +24597,7 @@ Parrot_lib_core_ops_load(PARROT_INTERP) { PMC *const lib = Parrot_pmc_new(interp, enum_class_ParrotLibrary); - ((Parrot_ParrotLibrary_attributes*)PMC_data(lib))->oplib_init = (void *) Parrot_DynOp_core_4_8_0; + ((Parrot_ParrotLibrary_attributes*)PMC_data(lib))->oplib_init = (void *) Parrot_DynOp_core_4_9_0; dynop_register(interp, lib); return lib; } diff --git a/src/ops/string.ops b/src/ops/string.ops index 9920dad3a9..5871271f32 100644 --- a/src/ops/string.ops +++ b/src/ops/string.ops @@ -59,6 +59,10 @@ inline op ord(out INT, in STR, in INT) { The character specified by codepoint integer $2 is returned in string $1. +For characters > 0xff an utf8 encoded string is returned, +for characters between 0x7f and 0xff a latin1 encoded string is returned, +for characters below 0x7f an ascii encoded string. + =cut inline op chr(out STR, in INT) { diff --git a/t/compilers/data_json/from_parrot.t b/t/compilers/data_json/from_parrot.t index 92bef5917f..bacc1761c2 100644 --- a/t/compilers/data_json/from_parrot.t +++ b/t/compilers/data_json/from_parrot.t @@ -1,5 +1,5 @@ #!./parrot -# Copyright (C) 2001-2010, Parrot Foundation. +# Copyright (C) 2001-2012, Parrot Foundation. =head1 NAME @@ -13,11 +13,13 @@ t/compilers/data_json/from_parrot.t - test parrot to JSON conversion. Tests JSON->Parrot conversions. +Note: This uses the old JSON.pbc not the new data_json compiler. + =cut .sub main :main .include 'test_more.pir' - plan(39) + plan(40) load_bytecode 'JSON.pbc' test_create_json_of_an_empty_string() @@ -356,7 +358,7 @@ OUTPUT .end -# no. 24..27 +# no. 24..28 .sub test_create_json_of_string_pmcs .local pmc s @@ -373,10 +375,13 @@ OUTPUT is($S0, '"12345\"67890"', 'Create JSON of String PMCs') $S0 = _json( s, 1 ) is($S0, "\"12345\\\"67890\"\n", 'Create JSON of String PMCs') + s = utf16:"\x{0}\u203e Pl\x{e4}ne" + $S0 = _json( s, 0 ) + is($S0, '"\x{0}\u203e Pl\x{e4}ne"', 'Create JSON of String PMCs') .end -# no. 28..31 +# no. 29..32 .sub test_create_json_of_integer_pmcs .local pmc i @@ -396,7 +401,7 @@ OUTPUT .end -# no. 32..35 +# no. 33..36 .sub test_create_json_of_boolean_pmcs .local pmc b @@ -416,7 +421,7 @@ OUTPUT .end -# no. 36..39 +# no. 37..40 .sub test_create_json_of_null_and_undef .local pmc n null n diff --git a/t/compilers/data_json/to_parrot.t b/t/compilers/data_json/to_parrot.t index 34e7861200..f151adbbe1 100644 --- a/t/compilers/data_json/to_parrot.t +++ b/t/compilers/data_json/to_parrot.t @@ -1,12 +1,12 @@ #!perl -# Copyright (C) 2001-2008, Parrot Foundation. +# Copyright (C) 2001-2012, Parrot Foundation. use strict; use warnings; use lib qw( t . lib ../lib ../../lib ); use Test::More; -use Parrot::Test tests => 60; +use Parrot::Test tests => 61; =head1 NAME @@ -20,6 +20,8 @@ t/compilers/data_json/to_parrot.t - test JSON to parrot conversions Tests JSON->Parrot conversions. +Note: This uses the new data_json compiler. + =cut json_dump_is( <<'JSON', <<'OUT', 'empty string' ); @@ -677,6 +679,21 @@ JSON ] OUT +json_dump_is( <<'JSON', <<'OUT', 'unicode chars' ); +["\u0000","\u00e4","\u007f","\u0080","\u0100","\u203e","Pl\u00e4ne"] +JSON +"JSON" => ResizablePMCArray (size:7) [ + "\x{0}", + "\x{e4}", + "\x{7f}", + "\x{80}", + "\u0100", + "\u203e", + "Pl\x{e4}ne" +] +OUT + + # GH #570 Need many more tests, exercising all aspects of http://www.json.org/ sub json_dump_is { diff --git a/t/library/mime_base64.t b/t/library/mime_base64.t index 940087a6ed..ef9c98c23f 100644 --- a/t/library/mime_base64.t +++ b/t/library/mime_base64.t @@ -24,12 +24,12 @@ Test cases taken from base64.t of MIME::Base64. load_bytecode 'PGE/Util.pbc' load_language 'data_json' - .local pmc plan, is, ok - plan = get_hll_global [ 'Test'; 'More' ], 'plan' - is = get_hll_global [ 'Test'; 'More' ], 'is' - ok = get_hll_global [ 'Test'; 'More' ], 'ok' + .local pmc plan, is, ok, lives_ok + plan = get_hll_global [ 'Test'; 'More' ], 'plan' + is = get_hll_global [ 'Test'; 'More' ], 'is' + ok = get_hll_global [ 'Test'; 'More' ], 'ok' - plan(550) + plan(551) .local pmc json json = compreg 'data_json' @@ -37,7 +37,7 @@ Test cases taken from base64.t of MIME::Base64. .local pmc encode_decode_tests, decode_tests encode_decode_tests = json.'compile'( <<'END_JSON' ) [ ["Hello, World!\n","SGVsbG8sIFdvcmxkIQo="], - ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh\nYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYQ=="], ["\u0000","AA=="], ["\u0001","AQ=="], @@ -167,136 +167,136 @@ Test cases taken from base64.t of MIME::Base64. ["}","fQ=="], ["~","fg=="], ["\u007f","fw=="], - ["\u0080","gA=="], - ["\u0081","gQ=="], - ["\u0082","gg=="], - ["\u0083","gw=="], - ["\u0084","hA=="], - ["\u0085","hQ=="], - ["\u0086","hg=="], - ["\u0087","hw=="], - ["\u0088","iA=="], - ["\u0089","iQ=="], - ["\u008a","ig=="], - ["\u008b","iw=="], - ["\u008c","jA=="], - ["\u008d","jQ=="], - ["\u008e","jg=="], - ["\u008f","jw=="], - ["\u0090","kA=="], - ["\u0091","kQ=="], - ["\u0092","kg=="], - ["\u0093","kw=="], - ["\u0094","lA=="], - ["\u0095","lQ=="], - ["\u0096","lg=="], - ["\u0097","lw=="], - ["\u0098","mA=="], - ["\u0099","mQ=="], - ["\u009a","mg=="], - ["\u009b","mw=="], - ["\u009c","nA=="], - ["\u009d","nQ=="], - ["\u009e","ng=="], - ["\u009f","nw=="], - ["\u00a0","oA=="], - ["\u00a1","oQ=="], - ["\u00a2","og=="], - ["\u00a3","ow=="], - ["\u00a4","pA=="], - ["\u00a5","pQ=="], - ["\u00a6","pg=="], - ["\u00a7","pw=="], - ["\u00a8","qA=="], - ["\u00a9","qQ=="], - ["\u00aa","qg=="], - ["\u00ab","qw=="], - ["\u00ac","rA=="], - ["\u00ad","rQ=="], - ["\u00ae","rg=="], - ["\u00af","rw=="], - ["\u00b0","sA=="], - ["\u00b1","sQ=="], - ["\u00b2","sg=="], - ["\u00b3","sw=="], - ["\u00b4","tA=="], - ["\u00b5","tQ=="], - ["\u00b6","tg=="], - ["\u00b7","tw=="], - ["\u00b8","uA=="], - ["\u00b9","uQ=="], - ["\u00ba","ug=="], - ["\u00bb","uw=="], - ["\u00bc","vA=="], - ["\u00bd","vQ=="], - ["\u00be","vg=="], - ["\u00bf","vw=="], - ["\u00c0","wA=="], - ["\u00c1","wQ=="], - ["\u00c2","wg=="], - ["\u00c3","ww=="], - ["\u00c4","xA=="], - ["\u00c5","xQ=="], - ["\u00c6","xg=="], - ["\u00c7","xw=="], - ["\u00c8","yA=="], - ["\u00c9","yQ=="], - ["\u00ca","yg=="], - ["\u00cb","yw=="], - ["\u00cc","zA=="], - ["\u00cd","zQ=="], - ["\u00ce","zg=="], - ["\u00cf","zw=="], - ["\u00d0","0A=="], - ["\u00d1","0Q=="], - ["\u00d2","0g=="], - ["\u00d3","0w=="], - ["\u00d4","1A=="], - ["\u00d5","1Q=="], - ["\u00d6","1g=="], - ["\u00d7","1w=="], - ["\u00d8","2A=="], - ["\u00d9","2Q=="], - ["\u00da","2g=="], - ["\u00db","2w=="], - ["\u00dc","3A=="], - ["\u00dd","3Q=="], - ["\u00de","3g=="], - ["\u00df","3w=="], - ["\u00e0","4A=="], - ["\u00e1","4Q=="], - ["\u00e2","4g=="], - ["\u00e3","4w=="], - ["\u00e4","5A=="], - ["\u00e5","5Q=="], - ["\u00e6","5g=="], - ["\u00e7","5w=="], - ["\u00e8","6A=="], - ["\u00e9","6Q=="], - ["\u00ea","6g=="], - ["\u00eb","6w=="], - ["\u00ec","7A=="], - ["\u00ed","7Q=="], - ["\u00ee","7g=="], - ["\u00ef","7w=="], - ["\u00f0","8A=="], - ["\u00f1","8Q=="], - ["\u00f2","8g=="], - ["\u00f3","8w=="], - ["\u00f4","9A=="], - ["\u00f5","9Q=="], - ["\u00f6","9g=="], - ["\u00f7","9w=="], - ["\u00f8","+A=="], - ["\u00f9","+Q=="], - ["\u00fa","+g=="], - ["\u00fb","+w=="], - ["\u00fc","/A=="], - ["\u00fd","/Q=="], - ["\u00fe","/g=="], - ["\u00ff","/w=="], - ["\u0000\u00ff","AP8="], - ["\u00ff\u0000","/wA="], + ["\u0080","woA="], + ["\u0081","woE="], + ["\u0082","woI="], + ["\u0083","woM="], + ["\u0084","woQ="], + ["\u0085","woU="], + ["\u0086","woY="], + ["\u0087","woc="], + ["\u0088","wog="], + ["\u0089","wok="], + ["\u008a","woo="], + ["\u008b","wos="], + ["\u008c","wow="], + ["\u008d","wo0="], + ["\u008e","wo4="], + ["\u008f","wo8="], + ["\u0090","wpA="], + ["\u0091","wpE="], + ["\u0092","wpI="], + ["\u0093","wpM="], + ["\u0094","wpQ="], + ["\u0095","wpU="], + ["\u0096","wpY="], + ["\u0097","wpc="], + ["\u0098","wpg="], + ["\u0099","wpk="], + ["\u009a","wpo="], + ["\u009b","wps="], + ["\u009c","wpw="], + ["\u009d","wp0="], + ["\u009e","wp4="], + ["\u009f","wp8="], + ["\u00a0","wqA="], + ["\u00a1","wqE="], + ["\u00a2","wqI="], + ["\u00a3","wqM="], + ["\u00a4","wqQ="], + ["\u00a5","wqU="], + ["\u00a6","wqY="], + ["\u00a7","wqc="], + ["\u00a8","wqg="], + ["\u00a9","wqk="], + ["\u00aa","wqo="], + ["\u00ab","wqs="], + ["\u00ac","wqw="], + ["\u00ad","wq0="], + ["\u00ae","wq4="], + ["\u00af","wq8="], + ["\u00b0","wrA="], + ["\u00b1","wrE="], + ["\u00b2","wrI="], + ["\u00b3","wrM="], + ["\u00b4","wrQ="], + ["\u00b5","wrU="], + ["\u00b6","wrY="], + ["\u00b7","wrc="], + ["\u00b8","wrg="], + ["\u00b9","wrk="], + ["\u00ba","wro="], + ["\u00bb","wrs="], + ["\u00bc","wrw="], + ["\u00bd","wr0="], + ["\u00be","wr4="], + ["\u00bf","wr8="], + ["\u00c0","w4A="], + ["\u00c1","w4E="], + ["\u00c2","w4I="], + ["\u00c3","w4M="], + ["\u00c4","w4Q="], + ["\u00c5","w4U="], + ["\u00c6","w4Y="], + ["\u00c7","w4c="], + ["\u00c8","w4g="], + ["\u00c9","w4k="], + ["\u00ca","w4o="], + ["\u00cb","w4s="], + ["\u00cc","w4w="], + ["\u00cd","w40="], + ["\u00ce","w44="], + ["\u00cf","w48="], + ["\u00d0","w5A="], + ["\u00d1","w5E="], + ["\u00d2","w5I="], + ["\u00d3","w5M="], + ["\u00d4","w5Q="], + ["\u00d5","w5U="], + ["\u00d6","w5Y="], + ["\u00d7","w5c="], + ["\u00d8","w5g="], + ["\u00d9","w5k="], + ["\u00da","w5o="], + ["\u00db","w5s="], + ["\u00dc","w5w="], + ["\u00dd","w50="], + ["\u00de","w54="], + ["\u00df","w58="], + ["\u00e0","w6A="], + ["\u00e1","w6E="], + ["\u00e2","w6I="], + ["\u00e3","w6M="], + ["\u00e4","w6Q="], + ["\u00e5","w6U="], + ["\u00e6","w6Y="], + ["\u00e7","w6c="], + ["\u00e8","w6g="], + ["\u00e9","w6k="], + ["\u00ea","w6o="], + ["\u00eb","w6s="], + ["\u00ec","w6w="], + ["\u00ed","w60="], + ["\u00ee","w64="], + ["\u00ef","w68="], + ["\u00f0","w7A="], + ["\u00f1","w7E="], + ["\u00f2","w7I="], + ["\u00f3","w7M="], + ["\u00f4","w7Q="], + ["\u00f5","w7U="], + ["\u00f6","w7Y="], + ["\u00f7","w7c="], + ["\u00f8","w7g="], + ["\u00f9","w7k="], + ["\u00fa","w7o="], + ["\u00fb","w7s="], + ["\u00fc","w7w="], + ["\u00fd","w70="], + ["\u00fe","w74="], + ["\u00ff","w78="], + ["\u0000\u00ff","AMO/"], + ["\u00ff\u0000","w78A"], ["\u0000\u0000\u0000","AAAA"], ["",""], ["a","YQ=="], @@ -332,7 +332,7 @@ END_JSON .local int count count = 0 .local pmc test_iterator, test_case - .local string plain, base64, comment, comment_count + .local string plain, base64, comment, comment_count, enc, esc_plain encode_decode_tests = encode_decode_tests() test_iterator = iter encode_decode_tests @@ -343,9 +343,15 @@ END_JSON base64 = shift test_case comment = 'encode' comment_count = count + $I0 = encoding plain + enc = encodingname $I0 + esc_plain = escape plain comment = concat comment, comment_count - # comment = concat comment, " " - # comment = concat comment, plain + comment = concat comment, " " + comment = concat comment, enc + comment = concat comment, ":\"" + comment = concat comment, esc_plain + comment = concat comment, "\"" test_encode( plain, base64, comment ) comment = 'decode' comment_count = count @@ -370,6 +376,24 @@ END_JSON goto dec_loop dec_loop_end: + gh813_base64_utf8() + +.end + +.sub gh813_base64_utf8 + .local pmc lives_ok + lives_ok = get_hll_global [ 'Test'; 'More' ], 'lives_ok' + + lives_ok(<<'CODE', 'enc_sub("\x{203e}") # Github issue #813') +.sub foo + .local pmc enc_sub + enc_sub = get_global [ "MIME"; "Base64" ], 'encode_base64' + + .local string result_encode + result_encode = enc_sub(utf8:"\x{203e}") +.end +CODE + .end .sub test_encode @@ -394,23 +418,66 @@ END_JSON .param string base64 .param string comment + .include "iglobals.pasm" + .local pmc interp + interp = getinterp + .local pmc config + config = interp[.IGLOBALS_CONFIG_HASH] + .local int has_icu + has_icu = config['has_icu'] + .local int bigendian + bigendian = config['bigendian'] + .local pmc dec_sub dec_sub = get_global [ "MIME"; "Base64" ], 'decode_base64' - .local pmc eight_to_six - eight_to_six = get_global 'eight_to_six' - - .local pmc is + .local pmc is, skip is = get_hll_global [ 'Test'; 'More' ], 'is' - - .local string result_decode - result_decode = dec_sub( base64 ) - is( result_decode, plain, comment ) + skip = get_hll_global [ 'Test'; 'More' ], 'skip' + + $S0 = 'AAAA' + ne base64, $S0, CONT_TEST + ## Note: also fails on solaris little-endian + skip(1, '\0\0\0 fails to compare for unknown reasons GH #855') + goto END + unless bigendian goto CONT_TEST + skip(1, 'multi-byte codepoint test in big-endian') + goto END + CONT_TEST: + .local string decode, result_decode + .local string enc, enc1 + $I0 = encoding plain + enc = encodingname $I0 + if $I0 > 2 goto DEC_ENC + # ascii, latin1 + decode = dec_sub( base64 ) + decode = trans_encoding decode, $I0 + goto DEC_2 + DEC_ENC: + decode = dec_sub( base64, enc ) + DEC_2: + $I1 = encoding decode + enc1 = encodingname $I1 + comment = concat comment, " " + comment = concat comment, enc1 + comment = concat comment, " <-" + comment = concat comment, enc + + .local string plain_norm, result_norm + if has_icu goto HAS_ICU + is( decode, plain, comment ) + goto END + HAS_ICU: + result_norm = compose decode + plain_norm = compose plain + is( result_norm, plain_norm, comment ) + END: .end + =head1 AUTHOR -Bernhard Schmalhofer and others. +Bernhard Schmalhofer and Reini Urban. =cut diff --git a/t/library/mime_base64u.t b/t/library/mime_base64u.t new file mode 100644 index 0000000000..e1126e9e97 --- /dev/null +++ b/t/library/mime_base64u.t @@ -0,0 +1,287 @@ +#!./parrot +# Copyright (C) 2006-2012, Parrot Foundation. + +=head1 NAME + +t/library/mime_base64u.t - test unicode [ 'MIME'; 'Base64' ] + +=head1 SYNOPSIS + + % prove t/library/mime_base64u.t + +=head1 DESCRIPTION + +Test non-ascii encoded MIME::Base64, without JSON. + +perl -MMIME::Base64 -e'for ([qq(a2)],[qq(c2a2)],[qw(203e)],[qw(3e 20)],[qw(1000)],[qw(00c7)],[qw(00ff 0000)]) +{ $s=pack "H*",@{$_}; printf "0x%s\t=> %s",join("",@{$_}),encode_base64($s) }' + +=cut + +.sub test :main + load_bytecode 'Test/More.pbc' + load_bytecode 'MIME/Base64.pir' + + .local pmc plan, is, ok + plan = get_hll_global [ 'Test'; 'More' ], 'plan' + is = get_hll_global [ 'Test'; 'More' ], 'is' + ok = get_hll_global [ 'Test'; 'More' ], 'ok' + + plan(20) + + .local pmc encode_decode_tests, decode_tests + .local int i, size + i = 0 + size = 9 + encode_decode_tests = new 'FixedPMCArray', size + + $P0 = new 'FixedStringArray', 2 + $P0[0] = binary:"\x3e\x20" # same as 0x203e bswapped + $P0[1] = "PiA=" + encode_decode_tests[i] = $P0 + inc i + $P0 = new 'FixedStringArray', 2 + $P0[0] = ucs2:"\x{203e}" # OVERLINE ‾ 0x203e + # TODO on big endian we'll get ID4= + $P0[1] = "PiA=" + encode_decode_tests[i] = $P0 + inc i + $P0 = new 'FixedStringArray', 2 + $P0[0] = utf8:"\u203e" # OVERLINE ‾ 0xe280be in utf8 + $P0[1] = "4oC+" + encode_decode_tests[i] = $P0 + inc i + $P0 = new 'FixedStringArray', 2 + $P0[0] = iso-8859-1:"\x{a2}" # ¢ + $P0[1] = "og==" + encode_decode_tests[i] = $P0 + inc i + $P0 = new 'FixedStringArray', 2 + $P0[0] = utf8:"\x{a2}" # ¢ 0xc2a2 in utf8 + $P0[1] = "wqI=" + encode_decode_tests[i] = $P0 + inc i + $P0 = new 'FixedStringArray', 2 + $P0[0] = ucs2:"\x{0100}" # Ā 0xc480 in utf8 + $P0[1] = "AAE=" + encode_decode_tests[i] = $P0 + inc i + $P0 = new 'FixedStringArray', 2 + $P0[0] = utf8:"\u0100" # Ā + $P0[1] = "xIA=" + encode_decode_tests[i] = $P0 + inc i + $P0 = new 'FixedStringArray', 2 + $P0[0] = utf16:"\x{00c7}" # Ç + $P0[1] = "xwA=" + encode_decode_tests[i] = $P0 + inc i + $P0 = new 'FixedStringArray', 2 + $P0[0] = ucs2:"\x{00ff}\x{0000}" # "ÿ\0" + $P0[1] = "/wAAAA==" + encode_decode_tests[i] = $P0 + inc i + + decode_tests = new 'FixedPMCArray', 2 + $P0 = new 'FixedStringArray', 2 + $P0[0] = "YWE=" + $P0[1] = "aa" + decode_tests[0] = $P0 + $P0 = new 'FixedStringArray', 2 + $P0[0] = "YQ" + $P0[1] = "a" + decode_tests[1] = $P0 + + .local int count, enc_num, hex_cnt, bufused, len + count = 0 + .local pmc test_iterator, test_case + .local string plain, base64, comment, comment_count, enc, esc_plain, hex_str + hex_str = '' + + .include "iglobals.pasm" + .local pmc interp + interp = getinterp + .local pmc config + config = interp[.IGLOBALS_CONFIG_HASH] + .local int bigendian + bigendian = config['bigendian'] + + unless bigendian goto BE_INIT + .local pmc be_result + be_result = new 'FixedStringArray', size + test_iterator = iter encode_decode_tests + be_loop: + unless test_iterator goto be_loop_end + test_case = shift test_iterator + base64 = test_case[1] + be_result[count] = base64 + inc count + goto be_loop + be_loop_end: + be_result[1] = 'ID4=' + be_result[5] = 'AQA=' + be_result[7] = 'AMc=' + be_result[8] = 'AP8AAA==' + + BE_INIT: + count = 0 + test_iterator = iter encode_decode_tests + enc_dec_loop: + unless test_iterator goto enc_dec_loop_end + test_case = shift test_iterator + plain = test_case[0] + base64 = test_case[1] + enc_num = encoding plain + enc = encodingname enc_num + esc_plain = escape plain + bufused = bytelength plain + len = length plain + hex_str = hex_chars( plain, enc ) + $P0 = new 'FixedPMCArray', 6 + $P0[0] = count + $P0[1] = enc + $P0[2] = esc_plain + $P0[3] = hex_str + $P0[4] = bufused + $P0[5] = len + comment = sprintf "encode%d %s:\"%s\" %s(size=%d,len=%d)", $P0 + + unless bigendian goto BE_SWAP + base64 = be_result[count] + BE_SWAP: + test_encode( plain, base64, comment ) + comment = 'decode' + comment_count = count + comment = concat comment, comment_count + test_decode( plain, base64, comment ) + inc count + goto enc_dec_loop + enc_dec_loop_end: + + test_iterator = iter decode_tests + dec_loop: + unless test_iterator goto dec_loop_end + test_case = shift test_iterator + base64 = test_case[0] + plain = test_case[1] + $I0 = encoding plain + enc = encodingname $I0 + esc_plain = escape plain + $P0 = new 'ResizablePMCArray' + push $P0, comment_count + push $P0, enc + push $P0, esc_plain + push $P0, $I0 + comment = sprintf "decode%d %s:\"%s\"(enc=%d)", $P0 + test_decode( plain, base64, comment ) + inc count + goto dec_loop + dec_loop_end: +.end + +.sub hex_chars + .param string chars + .param string enc + + .local string s + .local pmc bb + .local int len + bb = new 'ByteBuffer' + s = clone chars + bb = s + s = bb.'get_string'(enc) + len = length s + $P0 = new 'FixedPMCArray', 2 + $I1 = 0 + $S0 = '' +loop_1: + $I0 = ord s, $I1 + $P0[0] = $S0 + $P0[1] = $I0 + $S0 = sprintf '%s\x%x', $P0 + inc $I1 + if $I1 < len goto loop_1 + ## $S0 = chopn $S0, 2 + .return ($S0) +.end + +.sub test_encode + .param string plain + .param string base64 + .param string comment + + .local pmc enc_sub + enc_sub = get_global [ "MIME"; "Base64" ], 'encode_base64' + + .local pmc is + is = get_hll_global [ 'Test'; 'More' ], 'is' + + .local string result_encode + result_encode = enc_sub( plain ) + is( result_encode, base64, comment ) +.end + + +.sub test_decode + .param string plain + .param string base64 + .param string comment + + .include "iglobals.pasm" + .local pmc interp + interp = getinterp + .local pmc config + config = interp[.IGLOBALS_CONFIG_HASH] + .local int has_icu + has_icu = config['has_icu'] + .local int bigendian + bigendian = config['bigendian'] + + .local pmc dec_sub + dec_sub = get_global [ "MIME"; "Base64" ], 'decode_base64' + + .local pmc is + is = get_hll_global [ 'Test'; 'More' ], 'is' + + .local string decode, result_norm, plain_norm + .local string enc, enc1 + $I0 = encoding plain + enc = encodingname $I0 + + decode = dec_sub( base64, enc ) + + $I1 = encoding decode + enc1 = encodingname $I1 + $P0 = new 'ResizablePMCArray' + push $P0, comment + push $P0, enc1 + push $P0, enc + push $P0, $I0 + comment = sprintf "%s %s<-%s(enc=%d)", $P0 + if enc == 'binary' goto HEX_C + if has_icu goto HAS_ICU + is( decode, plain, comment ) + goto END + HAS_ICU: + result_norm = compose decode + plain_norm = compose plain + is( result_norm, plain_norm, comment ) + goto END + HEX_C: + $S0 = hex_chars(decode, enc1) + $S1 = hex_chars(plain, enc) + is( $S0, $S1, comment ) + END: +.end + +=head1 AUTHOR + +Reini Urban, Bernhard Schmalhofer and others. + +=cut + +# Local Variables: +# mode: pir +# fill-column: 100 +# End: +# vim: expandtab shiftwidth=4 ft=pir: diff --git a/t/native_pbc/annotations.pbc b/t/native_pbc/annotations.pbc index 6f5db6695a..c3d1bdbab1 100644 Binary files a/t/native_pbc/annotations.pbc and b/t/native_pbc/annotations.pbc differ diff --git a/t/native_pbc/integer.pbc b/t/native_pbc/integer.pbc index 3ccef08bc8..7ec20e7605 100644 Binary files a/t/native_pbc/integer.pbc and b/t/native_pbc/integer.pbc differ diff --git a/t/native_pbc/integer_4.pbc b/t/native_pbc/integer_4.pbc index 4f46b52ec2..6f51c19a76 100644 Binary files a/t/native_pbc/integer_4.pbc and b/t/native_pbc/integer_4.pbc differ diff --git a/t/native_pbc/number.pbc b/t/native_pbc/number.pbc index 0c8ec5634e..289548e7b8 100644 Binary files a/t/native_pbc/number.pbc and b/t/native_pbc/number.pbc differ diff --git a/t/native_pbc/number_4.pbc b/t/native_pbc/number_4.pbc index 169ddfc911..5789d5ab9e 100644 Binary files a/t/native_pbc/number_4.pbc and b/t/native_pbc/number_4.pbc differ diff --git a/t/native_pbc/number_5.pbc b/t/native_pbc/number_5.pbc index 5709a4844e..51ed335d3c 100644 Binary files a/t/native_pbc/number_5.pbc and b/t/native_pbc/number_5.pbc differ diff --git a/t/native_pbc/string.pbc b/t/native_pbc/string.pbc index b1dff67114..c9331a1ec9 100644 Binary files a/t/native_pbc/string.pbc and b/t/native_pbc/string.pbc differ diff --git a/t/native_pbc/string_4.pbc b/t/native_pbc/string_4.pbc index 6c1eda5fac..0d95ce9fdd 100644 Binary files a/t/native_pbc/string_4.pbc and b/t/native_pbc/string_4.pbc differ diff --git a/tools/release/release.json b/tools/release/release.json index 74b657ff32..a134058ccd 100644 --- a/tools/release/release.json +++ b/tools/release/release.json @@ -1,8 +1,8 @@ { - "release.version" : "4.8.0", - "release.name" : "Spix's Macaw", + "release.version" : "4.9.0", + "release.name" : "Proto-Hydra", "release.day" : "Tuesday", - "release.nextdate" : "16 October 2012", + "release.nextdate" : "12 November 2012", "web.root" : "http://parrot.org/", "web.source" : "download", @@ -11,10 +11,10 @@ "web.org_root" : "https://github.com/parrot", "bugday.day" : "Saturday", - "bugday.date" : "13 October 2012", + "bugday.date" : "10 November 2012", "wiki.root" : "https://github.com/parrot/parrot/wiki", "wiki.bugday" : "bug_day_2012_10_13", - "ftp.path" : "ftp://ftp.parrot.org/pub/parrot/releases/devel/4.8.0/" + "ftp.path" : "ftp://ftp.parrot.org/pub/parrot/releases/devel/4.9.0/" }