Skip to content

Commit

Permalink
Item13378: full internal unicode implementation, demonstrating how Ge…
Browse files Browse the repository at this point in the history 
…orge's TestCaseUtf8Errors is (almost) totally cured, and I think the remaining error is a testcase error. Most unit tests pass, but I'm having trouble with the logger tests, I don't understand the code.
  • Loading branch information
Comment committed May 7, 2015
1 parent 6e027c7 commit 17f94d3
Show file tree
Hide file tree
Showing 33 changed files with 291 additions and 1,229 deletions.
61 changes: 27 additions & 34 deletions CompareRevisionsAddOn/lib/Foswiki/Contrib/CompareRevisionsAddOn/Compare.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ package Foswiki::Contrib::CompareRevisionsAddOn::Compare;

use strict;
use warnings;
use Assert;

use Foswiki::UI ();
use Foswiki::Func ();
Expand Down Expand Up @@ -111,14 +112,16 @@ sub compare {
if defined $savedskin;
$query->param( 'skin', 'classic' );

# Get the HTML trees of the specified versions
# Get the HTML trees of the specified versions.
# Note that the trees are built using UNICODE text, and it is
# only at the end that the result is converted back to utf8 bytes.

my $tree2 = _getTree( $session, $webName, $topic, $rev2 );
if ( $tree2 =~ /^http:.*oops/ ) {
Foswiki::Func::redirectCgiQuery( $query, $tree2 );
}

# TablePlugin must reinitiatilise to reset all table counters (Item1911)
# TablePlugin must reinitialise to reset all table counters (Item1911)
if ( defined &Foswiki::Plugins::TablePlugin::initPlugin ) {
if ( defined &Foswiki::Plugins::TablePlugin::initialiseWhenRender ) {
Foswiki::Plugins::TablePlugin::initialiseWhenRender();
Expand Down Expand Up @@ -182,7 +185,6 @@ sub compare {
$tmpl_d = $tmpl_a unless $tmpl_d =~ /\S/;

# Start the output

my $output = '';

# Compare the trees
Expand Down Expand Up @@ -267,14 +269,6 @@ sub compare {

}

# Item12337: part of alternative fix for Item11755
my $charset = $Foswiki::cfg{Site}{CharSet} || 'iso-8859-1';
$output = Encode::encode( $charset, $output );

# Item12423: include rest of template after recoding
# (avoids double-encoding in header/footer)
$output = $tmpl_before . $output;

# Print remainder of document

my $revisions = "";
Expand Down Expand Up @@ -336,10 +330,11 @@ sub compare {

}

# Get the rendered version of a document as HTML tree
# CAUTION: the utf8 content of the topic is automatically decoded to unicode,
# and it's unicode that will appear in the tree.
sub _getTree {

# Purpose: Get the rendered version of a document as HTML tree

my ( $session, $webName, $topicName, $rev ) = @_;

# Read document
Expand All @@ -357,12 +352,6 @@ sub _getTree {
$text =~ s/^\s*//;
$text =~ s/\s*$//;

# Item12337: part of alternative fix for Item11755
my $charset = $Foswiki::cfg{Site}{CharSet} || 'iso-8859-1';
$text = Encode::decode( $charset, $text );

# Generate tree

my $tree = new HTML::TreeBuilder;
$tree->implicit_body_p_tag(1);
$tree->p_strict(1);
Expand All @@ -387,6 +376,7 @@ sub _getTree {
return $tree;
}

# All strings are UNICODE
sub _findSubChanges {

# Purpose: Finds and formats changes between two HTML::Elements.
Expand Down Expand Up @@ -418,8 +408,8 @@ sub _findSubChanges {
my @list2 = $e2->content_list;

if ( @list1 && @list2 ) { # Two non-empty lists
die "Huch!:" . $e1->tag . "!=" . $e2->tag
if $e1->tag ne $e2->tag;
ASSERT( $e1->tag eq $e2->tag, "Huch!:" . $e1->tag . "!=" . $e2->tag )
if DEBUG;
$text1 = $e1->starttag;
$text2 = $e2->starttag;
my @changes =
Expand Down Expand Up @@ -497,10 +487,10 @@ sub _addClass {
}
}

# Compare two text elements. Output as in _findSubChanges
# All strings are UNICODE
sub _compareText {

# Purpose: Compare two text elements. Output as in _findSubChanges

my ( $text1, $text2 ) = @_;

my @list1 = split( ' ', $text1 );
Expand Down Expand Up @@ -553,34 +543,37 @@ sub _compareText {
return ( $ctext1 || '', $ctext2 || '' );
}

# Format text with a class.
# All strings are UNICODE
sub _getTextWithClass {

# Purpose: Format text with a class

my ( $element, $class ) = @_;
my $result;

if ( ref($element) eq $HTMLElement ) {
_addClass( $element, $class ) if $class;

# Item11755: prevent entity mangling
# SMELL: Alternative to $tree->no_expand_entities(1); See Item12337 and Item12407
# This fix is not 100%, but helps in some cases. The real solution is to install
# latest HTML::Tree.
# Item11755: prevent entity mangling
# SMELL: Alternative to $tree->no_expand_entities(1);
# See Item12337 and Item12407
# This fix is not 100%, but helps in some cases.
# The real solution is to install the latest HTML::Tree!
my $entities = ($v40plus) ? '' : '<>&';
return $element->as_HTML( $entities, undef, {} );
$result = $element->as_HTML( $entities, undef, {} );
}
elsif ($class) {
return '<span class="' . $class . '">' . $element . '</span>';
$result = '<span class="' . $class . '">' . $element . '</span>';
}
else {
return $element;
$result = $element;
}
return $result;
}

# Get test from an action.
# All strings are UNICODE
sub _getTextFromAction {

# Purpose:

my $action = shift;

my ( $text1, $text2 );
Expand Down
2 changes: 1 addition & 1 deletion EditRowPlugin/lib/Foswiki/Plugins/EditRowPlugin/Get.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ sub process {
$response->header(
-status => $status,
-type => 'application/json',
-charset => $Foswiki::cfg{Site}{CharSet},
-charset => 'utf-8',

# HTTP/1.0
-Pragma => 'no-cache',
Expand Down
23 changes: 4 additions & 19 deletions JsonRpcContrib/lib/Foswiki/Contrib/JsonRpcContrib/Request.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ use strict;
use warnings;

use JSON ();
use Encode ();
use Foswiki::Contrib::JsonRpcContrib::Error ();
use Error qw( :try );
use Foswiki::Func ();
Expand All @@ -37,11 +36,9 @@ sub new {

# get json-rpc request object
my $data = $request->param('POSTDATA');
if ($data) {
$data = fromUtf8($data);
}
else {
# minimal stup
unless ($data) {

# minimal setup
$data = '{"jsonrpc":"2.0"}';
}
writeDebug("data=$data") if TRACE;
Expand All @@ -60,7 +57,7 @@ sub new {
# override json-rpc params using url params
foreach my $key ( $request->multi_param() ) {
next if $key =~ /^(POSTDATA|method|id|jsonrpc)$/; # these are different
my @vals = map( fromUtf8($_), $request->multi_param($key) );
my @vals = $request->multi_param($key);
if ( scalar(@vals) == 1 ) {
$this->param( $key => $vals[0] )
; # set json-rpc params using url params
Expand Down Expand Up @@ -194,16 +191,4 @@ sub writeDebug {
Foswiki::Func::writeDebug("- JsonRpcContrib::Request - $_[0]");
}

###############################################################################
sub fromUtf8 {
my $string = shift;

return $string unless $string;
return $string
if $Foswiki::Plugins::VERSION >
2.1; # not required on "newer" foswikis, is it?

return Encode::decode_utf8($string);
}

1;
60 changes: 15 additions & 45 deletions MailerContrib/test/unit/MailerContrib/MailerContribSuite.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package MailerContribSuite;
use strict;
use warnings;
use locale;
use utf8;

use FoswikiFnTestCase();
our @ISA = qw( FoswikiFnTestCase );
Expand Down Expand Up @@ -43,8 +42,7 @@ my %finalText = (
TestTopic21 => "smoke me a kipper, I'll be back for breakfast",
TestTopicDenied => " * Set ALLOWTOPICVIEW = TestUser1\n",

# High-bit chars - assumes {Site}{CharSet} is set for a high-bit
# encoding.
# High-bit chars
'RequêtesNon' => "makê it so, number onê",
'RequêtesOui' => "you're such a smêêêêêê heeee",

Expand Down Expand Up @@ -222,55 +220,27 @@ sub set_up {
email => "email11\@example.com",
entry => "email11\@example.com: FakeTestTopic1 FakeTestTopic11",
topicsout => ""
}
);
},

if ( !$high_bit_disabled ) {
if ( !$Foswiki::cfg{Site}{CharSet}
|| $Foswiki::cfg{Site}{CharSet} =~ /^iso-?8859/
|| $Foswiki::cfg{Site}{CharSet} =~ /^utf-8/ )
# High-bit chars
{
name => "High bit",
email => "test1\@example.com",
entry => "TestUser1 : Requêtes*",
topicsout => "RequêtesNon RequêtesOui",
},

# High-bit chars
push(
@specs, # Francais
{
name => "High bit",
email => "test1\@example.com",
entry => "TestUser1 : Requêtes*",
topicsout => "RequêtesNon RequêtesOui",
},
);
}
else {
print STDERR
"WARNING: No high-bit tests with $Foswiki::cfg{Site}{CharSet}\n";
$high_bit_disabled = 1;
}
if ( $Foswiki::cfg{Site}{CharSet} =~ /^utf-8/ ) {

# Multi-byte chars
push(
@specs, # Mandarin
{
name => 'Multi byte',
email => "test2\@example.com",
entry => 'TestUser1 : 官話',
topicsout => '官話'
},
);
}
else {
print STDERR
"WARNING: No multi-byte tests with $Foswiki::cfg{Site}{CharSet}\n";
# Multi-byte chars
{
name => 'Multi byte',
email => "test2\@example.com",
entry => 'TestUser1 : 官話',
topicsout => '官話'
}
}
);

my $s = "";
foreach my $spec (@specs) {
while ( my ( $k, $v ) = each %$spec ) {
$spec->{$k} = Encode::encode( $Foswiki::cfg{Site}{CharSet}, $v );
}
$s .= " * $spec->{entry}\n";
}
foreach my $web ( $this->{test_web}, $testWeb2 ) {
Expand Down
17 changes: 3 additions & 14 deletions NatEditPlugin/lib/Foswiki/Plugins/NatEditPlugin/RestSave.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ sub handle {

my $request = $session->{request};

# transform to site charset
# Hack the parameters
foreach my $key ( $request->multi_param() ) {
my @val = $request->multi_param($key);

Expand All @@ -38,10 +38,10 @@ sub handle {
}

if ( ref $val[0] eq 'ARRAY' ) {
$request->param( $key, [ map( toSiteCharSet($_), @{ $val[0] } ) ] );
$request->param( $key, $val[0] );
}
else {
$request->param( $key, [ map( toSiteCharSet($_), @val ) ] );
$request->param( $key, [@val] );
}
}

Expand Down Expand Up @@ -114,15 +114,4 @@ sub stringifyError {
return $s;
}

sub toSiteCharSet {
my $string = shift;

my $charSet = Encode::resolve_alias( $Foswiki::cfg{Site}{CharSet} );
return $string if ( !$charSet || $charSet =~ m/^utf-?8$/i );

# converts to {Site}{CharSet}, generating HTML NCR's when needed
my $octets = Encode::decode( 'utf-8', $string );
return Encode::encode( $charSet, $octets, &Encode::FB_HTMLCREF() );
}

1;
Loading

0 comments on commit 17f94d3

Please sign in to comment.