Item13378: standardised param decode in AJAX handlers. Working now wi…

…th UTF-8 web and topic names.
foswiki · May 19, 2015 · 047a158 · 047a158
1 parent 9da58dc
commit 047a158
Show file tree

Hide file tree

Showing 20 changed files with 163 additions and 58 deletions.
diff --git a/JQueryPlugin/lib/Foswiki/Plugins/JQueryPlugin/Plugins.pm b/JQueryPlugin/lib/Foswiki/Plugins/JQueryPlugin/Plugins.pm
@@ -282,7 +282,8 @@ sub load {
         eval "use $pluginDesc->{class};";
 
         if ($@) {
-            print STDERR "ERROR: can't load jQuery plugin $pluginName: $@\n";
+            Foswiki::Func::writeDebug(
+                "ERROR: can't load jQuery plugin $pluginName: $@");
             $pluginDesc->{instance} = 0;
         }
         else {

diff --git a/JsonRpcContrib/lib/Foswiki/Contrib/JsonRpcContrib/Request.pm b/JsonRpcContrib/lib/Foswiki/Contrib/JsonRpcContrib/Request.pm
@@ -20,6 +20,7 @@ use strict;
 use warnings;
 
 use JSON                                    ();
+use Encode                                  ();
 use Foswiki::Contrib::JsonRpcContrib::Error ();
 use Error qw( :try );
 use Foswiki::Func    ();
@@ -36,7 +37,10 @@ sub new {
 
     # get json-rpc request object
     my $data = $request->param('POSTDATA');
-    unless ($data) {
+    if ($data) {
+        $data = toSiteCharSet($data);
+    }
+    else {
 
         # minimal setup
         $data = '{"jsonrpc":"2.0"}';
@@ -57,7 +61,7 @@ sub new {
     # override json-rpc params using url params
     foreach my $key ( $request->multi_param() ) {
         next if $key =~ /^(POSTDATA|method|id|jsonrpc)$/;  # these are different
-        my @vals = $request->multi_param($key);
+        my @vals = map( toSiteCharSet($_), $request->multi_param($key) );
         if ( scalar(@vals) == 1 ) {
             $this->param( $key => $vals[0] )
               ;    # set json-rpc params using url params
@@ -73,7 +77,7 @@ sub new {
     $this->id($id) if defined $id;
 
     # copy method to json-rpc request
-    $method = $request->param("method") if defined $request->param("method");
+    $method = $request->param('method') if defined $request->param("method");
     $this->method($method) if defined $method;
 
     # check that this is a http POST
@@ -191,4 +195,25 @@ sub writeDebug {
     Foswiki::Func::writeDebug("- JsonRpcContrib::Request - $_[0]");
 }
 
+###############################################################################
+sub toSiteCharSet {
+    my $string = shift;
+
+    return $string unless $string;
+
+    # Convert to unicode if the core supports it
+    return Encode::decode_utf8($string)
+      if $Foswiki::USE_UNICODE;
+
+    return $string
+      if ( $Foswiki::cfg{Site}{CharSet} =~ /^utf-?8/i );
+
+    # If the site charset is not utf-8, need to convert it
+    return Encode::encode(
+        $Foswiki::cfg{Site}{CharSet},
+        Encode::decode_utf8($string),
+        Encode::FB_PERLQQ
+    );
+}
+
 1;
diff --git a/JsonRpcContrib/lib/Foswiki/Contrib/JsonRpcContrib/Server.pm b/JsonRpcContrib/lib/Foswiki/Contrib/JsonRpcContrib/Server.pm
@@ -92,7 +92,8 @@ sub dispatch {
 
     # get topic parameter and set the location overriding any
     #  other value derived from the namespace param
-    my $topic = $request->param("topic") || $Foswiki::cfg{HomeTopicName};
+    my $topic = $request->unicode_param('topic')
+      || $Foswiki::cfg{HomeTopicName};
     ( $session->{webName}, $session->{topicName} ) =
       Foswiki::Func::normalizeWebTopicName( $Foswiki::cfg{UsersWebName},
         $topic );
@@ -113,10 +114,10 @@ sub dispatch {
     }
 
     # if there's login info, try and apply it
-    my $userName = $request->param("username");
+    my $userName = $request->param('username');
     if ($userName) {
         writeDebug("checking password for $userName") if TRACE;
-        my $pass = $request->param("password") || '';
+        my $pass = $request->param('password') || '';
         unless ( $session->{users}->checkPassword( $userName, $pass ) ) {
             Foswiki::Contrib::JsonRpcContrib::Response->print(
                 $session,
@@ -183,7 +184,7 @@ sub dispatch {
     };
 
     # finally
-    my $redirectto = $request->param("redirectto");
+    my $redirectto = $request->unicode_param('redirectto');
     if ( $code == 0 && defined $redirectto ) {
         my $url;
         if ( $redirectto =~ /^https?:/ ) {

diff --git a/NatEditPlugin/lib/Foswiki/Plugins/NatEditPlugin/RestSave.pm b/NatEditPlugin/lib/Foswiki/Plugins/NatEditPlugin/RestSave.pm
@@ -25,7 +25,6 @@ sub handle {
 
     my $request = $session->{request};
 
-    # Hack the parameters
     foreach my $key ( $request->multi_param() ) {
         my @val = $request->multi_param($key);
 
@@ -38,10 +37,10 @@ sub handle {
         }
 
         if ( ref $val[0] eq 'ARRAY' ) {
-            $request->param( $key, $val[0] );
+            $request->param( $key, [ map( toSiteCharSet($_), @{ $val[0] } ) ] );
         }
         else {
-            $request->param( $key, [@val] );
+            $request->param( $key, [ map( toSiteCharSet($_), @val ) ] );
         }
     }
 
@@ -114,4 +113,25 @@ sub stringifyError {
     return $s;
 }
 
+sub toSiteCharSet {
+    my $string = shift;
+
+    return $string unless $string;
+
+    # Convert to unicode if the core supports it
+    return Encode::decode_utf8($string)
+      if $Foswiki::USE_UNICODE;
+
+    return $string
+      if ( $Foswiki::cfg{Site}{CharSet} =~ /^utf-?8/i );
+
+    # If the site charset is not utf-8, need to convert it
+    require Encode;
+    return Encode::encode(
+        $Foswiki::cfg{Site}{CharSet},
+        Encode::decode_utf8($string),
+        Encode::FB_PERLQQ
+    );
+}
+
 1;
diff --git a/TinyMCEPlugin/lib/Foswiki/Plugins/TinyMCEPlugin.pm b/TinyMCEPlugin/lib/Foswiki/Plugins/TinyMCEPlugin.pm
@@ -69,12 +69,14 @@ sub beforeEditHandler {
 
     my $mess = _notAvailable();
     if ($mess) {
-        if ( ( $mess !~ /^Disabled/ || DEBUG )
+        my $disabled = ( $mess !~ /^Disabled/ );
+        $mess = 'WYSIWYG could not be started: ' . $mess;
+        if ( ( $disabled || DEBUG )
             && defined &Foswiki::Func::setPreferencesValue )
         {
-            Foswiki::Func::setPreferencesValue( 'EDITOR_MESSAGE',
-                'WYSIWYG could not be started: ' . $mess );
+            Foswiki::Func::setPreferencesValue( 'EDITOR_MESSAGE', $mess );
         }
+        Foswiki::Func::writeDebug($mess) if DEBUG;
         return;
     }
     if ( defined &Foswiki::Func::setPreferencesValue ) {
@@ -108,11 +110,12 @@ sub beforeEditHandler {
 
     $mess = Foswiki::Plugins::WysiwygPlugin::notWysiwygEditable($text);
     if ($mess) {
+        $mess = 'WYSIWYG could not be started: ' . $mess;
         if ( defined &Foswiki::Func::setPreferencesValue ) {
-            Foswiki::Func::setPreferencesValue( 'EDITOR_MESSAGE',
-                'WYSIWYG could not be started: ' . $mess );
-            Foswiki::Func::setPreferencesValue( 'EDITOR_HELP', undef );
+            Foswiki::Func::setPreferencesValue( 'EDITOR_MESSAGE', $mess );
+            Foswiki::Func::setPreferencesValue( 'EDITOR_HELP',    undef );
         }
+        Foswiki::Func::writeDebug($mess) if DEBUG;
         return;
     }
 

diff --git a/WysiwygPlugin/lib/Foswiki/Plugins/WysiwygPlugin.pm b/WysiwygPlugin/lib/Foswiki/Plugins/WysiwygPlugin.pm
@@ -42,7 +42,7 @@ our $recursionBlock;
 our %FoswikiCompatibility;
 
 # Set to 1 for reasons for rejection
-sub WHY { 0 }
+use constant WHY => 1;
 
 #simple Browser detection.
 our %defaultINIT_BROWSER = (
@@ -302,8 +302,8 @@ sub wysiwygEditingNotPossibleForThisContent {
         );
     };
     if ($@) {
-        print STDERR
-          "WYSIWYG_DEBUG: TML2HTML conversion threw an exception: $@\n"
+        Foswiki::Func::writeDebug(
+            "WYSIWYG_DEBUG: TML2HTML conversion threw an exception: $@")
           if (WHY);
         return "TML2HTML conversion fails";
     }

diff --git a/WysiwygPlugin/lib/Foswiki/Plugins/WysiwygPlugin/Handlers.pm b/WysiwygPlugin/lib/Foswiki/Plugins/WysiwygPlugin/Handlers.pm
@@ -23,6 +23,26 @@ our %xmltagPlugin;
 our $SECRET_ID =
 'WYSIWYG content - do not remove this comment, and never use this identical text in your topics';
 
+sub toSiteCharSet {
+    my $string = shift;
+
+    return $string unless $string;
+
+    # Convert to unicode if the core supports it
+    return Encode::decode_utf8($string)
+      if $Foswiki::USE_UNICODE;
+
+    return $string
+      if ( $Foswiki::cfg{Site}{CharSet} =~ /^utf-?8/i );
+
+    # If the site charset is not utf-8, need to convert it
+    return Encode::encode(
+        $Foswiki::cfg{Site}{CharSet},
+        Encode::decode_utf8($string),
+        Encode::FB_PERLQQ
+    );
+}
+
 sub _SECRET_ID {
     $SECRET_ID;
 }
@@ -35,7 +55,8 @@ sub _OWEBTAG {
     return $web unless $query;
 
     if ( defined( $query->param('templatetopic') ) ) {
-        my @split = split( /\./, $query->param('templatetopic') );
+        my @split =
+          split( /\./, toSiteCharSet( $query->param('templatetopic') ) );
 
         if ( $#split == 0 ) {
             return $web;
@@ -56,7 +77,8 @@ sub _OTOPICTAG {
     return $topic unless $query;
 
     if ( defined( $query->param('templatetopic') ) ) {
-        my @split = split( /\./, $query->param('templatetopic') );
+        my @split =
+          split( /\./, toSiteCharSet( $query->param('templatetopic') ) );
 
         return $split[$#split];
     }
@@ -80,7 +102,7 @@ sub beforeEditHandler {
             # redirect
             my $query = Foswiki::Func::getCgiQuery();
             foreach my $p (qw( skin cover )) {
-                my $arg = $query->param($p);
+                my $arg = toSiteCharSet( $query->param($p) );
                 if ( $arg && $arg =~ s/\b$skin\b// ) {
                     if ( $arg =~ /^[\s,]*$/ ) {
                         $query->delete($p);
@@ -203,9 +225,12 @@ sub beforeCommonTagsHandler {
     my $web   = $_[2];
     my ( $meta, $text );
     my $altText = $query->param('templatetopic');
-    if ( $altText && Foswiki::Func::topicExists( $web, $altText ) ) {
-        ( $web, $topic ) =
-          Foswiki::Func::normalizeWebTopicName( $web, $altText );
+    if ($altText) {
+        $altText = toSiteCharSet($altText);
+        if ( Foswiki::Func::topicExists( $web, $altText ) ) {
+            ( $web, $topic ) =
+              Foswiki::Func::normalizeWebTopicName( $web, $altText );
+        }
     }
 
     $_[0] = _WYSIWYG_TEXT( $Foswiki::Plugins::SESSION, {}, $topic, $web );
@@ -644,13 +669,15 @@ sub REST_TML2HTML {
 
     my $tml = Foswiki::Func::getCgiQuery()->param('text');
 
+    return '' unless $tml;
+
     # if the secret ID is present, don't convert again. We are probably
     # going 'back' to this page (doesn't work on IE :-( )
     if ( $tml =~ /<!--$SECRET_ID-->/ ) {
         return $tml;
     }
 
-    my $html = TranslateTML2HTML($tml);
+    my $html = TranslateTML2HTML( toSiteCharSet($tml) );
 
     # Add the secret id to trigger reconversion. Doesn't work if the
     # editor eats HTML comments, so the editor may need to put it back
@@ -668,6 +695,10 @@ sub REST_HTML2TML {
 
     my $html = Foswiki::Func::getCgiQuery()->param('text');
 
+    return '' unless $html;
+
+    $html = toSiteCharSet($html);
+
     $html =~ s/<!--$SECRET_ID-->//go;
     unless ($html2tml) {
         require Foswiki::Plugins::WysiwygPlugin::HTML2TML;

diff --git a/WysiwygPlugin/lib/Foswiki/Plugins/WysiwygPlugin/TML2HTML.pm b/WysiwygPlugin/lib/Foswiki/Plugins/WysiwygPlugin/TML2HTML.pm
@@ -970,7 +970,7 @@ m/$startww(($Foswiki::regex{webNameRegex}\.)?$Foswiki::regex{wikiWordRegex}($Fos
         $class = " class='$class'";
     }
 
-    $url = _percentHighBit($url);
+    $url = Foswiki::urlEncode($url);
     return $this->_liftOutGeneral(
         "<a$class$dataWikiword href=\"$url\">$text<\/a>",
         { tag => 'NONE', protect => 0, tmltag => 0 }

diff --git a/core/lib/Foswiki.pm b/core/lib/Foswiki.pm
@@ -72,8 +72,9 @@ our %macros;
 our %contextFreeSyntax;
 our $VERSION;
 our $RELEASE;
-our $TRUE  = 1;
-our $FALSE = 0;
+our $UNICODE = 1;  # flag that extensions can use to test if the core is unicode
+our $TRUE    = 1;
+our $FALSE   = 0;
 our $engine;
 our $TranslationToken = "\0";    # Do not deprecate - used in many plugins
 our $system_message;             # Important broadcast message from the system
@@ -1002,7 +1003,7 @@ Returns undef if the target is not valid, and the target URL otherwise.
 sub redirectto {
     my ( $this, $url ) = @_;
 
-    my $redirecturl = $this->{request}->param('redirectto');
+    my $redirecturl = $this->{request}->unicode_param('redirectto');
     $redirecturl = $url unless $redirecturl;
 
     return unless $redirecturl;
@@ -1657,7 +1658,7 @@ sub _parsePath {
 
     foreach (@parts) {
 
-        # Lax check on name to elimnate evil characters.
+        # Lax check on name to eliminate evil characters.
         my $p = Foswiki::Sandbox::untaint( $_,
             \&Foswiki::Sandbox::validateTopicName );
         unless ($p) {
@@ -2038,11 +2039,12 @@ sub new {
     # Set the default for web
     # Development.AddWebParamToAllCgiScripts: enables
     # bin/script?topic=WebPreferences;defaultweb=Sandbox
-    my $defaultweb = $query->param('defaultweb') || $Foswiki::cfg{UsersWebName};
+    my $defaultweb =
+      $query->unicode_param('defaultweb') || $Foswiki::cfg{UsersWebName};
 
-    my $webtopic      = $query->path_info() || '';
+    my $webtopic      = Encode::decode_utf8( $query->path_info() || '' );
     my $topicOverride = '';
-    my $topic         = $query->param('topic');
+    my $topic         = $query->unicode_param('topic');
     if ( defined $topic ) {
         if ( $topic =~ m/[\/.]+/ ) {
             $webtopic = $topic;
@@ -2818,6 +2820,8 @@ this method.
 sub urlEncode {
     my $text = shift;
 
+    # URLs work quite happily with %-encoded utf-8 characters
+    $text = Encode::encode_utf8($text);
     $text =~ s{([^0-9a-zA-Z-_.:~!*/])}{sprintf('%%%02x',ord($1))}ge;
 
     return $text;

diff --git a/core/lib/Foswiki/If/OP_dollar.pm b/core/lib/Foswiki/If/OP_dollar.pm
@@ -40,7 +40,7 @@ sub evaluate {
       unless $session;
     my $text = $a->_evaluate(@_) || '';
     if ( $text && defined( $session->{request}->param($text) ) ) {
-        return scalar( $session->{request}->param($text) );
+        return $session->{request}->unicode_param($text);
     }
 
     $text = "%$text%";

diff --git a/core/lib/Foswiki/LoginManager.pm b/core/lib/Foswiki/LoginManager.pm
@@ -632,8 +632,8 @@ sub redirectToLoggedOutUrl {
     );
 
     #TODO: consider if we should risk passing on the urlparams on logout
-    my $path_info = $session->{request}->path_info();
-    if ( my $topic = $session->{request}->param('topic') )
+    my $path_info = Encode::decode_utf8( $session->{request}->path_info() );
+    if ( my $topic = $session->{request}->unicode_param('topic') )
     {    #we should at least respect the ?topic= request
         my $topicRequest = Foswiki::Sandbox::untaintUnchecked($topic);
         ( my $web, $topic ) =

diff --git a/core/lib/Foswiki/Macros/TOC.pm b/core/lib/Foswiki/Macros/TOC.pm
@@ -117,7 +117,7 @@ sub TOC {
         next if ( $name eq 'topic' );
         next if ( $name eq 'text' );
         next if ( $name eq 'refresh' );
-        push( @qparams, $name => scalar( $query->param($name) ) );
+        push( @qparams, $name => scalar( $query->unicode_param($name) ) );
     }
 
     # Extract anchor targets. This has to generate *identical* anchor