Skip to content

Commit

Permalink
Item13315: Improve characterset checkers
Browse files Browse the repository at this point in the history
  • Loading branch information
gac410 committed May 25, 2015
1 parent 322871c commit 545165b
Show file tree
Hide file tree
Showing 5 changed files with 159 additions and 39 deletions.
4 changes: 3 additions & 1 deletion core/lib/Foswiki.spec
Expand Up @@ -645,7 +645,7 @@ $Foswiki::cfg{MinPasswordLength} = 7;
# email addresses from an existing file.
$Foswiki::cfg{Htpasswd}{FileName} = '$Foswiki::cfg{DataDir}/.htpasswd';

# **STRING LABEL="Password File Character Encodingname" DISPLAY_IF="/htpasswd/i.test({PasswordManager})" CHECK="iff:'{PasswordManager}=~/htpasswd/i'"**
# **STRING LABEL="Password File Character Encodingname" DISPLAY_IF="/htpasswd/i.test({PasswordManager})" CHECK="undefok iff:'{PasswordManager}=~/htpasswd/i'"**
# Character encoding used in the password file. This will default to utf-8, which allows any unicode
# character to be used in usernames, passwords and email addresses. The only time you should change it
# is if you have an existing password file that uses a different encoding (and even then only if there
Expand Down Expand Up @@ -1339,6 +1339,8 @@ $Foswiki::cfg{PluralToSingular} = $TRUE;
# This option takes the name of the encoding e.g. 'iso-8859-1'.
# Note: you are *STRONGLY* recommended to convert the entire store to
# UTF-8 in-place, as setting this option will incur a performance penatly.
# See =tools/bulk_copy.pl= for details on converting your store.
# Leave this undefined to use the default of utf-8 encoding.
# $Foswiki::cfg{Store}{Encoding} = undef;

# **PERL LABEL="Implementation Classes" EXPERT**
Expand Down
109 changes: 109 additions & 0 deletions core/lib/Foswiki/Configure/Checkers/Htpasswd/CharacterEncoding.pm
@@ -0,0 +1,109 @@
# See bottom of file for license and copyright information
package Foswiki::Configure::Checkers::Htpasswd::CharacterEncoding;

use strict;
use warnings;

use Foswiki::Configure::Checker ();
our @ISA = ('Foswiki::Configure::Checker');

sub check_current_value {
my ( $this, $reporter ) = @_;

if ( $Foswiki::cfg{Htpasswd}{CharacterEncoding} ) {

# Test if this is actually an available encoding:
eval {
require Encode;
Encode::encode( $Foswiki::cfg{Htpasswd}{CharacterEncoding},
'test', 0 );
};
if ($@) {
$reporter->ERROR("Unknown store character set requested.");
print STDERR "encode failed $@ \n";
return;
}

if ( $Foswiki::cfg{Htpasswd}{CharacterEncoding} =~
m/^(?:iso-?2022-?|hz-?|gb2312|gbk|gb18030|.*big5|.*shift_?jis|ms.kanji|johab|uhc)/i
)
{

$reporter->ERROR(
<<HERE
Cannot use this multi-byte encoding ('$Foswiki::cfg{Htpasswd}{CharacterEncoding}')
as {Htpasswd}{CharacterEncoding}. Please set a different character encoding setting.
HERE
);
}

# Extract the character set from locale for consistency check
my $charset;
$Foswiki::cfg{Site}{Locale} =~ m/\.([a-z0-9_-]+)$/i;
$charset = $1 || ''; # no guess?
$charset =~ s/^utf8$/utf-8/i;
$charset =~ s/^eucjp$/euc-jp/i;
$charset = lc($charset);

if ( $charset
&& ( lc( $Foswiki::cfg{Htpasswd}{CharacterEncoding} ) ne $charset )
)
{
$reporter->ERROR(
<<HERE
The Character set determined by the configured Locale, and this character set,
are inconsistent. Recommended setting: =$charset=
HERE
);
}
}

if ( $Foswiki::cfg{isBOOTSTRAPPING} ) {
if (
!$Foswiki::cfg{Htpasswd}{CharacterEncoding}
|| (
lc(
$Foswiki::cfg{Htpasswd}{CharacterEncoding} ne 'iso-8859-1'
)
)
)
{
$reporter->WARN( <<HERE );
The BOOTSTRAP process has left the =.htpasswd= file character set encoding as utf-8.
In most systems this should work fine. However if you are migrating existing data
and have user information that doesn't map to utf-8, you should set this field to
the encoding used in your existing system. In most cases this will be =iso-8859-1=.
Only set this if errors are encountered when processing your existing password file.
HERE
}
}

return '';
}

1;
__END__
Foswiki - The Free and Open Source Wiki, http://foswiki.org/
Copyright (C) 2008-2015 Foswiki Contributors. Foswiki Contributors
are listed in the AUTHORS file in the root of this distribution.
NOTE: Please extend that file, not this notice.
Additional copyrights apply to some or all of the code in this
file as follows:
Copyright (C) 2000-2006 TWiki Contributors. All Rights Reserved.
TWiki Contributors are listed in the AUTHORS file in the root
of this distribution. NOTE: Please extend that file, not this notice.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version. For
more details read LICENSE in the root of this distribution.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
As per the GPL, removal of this notice is prohibited.
82 changes: 44 additions & 38 deletions core/lib/Foswiki/Configure/Checkers/Store/Encoding.pm
Expand Up @@ -10,57 +10,63 @@ our @ISA = ('Foswiki::Configure::Checker');
sub check_current_value {
my ( $this, $reporter ) = @_;

# Test if this is actually an available encoding:
eval {
require Encode;
Encode::encode( $Foswiki::cfg{Store}{Encoding}, 'test', 0 );
};
if ($@) {
$reporter->ERROR("Unknown store character set requested.");
print STDERR "encode failed $@ \n";
return;
}

if ( $Foswiki::cfg{Store}{Encoding} =~
if ( $Foswiki::cfg{Store}{Encoding} ) {

# Test if this is actually an available encoding:
eval {
require Encode;
Encode::encode( $Foswiki::cfg{Store}{Encoding}, 'test', 0 );
};
if ($@) {
$reporter->ERROR("Unknown store character set requested.");
print STDERR "encode failed $@ \n";
return;
}

if ( $Foswiki::cfg{Store}{Encoding} =~
m/^(?:iso-?2022-?|hz-?|gb2312|gbk|gb18030|.*big5|.*shift_?jis|ms.kanji|johab|uhc)/i
)
{
)
{

$reporter->ERROR(
<<HERE
$reporter->ERROR(
<<HERE
Cannot use this multi-byte encoding ('$Foswiki::cfg{Store}{Encoding}')
as {Store}{Encoding}. Please set a different character encoding setting.
HERE
);
}

# Extract the character set from locale for consistency check
my $charset;
$Foswiki::cfg{Site}{Locale} =~ m/\.([a-z0-9_-]+)$/i;
$charset = $1 || ''; # no guess?
$charset =~ s/^utf8$/utf-8/i;
$charset =~ s/^eucjp$/euc-jp/i;
$charset = lc($charset);

if ( $charset && ( lc( $Foswiki::cfg{Store}{Encoding} ) ne $charset ) ) {
$reporter->ERROR(
<<HERE
);
}

# Extract the character set from locale for consistency check
my $charset;
$Foswiki::cfg{Site}{Locale} =~ m/\.([a-z0-9_-]+)$/i;
$charset = $1 || ''; # no guess?
$charset =~ s/^utf8$/utf-8/i;
$charset =~ s/^eucjp$/euc-jp/i;
$charset = lc($charset);

if ( $charset && ( lc( $Foswiki::cfg{Store}{Encoding} ) ne $charset ) )
{
$reporter->ERROR(
<<HERE
The Character set determined by the configured Locale, and this character set,
are inconsistent. Recommended setting: =$charset=
HERE
);
);
}
}

if ( $Foswiki::cfg{isBOOTSTRAPPING}
&& ( lc( $Foswiki::cfg{Store}{Encoding} ne 'iso-8859-1' ) ) )
{
$reporter->WARN( <<HERE );
The BOOTSTRAP process has guessed the store is using utf-8.
if ( $Foswiki::cfg{isBOOTSTRAPPING} ) {
if ( !$Foswiki::cfg{Store}{Encoding}
|| ( lc( $Foswiki::cfg{Store}{Encoding} ne 'iso-8859-1' ) ) )
{
$reporter->WARN( <<HERE );
The BOOTSTRAP process has set the default character set encoding to utf-8.
This is different from the Foswiki 1.1 default of =iso-8859-1=.
If you intend to migrate data from prior releases of Foswiki or TWiki,
you should either match the previously used {Site}{CharSet}
or migrate data using <a href="http://foswiki.org/Extensions/CharsetConverterContrib" target="_blank">CharsetConverterContrib</a>
or migrate data using =tools/bulk_copy.pl=.
HERE
}
}

return '';
Expand All @@ -70,7 +76,7 @@ HERE
__END__
Foswiki - The Free and Open Source Wiki, http://foswiki.org/
Copyright (C) 2008-2010 Foswiki Contributors. Foswiki Contributors
Copyright (C) 2008-2015 Foswiki Contributors. Foswiki Contributors
are listed in the AUTHORS file in the root of this distribution.
NOTE: Please extend that file, not this notice.
Expand Down
1 change: 1 addition & 0 deletions core/lib/Foswiki/Contrib/core/MANIFEST
Expand Up @@ -439,6 +439,7 @@ lib/Foswiki/Configure/Checkers/Email/SSLCrlFile.pm 0444
lib/Foswiki/Configure/Checkers/ExtensionsRepositories.pm 0444
lib/Foswiki/Configure/Checkers/FormTypes.pm 0444
lib/Foswiki/Configure/Checkers/Htpasswd/AutoDetect.pm 0444
lib/Foswiki/Configure/Checkers/Htpasswd/CharacterEncoding.pm 0444
lib/Foswiki/Configure/Checkers/Htpasswd/Encoding.pm 0444
lib/Foswiki/Configure/Checkers/Htpasswd/FileName.pm 0444
lib/Foswiki/Configure/Checkers/Htpasswd/LockFileName.pm 0444
Expand Down
2 changes: 2 additions & 0 deletions core/tools/configure
Expand Up @@ -309,6 +309,8 @@ unless ( $Foswiki::cfg{isVALID} ) {
_prompt( $root, '{ToolsDir}', $Foswiki::cfg{ToolsDir} );
_prompt( $root, '{Store}{Implementation}',
$Foswiki::cfg{Store}{Implementation} );
_prompt( $root, '{Store}{Encoding}', $Foswiki::cfg{Store}{Encoding},
undef, 1 );
_prompt( $root, '{Store}{SearchAlgorithm}',
$Foswiki::cfg{Store}{SearchAlgorithm} );
}
Expand Down

0 comments on commit 545165b

Please sign in to comment.