Skip to content

Commit

Permalink
Item13405: Better NFD detection and other improvements
Browse files Browse the repository at this point in the history 
JozefMojzis rewrote the NFD detection code and tested it on an OS X
system.

Also:
 - Added a "n" check normalization option to the file path checker
 - Improved the reporting from the file checks.
 - Resolved an issue where some messages were not reported.
 - Comment out the Foswiki.spec definition for NFCNormalize - it's
   bootstrapped, so should not be defined.
  • Loading branch information
@gac410
gac410 committed Jan 2, 2016
1 parent ac918e0 commit 2fdc6c1
Show file tree
Hide file tree
Showing 5 changed files with 140 additions and 89 deletions.
12 changes: 7 additions & 5 deletions core/lib/Foswiki.spec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,13 +136,13 @@ $Foswiki::cfg{PermittedRedirectHostUrls} = '';
# This is the file system path used to access the Foswiki bin directory.
# $Foswiki::cfg{ScriptDir} = '/home/httpd/foswiki/bin';

# **PATH LABEL="Pub Directory" FEEDBACK="icon='ui-icon-check';label='Validate Permissions'; method='validate_permissions';title='Validate file permissions. WARNING: this may take a long time on a large system'" CHECK="noemptyok perms:r,'*',wD,'(,v|,pfv)$'" **
# **PATH LABEL="Pub Directory" FEEDBACK="icon='ui-icon-check';label='Validate Permissions'; method='validate_permissions';title='Validate file permissions. WARNING: this may take a long time on a large system'" CHECK="noemptyok perms:r,'*',wDn,'(,v|,pfv)$'" **
# Attachments store (file path, not URL), must match the attachments URL
# path =/foswiki/pub= - for example =/usr/local/foswiki/pub= This directory is
# normally accessible from the web.
# $Foswiki::cfg{PubDir} = '/home/httpd/foswiki/pub';

# **PATH LABEL="Data Directory" FEEDBACK="icon='ui-icon-check';label='Validate Permissions'; method='validate_permissions';title='Validate file permissions. WARNING: this may take a long time on a large system'" CHECK="noemptyok perms:rwDpd,'(,v|,pfv)$',r" **
# **PATH LABEL="Data Directory" FEEDBACK="icon='ui-icon-check';label='Validate Permissions'; method='validate_permissions';title='Validate file permissions. WARNING: this may take a long time on a large system'" CHECK="noemptyok perms:rwDnpd,'(,v|,pfv)$',r" **
# Topic files store (file path, not URL). For example =/usr/local/foswiki/data=.
# This directory must not be web accessible.
# $Foswiki::cfg{DataDir} = '/home/httpd/foswiki/data';
Expand Down Expand Up @@ -1359,12 +1359,14 @@ $Foswiki::cfg{PluralToSingular} = $TRUE;

# **BOOLEAN LABEL="NFC Normalize Filenames" EXPERT **
# Foswiki uses NFC normalization for all network operations, but assumes
# that the file system is also NFC normalized. Some systems such as OSx
# that the file system is also NFC normalized. Some systems such as OS X
# enforce NFD normalization for filenames. If Foswiki is installed on one
# of these sysetms, or accesses such a system via a remote file system
# like NFS, then all directory / filename read operations must be NFC
# normalized.
$Foswiki::cfg{NFCNormalizeFilenames} = $FALSE;
# normalized. Need for NFC normalization is detected and set during bootstrap
# but may need to be overridden if individual webs are symlinked to a NFD
# filesystem.
#$Foswiki::cfg{NFCNormalizeFilenames} = $FALSE;

# **PERL LABEL="Implementation Classes" EXPERT**
# Customisation of the Foswiki Store implementation. This allows
Expand Down
48 changes: 14 additions & 34 deletions core/lib/Foswiki/Configure/Checkers/NFCNormalizeFilenames.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,48 +6,28 @@ use warnings;

use Encode;
use Unicode::Normalize;
use Foswiki::Configure::FileUtil ();

use Foswiki::Configure::Checker ();
our @ISA = ('Foswiki::Configure::Checker');

sub check_current_value {
my ( $this, $reporter ) = @_;
my $e;

# Determine if the file system is NFC or NFD.
# Write a UTF8 filename to the data directory, and then read the directory.
# If the filename is returned in NFD format, then the NFCNormalizeFilename flag is enabled.
my $nfcok =
Foswiki::Configure::FileUtil::canNfcFilenames( $Foswiki::cfg{DataDir} );

my $testfile = 'ČáŘý.testCfgNFC';
if (
open(
my $F, '>', Encode::encode_utf8("$Foswiki::cfg{DataDir}/$testfile")
)
)
{
close($F);
opendir( my $dh, Encode::encode_utf8( $Foswiki::cfg{DataDir} ) )
or die $!;
my @list = grep { /testCfgNFC/ }
map { Encode::decode_utf8($_) } readdir($dh);
if ( scalar @list && $list[0] eq $testfile ) {
$e .= $reporter->NOTE("NFC Data Storage Detected");
$Foswiki::cfg{NFCNormalizeFilenames} = 0;
}
else {
if ( scalar @list && $testfile eq $list[0] ) {
$e .= $reporter->NOTE("NFD Data Storage Detected");
$e .= $reporter->ERROR(
"Filename Normalization should be enabled on NFD File Systems."
) unless ( $Foswiki::cfg{NFCNormalizeFilenames} );
}
else {
$e .= $reporter->WARN(
"Unable to detect Normalization. Read/write of test file failed."
);
}
}
unlink "$Foswiki::cfg{DataDir}/$testfile";
if ( defined $nfcok && $nfcok == 1 ) {
$reporter->NOTE("Data Storage allows NFC filenames");
}
elsif ( defined($nfcok) && $nfcok == 0 ) {
$reporter->NOTE("Data Storage enforces NFD filenames");
$reporter->WARN(
"Filename Normalization should be enabled on NFD File Systems.")
unless ( $Foswiki::cfg{NFCNormalizeFilenames} );
}
else {
$reporter->ERROR("Unable to detect Normalization.");
}
}

Expand Down
44 changes: 31 additions & 13 deletions core/lib/Foswiki/Configure/Checkers/PATH.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,15 +76,16 @@ sub validate_permissions {

my $path = eval("\$Foswiki::cfg$this->{item}->{keys}");

my $fileCount = 0;
my $fileErrors = 0;
my $excessPerms = 0;
my $missingFile = 0;
my @messages;

my $check = $this->{item}->{CHECK}->{perms};

while (@$check) {
my $fileCount = 0;
my $fileErrors = 0;
my $dirErrors = 0;
my $excessPerms = 0;
my $missingFile = 0;
my @messages;

my $perms = shift @$check;
my $filter = shift @$check || '';
$filter = '' if ( $filter eq '*' );
Expand All @@ -104,6 +105,7 @@ sub validate_permissions {
filter => $filter );
$fileCount = $report->{fileCount};
$fileErrors = $report->{fileErrors};
$dirErrors = $report->{dirErrors};
$excessPerms = $report->{excessPerms};
$missingFile = $report->{missingFile};
push( @messages, @{ $report->{messages} } );
Expand All @@ -112,6 +114,7 @@ sub validate_permissions {
my $fperm = sprintf( '%04o', $Foswiki::cfg{Store}{filePermission} );

if ($fileErrors) {
$reporter->NOTE("Insufficient permission checks:");
my $insufficientMsg =
$fileErrors == 1
? "a directory or file has insufficient permissions."
Expand All @@ -120,29 +123,44 @@ sub validate_permissions {
( $perms =~ m/[df]/ )
? "Verify that the Store expert settings of {Store}{filePermission} ($fperm) and {Store}{dirPermission} ($dperm) are correct for your environment, and correct the file permissions listed below"
: '';
$reporter->ERROR( <<ERRMSG, @messages )
$reporter->ERROR( <<ERRMSG )
$insufficientMsg Insufficient permissions could prevent Foswiki or the web server from accessing or updating the files. $storeMsg
ERRMSG
}

if ( $this->{missingFile} ) {
if ($dirErrors) {
$reporter->NOTE("Directory checks:");
my $dirMsg =
$dirErrors == 1
? "a directory issue has been encountered."
: "$dirErrors directories have encountered issues.";
$reporter->ERROR( <<ERRMSG )
$dirMsg Review the reported errors and correct the issue.
ERRMSG
}

if ($missingFile) {
$reporter->NOTE("Missing file checks:");
my $missingMsg =
$this->{missingFile} == 1
? "a file is missing."
: "$this->{missingFile} files are missing.";
$reporter->WARN( <<PREFS, @messages )
$missingFile == 1
? "A file is missing."
: "$missingFile files are missing.";
$reporter->WARN( <<PREFS )
This warning can be safely ignored in many cases. The web directories have been checked for a $Foswiki::cfg{WebPrefsTopicName} topic and $missingMsg If this file is missing, Foswiki will not recognize the directory as a Web and the contents will not be accessible to Foswiki. This is expected with some extensions and might not be a problem. Verify whether or not each directory listed as missing $Foswiki::cfg{WebPrefsTopicName} is intended to be a web. If Foswiki web access is desired, copy in a $Foswiki::cfg{WebPrefsTopicName} topic.
PREFS
}

if ($excessPerms) {
$reporter->WARN( << "PERMS", @messages );
$reporter->NOTE("Excess permission checks:");
$reporter->WARN( << "PERMS" );
$excessPerms or more directories appear to have more access permission than requested in the Store configuration. Excess permissions might allow other users on the web server to have undesired access to the files. Verify that the Store expert settings of {Store}{filePermission} ($fperm} and {Store}{dirPermission}) ($dperm}) are set correctly for your environment and correct the file permissions listed below. (Files were not checked for excessive permissions.)
PERMS
}
$reporter->NOTE(
"Finished checking $fileCount files, Permission: $perms Filter: $filter\n"
);
my $rpt = join( "\n", @messages );
$reporter->NOTE($rpt);
}
}
return;
Expand Down
73 changes: 69 additions & 4 deletions core/lib/Foswiki/Configure/FileUtil.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ Basic file utilities used by Configure and admin scripts

use strict;
use warnings;
use utf8;

use Assert;

Expand Down Expand Up @@ -157,7 +158,8 @@ sub findPackages {
$pattern =~ s/\*/.*/g;
my @path = split( /::/, $pattern );

my $places = \@INC;
my @NFCINC = map { NFC( decode_utf8($_) ) } @INC;
my $places = \@NFCINC;
my $dir;

while ( scalar(@path) > 1 && @$places ) {
Expand Down Expand Up @@ -276,6 +278,7 @@ Enhanced checks:
* f - File permission matches the permission in
{Store}{filePermission} (FUTURE)
* p - Verify that a WebPreferences exists for each web
* n - Verify normalization of the directory location
%options may include the following:
* =filter= is a regular expression. Files matching the regex
Expand Down Expand Up @@ -313,6 +316,7 @@ sub checkTreePerms {
my %report = (
fileCount => 0,
fileErrors => 0,
dirErrors => 0,
missingFile => 0,
excessPerms => 0,
messages => []
Expand All @@ -329,6 +333,7 @@ sub checkTreePerms {
return \%report if ( $path eq '.git' );

$options{maxFileErrors} = 10 unless defined $options{maxFileErrors};
$options{maxDirErrors} = 10 unless defined $options{maxDirErrors};
$options{maxExcessPerms} = 10 unless defined $options{maxExcessPerms};
$options{maxMissingFile} = 10 unless defined $options{maxMissingFile};

Expand Down Expand Up @@ -427,15 +432,30 @@ sub checkTreePerms {
push( @{ $report{messages} }, "=$path= $rwxString" );
}

return \%report if scalar( @{ $report{messages} } );

return \%report unless -d $path;

# Stop at this directory, if it doesn't have -x - readdir permission
if ( -d $path && !-x $path ) {
unshift( @{ $report{messages} }, " * $path missing -x permission" );
$report{dirErrors}++;
return \%report;
}

# The NFC check requires readdir permission.
if ( $perms =~ m/n/
&& !$Foswiki::cfg{NFCNormalizeFilenames}
&& -d $path
&& ( substr( $path, -4 ) ne ',pfv' ) )
{
my $nfcok = Foswiki::Configure::FileUtil::canNfcFilenames($path);
if ( !$nfcok && $report{dirErrors}++ < $options{maxDirErrors} ) {
push(
@{ $report{messages} },
" * =$path= NFD File System detected, Normalization should be enabled"
);
}
}

opendir( my $Dfh, $path )
or return "Directory $path is not readable.";

Expand All @@ -444,7 +464,7 @@ sub checkTreePerms {
my $subreport = checkTreePerms( $p, $perms, %options );
while ( my ( $k, $v ) = each %report ) {
if ( ref($v) eq 'ARRAY' ) {
push( @$v, @{ $subreport->{$k} } );
push( @{ $report{$k} }, @{ $subreport->{$k} } );
}
else {
$report{$k} += $subreport->{$k};
Expand Down Expand Up @@ -975,6 +995,51 @@ sub rewriteShebang {
return '';
}

=begin TML
---++ StaticMethod canNfcFilenames($testdir)
Determine if the file system is NFC or NFD.
Write a UTF8 filename to the data directory, and then read the directory.
If the filename is returned in NFD format, then the NFCNormalizeFilename flag is enabled.
returns:
* 1 if NFC filenames are accepted by the filesystem
* 0 if the NFC is converted to NFD
* undef in any other case (errors)
=cut

sub canNfcFilenames {
my $testdir = shift;

ASSERT( $testdir, "missing argument to canNfcFilenames" );

#die as BUG if the testdir contains non-ascii characters and it isn't unicode string
ASSERT( !( $testdir =~ /\P{Ascii}/ && !utf8::is_utf8($testdir) ),
"CORE bug, got a [$testdir] as bytes" );

my $ext = '.CfgNfcTmpFile';
my $testname = '_ÁčňÖüß' . $ext;
my $fullpath =
NFC( File::Spec->catfile( $testdir, $testname ) ); #ensure full NFC path
my $fsnorm;

unlink $fullpath;
if ( open my $fd, '>', $fullpath ) {
close $fd;
opendir my $dh, $testdir or return; #or die?
my @list = grep { /$ext/ } map { decode_utf8($_) } readdir $dh;
closedir $dh;
return unless ( scalar @list == 1 );
$fsnorm =
( $list[0] eq $testname ) ? 1
: ( $list[0] eq NFD($testname) ) ? 0
: undef;
unlink $fullpath;
}
return $fsnorm;
}

1;
__END__
Foswiki - The Free and Open Source Wiki, http://foswiki.org/
Expand Down
52 changes: 19 additions & 33 deletions core/lib/Foswiki/Configure/Load.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package Foswiki::Configure::Load;

use strict;
use warnings;
use utf8; # Needed to probe NFC/NFD filesystem

use Cwd qw( abs_path );
use Assert;
Expand Down Expand Up @@ -635,38 +634,25 @@ sub _bootstrapStoreSettings {
}
}

# Determine if the file system is NFC or NFD.
# Write a UTF8 filename to the data directory, and then read the directory.
# If the filename is returned in NFD format, then the NFCNormalizeFilename flag is enabled.

my $testfile = 'ČáŘý.testCfgNFC';
if (
open(
my $F, '>', Encode::encode_utf8("$Foswiki::cfg{DataDir}/$testfile")
)
)
{
close($F);
opendir( my $dh, Encode::encode_utf8( $Foswiki::cfg{DataDir} ) )
or die $!;
my @list = grep { /testCfgNFC/ }
map { Encode::decode_utf8($_) } readdir($dh);
if ( scalar @list && $list[0] eq $testfile ) {
print STDERR "AUTOCONFIG: NFC Data Storage Detected\n" if (TRAUTO);
$Foswiki::cfg{NFCNormalizeFilenames} = 0;
}
else {
if ( scalar @list && NFD($testfile) eq $list[0] ) {
print STDERR "AUTOCONFIG: NFD Data Storage Detected\n"
if (TRAUTO);
$Foswiki::cfg{NFCNormalizeFilenames} = 1;
}
else {
print STDERR
"AUTOCONFIG: WARNING: Unable to detect Normalization.\n";
}
}
unlink "$Foswiki::cfg{DataDir}/$testfile";
# Detect the NFC / NDF normalization of the file system, and set
# NFCNormalizeFilenames if needed.
# SMELL: Really this should be done per web, both in data and pub.
my $nfcok =
Foswiki::Configure::FileUtil::canNfcFilenames( $Foswiki::cfg{DataDir} );
if ( defined $nfcok && $nfcok == 1 ) {
print STDERR "AUTOCONFIG: Data Storage allows NFC filenames\n"
if (TRAUTO);
$Foswiki::cfg{NFCNormalizeFilenames} = 0;
}
elsif ( defined($nfcok) && $nfcok == 0 ) {
print STDERR "AUTOCONFIG: Data Storage enforces NFD filenames\n"
if (TRAUTO);
$Foswiki::cfg{NFCNormalizeFilenames} = 1
; #the configure's interface still shows unchecked - so, don't understand.. ;(
}
else {
print STDERR "AUTOCONFIG: WARNING: Unable to detect Normalization.\n";
$Foswiki::cfg{NFCNormalizeFilenames} = 1; #enable too - safer as none
}
}

Expand Down

0 comments on commit 2fdc6c1

Please sign in to comment.