Skip to content

Commit

Permalink
Item14611: add flatdir generator. Fix resources in flatfile generator…
Browse files Browse the repository at this point in the history 
… too. Also make templates for generated topics, such as index. Add debug mode for tracing URL processing.
  • Loading branch information
cdot committed Jan 28, 2018
1 parent 0edf3a1 commit a7bc2ef
Show file tree
Hide file tree
Showing 8 changed files with 230 additions and 88 deletions.
17 changes: 15 additions & 2 deletions data/System/PublishPlugin.txt
View file
Expand Up @@ -130,7 +130,7 @@ table.publishForm .paramCol {
<td class="infoCol">
The =topics= parameter is used to give a (comma-separated) list of web.topic names. These can be specified using [[#WildcardPattern][wildcard patterns]].
* =Myweb.*= will publish all topics in the =Myweb= web (but not in subwebs)
* =Myweb*.*= will publish all topics in the =Myweb= web and all it's subwebs
* =Myweb*.*= will publish all topics in the =Myweb= web and all its subwebs
* =*.*= will publish all topics in the wiki
* =Web.= implies =Web.*=, and =.Topic= and =Topic= both imply =*.Topic=
* The list is expanded in left-right order. You can edit the list at any point by prefixing an entry with a =-= sign e.g. =*.*,-*.Last*= will publish all topics in the wiki in web.topic order except topics starting with =Last=
Expand Down Expand Up @@ -335,6 +335,18 @@ don't need a history.
=history=
</td>
</tr>
<tr>
<td class="nameCol">Debug</td>
<td class="inputCol">
<input type="checkbox" name="debug" class="foswikiCheckbox" />
</td>
<td class="infoCol">
Enable to get a lot of debug messages, mainly relating to the processing of URL.
</td>
<td class="paramCol">
=debug=
</td>
</tr>
</table>
%ENDTWISTY%
</td>
Expand Down Expand Up @@ -454,9 +466,10 @@ If you are using an on-disk file store, such as !PlainFile or one of the RCS sto

%X% Note that overwriting attachments this way is extremely dangerous, so this should only be done by experts! You have been warned. %X%

* Open =configure=
* First set the ={Plugins}{PublishPlugin}{Dir}= to the same as ={PubDir}=
* Then publish with a =relativedir= setting that corresponds to the attachment directory for the web/topic that you want to attach to
* If {AutoAttachPubFiles} is enabled, it will automatically be attached to the topic.
* If ={AutoAttachPubFiles}= is enabled, it will automatically be attached to the topic.

---++ Installation Instructions

Expand Down
10 changes: 6 additions & 4 deletions lib/Foswiki/Plugins/PublishPlugin/BackEnd.pm
View file
Expand Up @@ -38,8 +38,8 @@ use Assert;
Construct a new back end.
* =$params= - optional parameter hash, may contain generator-specific
options
* =$logger= - ref to an object that supports logWarn, logInfo and logError
methods (see Publisher.pm)
* =$logger= - ref to an object that supports logDebug, logWarn, logInfo
and logError methods (see Publisher.pm)
=cut

sub new {
Expand All @@ -55,13 +55,15 @@ sub new {
return $this;
}

# Like join, but for dir and url paths, for subclasses
# Like join, but for dir and url paths, protected utility for subclasses
sub pathJoin {
my $this = shift;

# use length() to exclude undef and empty path els
my $all = join( '/', grep { length($_) } @_ );
$all =~ s://+:/:g; # doubled slash
$all =~ s:/+$::; # trailing /
$all =~ s!^([a-zA-Z0-9]+:/)!$1/!; # reslash abs urls
$all =~ s!^([a-zA-Z0-9]+:/)!$1/!; # reslash absolute urls
return $all;
}

Expand Down
113 changes: 65 additions & 48 deletions lib/Foswiki/Plugins/PublishPlugin/BackEnd/file.pm
View file
Expand Up @@ -36,22 +36,31 @@ sub new {

my $this = $class->SUPER::new( $params, $logger );

# See param_schema for information about outfile and relativedir
$this->{output_file} = $params->{relativedir} || '';
$this->{output_file} =
$this->pathJoin( $this->{output_file}, $params->{outfile} )
if $params->{outfile};

# The root of the directory structure we are writing to.
$this->{file_root} =
$this->pathJoin( $Foswiki::cfg{Plugins}{PublishPlugin}{Dir},
$this->{output_file} );
$this->{logger}->logDebug( '', 'Publishing to ', $this->{file_root} );

$this->{resource_id} = 0;

# List of web.topic paths to already-published topics.
$this->{last_published} = {};

# Capture HTML generated for use by subclasses
# Capture HTML generated.
$this->{html_files} = [];

# Note that both html_files and last_published are indexed on the
# final generated path for the HTML. This *may* look like the
# web.topic path, but that cannot be assumed as getTopicPath may
# have changed it significantly.

if ( !$params->{keep} || $params->{keep} eq 'nothing' ) {

# Don't keep anything
Expand All @@ -60,26 +69,29 @@ sub new {
elsif ( !$params->{dont_keep_existing} ) {

# See what's worth keeping
$this->_scanExistingHTML( $this->{file_root}, '' );
$this->_scanExistingHTML('');
}

return $this;
}

# Find existing HTML in published dir structure to add to sitemap etc
# Find existing HTML in published dir structure to add to sitemap and act
# as targets for links.
# $w - path relative to publishing root
sub _scanExistingHTML {
my ( $this, $root, $relpath ) = @_;
my ( $this, $w ) = @_;
my $d;
return unless ( opendir( $d, "$root/$relpath" ) );
return unless ( opendir( $d, "$this->{file_root}/$w" ) );
while ( my $f = readdir($d) ) {
next if $f =~ /^\./;
if ( -d "$root/$relpath/$f" ) {
$this->_scanExistingHTML( $root, $relpath ? "$relpath/$f" : $f );
if ( -d "$this->{file_root}/$w/$f" ) {
$this->_scanExistingHTML( $w ? "$w/$f" : $f );
}
elsif ( $relpath && $f =~ /\.html$/ ) {
my $p = "$relpath/$f";
push( @{ $this->{html_files} }, Encode::decode_utf8($p) );
$this->{last_published}->{$p} = ( stat("$root/$p") )[9];
elsif ( $w && $f =~ /^\.html$/ ) {
my $p = "$w/$f"; # path relative to file_root
push( @{ $this->{html_files} }, $p );
$this->{last_published}->{$p} =
( stat("$this->{file_root}/$p") )[9];
}
}
closedir($d);
Expand Down Expand Up @@ -111,6 +123,16 @@ sub validatePath {
return $v;
}

# Validate that the parameter refers to an existing web/topic
sub validateWebTopic {
my ( $v, $k ) = @_;
return $v unless $v;
my @wt = Foswiki::Func::normalizeWebTopicName( 'NOT_A_WEB', $v );
die "$k ($v) is not an existing topic " . join( ';', @wt )
unless Foswiki::Func::topicExists(@wt);
return $v;
}

# Implement Foswiki::Plugins::PublishPlugin::BackEnd
sub param_schema {
my $class = shift;
Expand All @@ -123,9 +145,10 @@ sub param_schema {
validator => \&validateFilename
},
defaultpage => {
desc => 'Web.Topic to redirect to from index.html / default.html',
desc =>
'Web.Topic to redirect to from index.html / default.html. If you leave this blank, the index will contain a simple list of all the published topics.',
default => '',
validator => \&validatePath
validator => \&validateWebTopic
},
googlefile => {
desc =>
Expand Down Expand Up @@ -158,16 +181,17 @@ sub param_schema {
# Implement Foswiki::Plugins::PublishPlugin::BackEnd
sub getTopicPath {
my ( $this, $web, $topic ) = @_;

my @path = split( /\/+/, $web );
push( @path,, $topic . '.html' );
push( @path, $topic . '.html' );
return $this->pathJoin(@path);
}

# Implement Foswiki::Plugins::PublishPlugin::BackEnd
sub alreadyPublished {
my ( $this, $web, $topic ) = @_;
return 0 unless ( $this->{params}->{keep} // '' ) eq 'unchanged';
my $pd = $this->{last_published}->{"$web/$topic.html"};
my $pd = $this->{last_published}->{ $this->getTopicPath( $web, $topic ) };
return 0 unless $pd;
my ($cd) = Foswiki::Func::getRevisionInfo( $web, $topic );
return 0 unless $cd;
Expand All @@ -178,10 +202,7 @@ sub alreadyPublished {
sub addTopic {
my ( $this, $web, $topic, $text ) = @_;

my @path = grep { length($_) } split( /\/+/, $web );
push( @path, $topic . '.html' );

my $path = $this->pathJoin(@path);
my $path = $this->getTopicPath( $web, $topic );
push( @{ $this->{html_files} }, $path );
return $this->addByteData( $path, Encode::encode_utf8($text) );
}
Expand All @@ -190,11 +211,9 @@ sub addTopic {
sub addAttachment {
my ( $this, $web, $topic, $attachment, $data ) = @_;

my @path = grep { length($_) } split( /\/+/, $web );
push( @path, $topic . '.attachments' );
push( @path, $attachment );

my $path = $this->pathJoin(@path);
my $path =
$this->pathJoin( $this->getTopicPath( $web, "$topic.attachments" ),
$attachment );
return $this->addByteData( $path, $data );
}

Expand Down Expand Up @@ -251,48 +270,46 @@ sub addByteData {
sub close {
my $this = shift;

Foswiki::Func::loadTemplate('PublishPlugin');

# write sitemap.xml
my $sitemap =
'<?xml version="1.0" encoding="UTF-8"?>'
. '<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">'
. join( "\n",
map { '<url><loc>' . $_ . '</loc></url>'; } @{ $this->{html_files} } )
. '</urlset>';
my $smurl = Foswiki::Func::expandTemplate('PublishPlugin:sitemap_url');
my $smurls = join( "\n",
map { my $x = $smurl; $x =~ s/%URL%/$_/g; $x }
@{ $this->{html_files} } );
my $sitemap = Foswiki::Func::expandTemplate('PublishPlugin:sitemap');
$sitemap =~ s/%URLS%/$smurls/g;

$this->addByteData( 'sitemap.xml', Encode::encode_utf8($sitemap) );

# Write Google verification files (comma separated list)
if ( $this->{params}->{googlefile} ) {
my @files = split( /\s*,\s*/, $this->{params}->{googlefile} );
for my $file (@files) {
my $simplehtml =
'<html><title>'
. $file
. '</title><body>Google verification</body></html>';
Foswiki::Func::expandTemplate('PublishPlugin:googlefile');
$simplehtml =~ s/%FILE%/$file/g;
$this->addByteData( $file, Encode::encode_utf8($simplehtml) );
}
}

# Write default.htm and index.html
my $html = <<'HEAD';
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en-US" xml:lang="en-US">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
HEAD
my $html = Foswiki::Func::expandTemplate('PublishPlugin:redirect');
if ( $this->{params}->{defaultpage} ) {
$this->{params}->{defaultpage} =~ s/\./\//g;
$html .= '<meta http-equiv="REFRESH" content="0; url=';
$html .= $this->{params}->{defaultpage};
$html .=
".html\" />\n</head><body>$this->{params}->{defaultpage} - please wait";
my @wt =
Foswiki::Func::normalizeWebTopicName( undef,
$this->{params}->{defaultpage} );
my $mtag = "<meta http-equiv=\"REFRESH\" content=\"0; url="
. $this->getTopicPath(@wt) . "\" />";
$html =~ s/%HEAD%/$mtag/g;
$html =~ s/%BODY%/$this->{params}->{defaultpage} - please wait/g;
}
else {
$html .= "</head><body>";
$html .= join( "</br>\n",
$html =~ s/%HEAD%//g;
my $bod = join( "</br>\n",
map { "<a href='$_'>$_</a>" } @{ $this->{html_files} } );
$html =~ s/%BODY%/$bod/g;
}
$html .= "\n</body></html>";
$html = Encode::encode_utf8($html);
$this->addByteData( 'default.htm', $html );
$this->addByteData( 'index.html', $html );
Expand Down
35 changes: 35 additions & 0 deletions lib/Foswiki/Plugins/PublishPlugin/BackEnd/flatdir.pm
View file
@@ -0,0 +1,35 @@
#
# Copyright (C) 2018 Crawford Currie, http://c-dot.co.uk
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details, published at
# http://www.gnu.org/copyleft/gpl.html
#
# File writer module for PublishPlugin. Generates a flat directory of HTML
# files, with a _rsrc subdirectory for external resources.
#
package Foswiki::Plugins::PublishPlugin::BackEnd::flatdir;

use strict;

use Foswiki::Plugins::PublishPlugin::BackEnd::file;
our @ISA = ('Foswiki::Plugins::PublishPlugin::BackEnd::file');

use constant DESCRIPTION =>
'Flat directory of HTML files. Attachments (and external resources if =copyexternal is selected=) will be saved to a top level =_rsrc= directory next to the HTML.';

# Override Foswiki::Plugins::PublishPlugin::BackEnd
sub getTopicPath {
my ( $this, $web, $topic ) = @_;
my @path = split( /\/+/, $web );
return join( '_', @path, $topic . '.html' );
}

1;

0 comments on commit a7bc2ef

Please sign in to comment.