From adc103f3b6cabc34044f48c13fceb55d8d27d00e Mon Sep 17 00:00:00 2001 From: MichaelDaum Date: Fri, 29 Apr 2022 10:51:36 +0200 Subject: [PATCH] Item15096: added =decode=, =include= and =exclude= params --- data/System/FeedPlugin.txt | 20 ++-- lib/Foswiki/Plugins/FeedPlugin.pm | 6 +- lib/Foswiki/Plugins/FeedPlugin/Config.spec | 3 + lib/Foswiki/Plugins/FeedPlugin/Core.pm | 110 ++++++++++++++------ lib/Foswiki/Plugins/FeedPlugin/DEPENDENCIES | 1 + 5 files changed, 93 insertions(+), 47 deletions(-) diff --git a/data/System/FeedPlugin.txt b/data/System/FeedPlugin.txt index 55c90ae..fe30b8c 100644 --- a/data/System/FeedPlugin.txt +++ b/data/System/FeedPlugin.txt @@ -1,4 +1,4 @@ -%META:TOPICINFO{author="ProjectContributor" comment="" date="1461667167" format="1.1" version="1"}% +%META:TOPICINFO{author="micha" comment="reprev" date="1626170466" format="1.1" reprev="2" version="2"}% ---+!! %TOPIC% %FORMFIELD{"Description"}% @@ -15,7 +15,7 @@ List headlines at http://blog.foswiki.org; auto-discover news feed: %STARTSECTION{"example1"}% %FEED{ - "http://blog.foswiki.org" + "https://blog.foswiki.org" discover="on" header="
$n" footer="$n
" @@ -23,16 +23,12 @@ List headlines at http://blog.foswiki.org; auto-discover news feed: %ENDSECTION{"example1"}% -%IF{"context FeedPluginEnabled" - then="$percntINCLUDE{\"%WEB%.%TOPIC%\" section=\"example1\"}$percnt" -}% - Format the most recent posting on the Foswiki Blog: %STARTSECTION{"example2"}% %FEED{ - "http://blog.foswiki.org/feed/" + "https://blog.foswiki.org/Blog/WebRss" limit="1" header="
$n" format="---+!!
$date
[[$link][$title]] @@ -42,10 +38,6 @@ Format the most recent posting on the Foswiki Blog: %ENDSECTION{"example2"}% -%IF{"context FeedPluginEnabled" - then="$percntINCLUDE{\"%WEB%.%TOPIC%\" section=\"example2\"}$percnt" -}% - ---++ Syntax | *Parameter* | *Description* | *Default* | @@ -58,6 +50,9 @@ Format the most recent posting on the Foswiki Blog: | =footer="..."= | format string to be appended to the list of items in the feed | | | =separator="..."= | format string to separte items in the feed | | | =discover="on/off"= | switch on feed discovery starting at the source url | =off= | +| =decode="on/off"= | switch on entity decoding of the feed before parsing it | =off= | +| =exclude="..."= | regular expression of items not to be included | | +| =include="..."= | regular expression of items to be included | | An empty result will be returned when no items have been found in the feed (or all items have been skipped). @@ -105,6 +100,7 @@ In addition the =header=, =format=, =separator= and =footer= format strings may ---++ Change History %TABLE{columnwidths="7em" tablewidth="100%"}% +| 28 Apr 2022 | added parameter to decode html entities; added =include= and =exclude= parameters | | 20 Oct 2020 | use Foswiki:Extensions/CacheContrib instead of local caching code | | 28 May 2018 | support =$date= as a synonym for =$issued=, added =expire= parameter (basically an alias for =refresh= but more intuitive); supports Foswiki's standard proxy/noproxy settings | | 24 Apr 2016 | fixed docu; fixed discover mode; added support for non-unicode Foswiki engines | @@ -117,7 +113,7 @@ In addition the =header=, =format=, =separator= and =footer= format strings may %META:FIELD{name="Release" title="Release" value="%25$RELEASE%25"}% %META:FIELD{name="Description" title="Description" value="%25$SHORTDESCRIPTION%25"}% %META:FIELD{name="Repository" title="Repository" value="https://github.com/foswiki/%25$ROOTMODULE%25"}% -%META:FIELD{name="Copyright" title="Copyright" value="2016-2020, Michael Daum, All Rights Reserved"}% +%META:FIELD{name="Copyright" title="Copyright" value="2016-2022, Michael Daum, All Rights Reserved"}% %META:FIELD{name="License" title="License" value="GPL ([[http://www.gnu.org/copyleft/gpl.html][GNU General Public License]])"}% %META:FIELD{name="Home" title="Home" value="https://foswiki.org/Extensions/%25$ROOTMODULE%25"}% %META:FIELD{name="Support" title="Support" value="https://foswiki.org/Support/%25$ROOTMODULE%25"}% diff --git a/lib/Foswiki/Plugins/FeedPlugin.pm b/lib/Foswiki/Plugins/FeedPlugin.pm index 5ee0440..b96ae59 100644 --- a/lib/Foswiki/Plugins/FeedPlugin.pm +++ b/lib/Foswiki/Plugins/FeedPlugin.pm @@ -1,6 +1,6 @@ # Plugin for Foswiki - The Free and Open Source Wiki, http://foswiki.org/ # -# FeedPlugin is Copyright (C) 2016-2020 Michael Daum http://michaeldaumconsulting.com +# FeedPlugin is Copyright (C) 2016-2022 Michael Daum http://michaeldaumconsulting.com # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -20,8 +20,8 @@ use warnings; use Foswiki::Func (); -our $VERSION = '2.00'; -our $RELEASE = '20 Oct 2020'; +our $VERSION = '2.10'; +our $RELEASE = '28 Apr 2022'; our $SHORTDESCRIPTION = 'Syndication feed parser'; our $NO_PREFS_IN_TOPIC = 1; our $core; diff --git a/lib/Foswiki/Plugins/FeedPlugin/Config.spec b/lib/Foswiki/Plugins/FeedPlugin/Config.spec index 7da3296..5bf6395 100644 --- a/lib/Foswiki/Plugins/FeedPlugin/Config.spec +++ b/lib/Foswiki/Plugins/FeedPlugin/Config.spec @@ -4,4 +4,7 @@ # **NUMBER EXPERT** $Foswiki::cfg{FeedPlugin}{Timeout} = 10; +# **STRING EXPERT** +$Foswiki::cfg{FeedPlugin}{CacheExpire} = '1 d'; + 1; diff --git a/lib/Foswiki/Plugins/FeedPlugin/Core.pm b/lib/Foswiki/Plugins/FeedPlugin/Core.pm index 4b4547b..1119c90 100644 --- a/lib/Foswiki/Plugins/FeedPlugin/Core.pm +++ b/lib/Foswiki/Plugins/FeedPlugin/Core.pm @@ -1,6 +1,6 @@ # Plugin for Foswiki - The Free and Open Source Wiki, http://foswiki.org/ # -# FeedPlugin is Copyright (C) 2016-2020 Michael Daum http://michaeldaumconsulting.com +# FeedPlugin is Copyright (C) 2016-2022 Michael Daum http://michaeldaumconsulting.com # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -23,7 +23,9 @@ use Foswiki::Time (); use Foswiki::Contrib::CacheContrib (); use XML::Feed(); use Encode (); +use HTML::Entities (); use Error qw(:try); +#use Data::Dump qw(dump); use constant TRACE => 0; # toggle me @@ -58,6 +60,10 @@ sub _inlineError { return "".$_[0].""; } +sub _ua { + return Foswiki::Contrib::CacheContrib::getUserAgent("FeedPlugin"); +} + =begin TML ---++ FEED() @@ -83,14 +89,17 @@ sub FEED { $url = shift @feeds; } + my $expire = $params->{refresh} // $params->{expire} // $Foswiki::cfg{FeedPlugin}{CacheExpire} // "1 d"; + my $request = Foswiki::Func::getRequestObject(); my $doRefresh = $request->param("refresh") || ''; - Foswiki::Contrib::CacheContrib::getCache("UserAgent")->remove($url) if $doRefresh =~ /^(on|feed)$/; + my $cache = _ua->getCache($expire); + $cache->remove($url) if $doRefresh =~ /^(on|feed)$/; my $error; my $res; try { - $res = Foswiki::Contrib::CacheContrib::getExternalResource($url); + $res = _ua->get($url); throw Error::Simple("error fetching url") unless $res; @@ -106,7 +115,17 @@ sub FEED { return _inlineError($error) if defined $error; my $text = $res->decoded_content(); - my $feed = XML::Feed->parse(\$text) or return _inlineError("Error: ". XML::Feed->errstr);; + $text = HTML::Entities::decode_entities($text) if Foswiki::Func::isTrue($params->{decode}); + my $since = _parseTime($params->{since} // 0); + + my $feed; + try { + $feed = XML::Feed->parse(\$text) or $error = XML::Feed->errstr; + } otherwise { + $error = shift; + $error =~ s/ at .*$//; + }; + return _inlineError("Error parsing feed: $error") if defined $error; my $format = $params->{format} || ' * [[$link][$title]]$n'; my $header = $params->{header} || ''; @@ -114,37 +133,46 @@ sub FEED { my $separator = $params->{separator} || ''; my $limit = $params->{limit} || 0; my $skip = $params->{skip} || 0; + my $exclude = $params->{exclude}; + my $include = $params->{include}; my @result = (); my $index = 0; foreach my $entry ($feed->entries) { $index++; + my $title = _decode($entry->title); + + next if $exclude && $title =~ /$exclude/; + next if $include && $title !~ /$include/; + last if $limit && $index > ($limit+$skip); next if $skip && $index <= $skip; - my $line = $format; - my $category = _encode(join(", ", $entry->category()) || ''); - my $tags = _encode(join(", ", $entry->tags()) || ''); - my $content = ""._encode($entry->content->body()).""; - my $summary = _encode($entry->summary->body()); + my $line = $format; + my $category = _decode(join(", ", $entry->category()) || ''); + my $tags = _decode(join(", ", $entry->tags()) || ''); + my $content = ""._decode($entry->content->body()).""; + my $summary = _decode($entry->summary->body()) // ''; my $issued = $entry->issued; $issued = $issued->epoch if defined $issued; my $modified = $entry->modified; $modified = $modified->epoch if defined $modified; - $line =~ s/\$author/_encode($entry->author)/ge; - $line =~ s/\$base/_encode($entry->base)/ge; - $line =~ s/\$category/$category/g; - $line =~ s/\$content/$content/g; - $line =~ s/\$id/_encode($entry->id)/ge; - $line =~ s/\$index/$index/g; + next if $since && $issued && $issued < $since; + next if $since && $modified && $modified < $since; + + $line =~ s/\$author\b/_decode($entry->author)/ge; + $line =~ s/\$base\b/_decode($entry->base)/ge; + $line =~ s/\$category\b/$category/g; + $line =~ s/\$content\b/$content/g; + $line =~ s/\$id\b/_decode($entry->id)/ge; + $line =~ s/\$index\b/$index/g; $line =~ s/\$(?:issued|date)(?:\((.*?)\))?/defined($issued)?Foswiki::Time::formatTime($issued, $1 || '$day $month $year'):""/ge; - $line =~ s/\$link/_encode($entry->link)/ge; + $line =~ s/\$link\b/_decode($entry->link)/ge; $line =~ s/\$modified(?:\((.*?)\))?/defined($modified)?Foswiki::Time::formatTime($modified, $1 || '$day $month $year'):""/ge; - $line =~ s/\$summary/$summary/g; - $line =~ s/\$tags/$tags/g; - $line =~ s/\$title/_encode($entry->title)/ge; - + $line =~ s/\$summary\b/$summary/g; + $line =~ s/\$tags\b/$tags/g; + $line =~ s/\$title\b/$title/g; $line =~ s/%(\w)/%$1/g; push @result, $line; } @@ -152,25 +180,43 @@ sub FEED { my $result = $header.join($separator, @result).$footer; - $result =~ s/\$feed_author/_encode($feed->author)/ge; - $result =~ s/\$feed_base/_encode($feed->base)/ge; - $result =~ s/\$feed_copyright/_encode($feed->copyright)/ge; - $result =~ s/\$feed_format/_encode($feed->format)/ge; - $result =~ s/\$feed_generator/_encode($feed->generator)/ge; - $result =~ s/\$feed_language/_encode($feed->language)/ge; - $result =~ s/\$feed_link/_encode($feed->link)/ge; - $result =~ s/\$feed_modified/_encode($feed->modified)/ge; + $result =~ s/\$feed_author/_decode($feed->author)/ge; + $result =~ s/\$feed_base/_decode($feed->base)/ge; + $result =~ s/\$feed_copyright/_decode($feed->copyright)/ge; + $result =~ s/\$feed_format/_decode($feed->format)/ge; + $result =~ s/\$feed_generator/_decode($feed->generator)/ge; + $result =~ s/\$feed_language/_decode($feed->language)/ge; + $result =~ s/\$feed_link/_decode($feed->link)/ge; + $result =~ s/\$feed_modified/_decode($feed->modified)/ge; $result =~ s/\$feed_modified(?:\((.*?)\))?/Foswiki::Time::formatTime($feed->modified->epoch, $1 || '$day $month $year')/ge; - $result =~ s/\$feed_tagline/_encode($feed->tagline)/ge; - $result =~ s/\$feed_title/_encode($feed->title)/ge; + $result =~ s/\$feed_tagline/_decode($feed->tagline)/ge; + $result =~ s/\$feed_title/_decode($feed->title)/ge; return Foswiki::Func::decodeFormatTokens($result); } +sub _parseTime { + my $string = shift; + + return unless defined $string; + return $string if $string =~ /^\d+$/; + return Foswiki::Time::parseTime($string, undef, {lang => "en"}); +} + +sub _decode { + my $string = shift; + + return "" unless defined $string; +return $string; ### SMELL + + return Encode::decode_utf8($string); +} + sub _encode { my $string = shift; - $string = Encode::encode($Foswiki::cfg{Site}{CharSet}, $string) unless $Foswiki::UNICODE; - return $string; + + return "" unless defined $string; + return Encode::encode_utf8($string); } 1; diff --git a/lib/Foswiki/Plugins/FeedPlugin/DEPENDENCIES b/lib/Foswiki/Plugins/FeedPlugin/DEPENDENCIES index 0d24129..5d0fbf4 100644 --- a/lib/Foswiki/Plugins/FeedPlugin/DEPENDENCIES +++ b/lib/Foswiki/Plugins/FeedPlugin/DEPENDENCIES @@ -1,2 +1,3 @@ Foswiki::Contrib::CacheContrib,>=0,perl,Required XML::Feed,>=0.53,cpan,Required +HTML::Entities,>=0,cpan,required