Item14115: oo-ify core of plugin, ...

- work around broken mime types for images - fix encoding problems - make it work with newer JsonRpcContrib
foswiki · Sep 2, 2016 · 497b847 · 497b847
1 parent a348a9b
commit 497b847
Show file tree

Hide file tree

Showing 8 changed files with 174 additions and 176 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,11 +1,13 @@
 *.gz
 *.swp
+*.jslint
+igp_*
 pub/System/HarvestPlugin/harvest.css
 pub/System/HarvestPlugin/harvest.js
-HarvestPlugin.md5
-HarvestPlugin.sha1
-HarvestPlugin.tgz
-HarvestPlugin.txt
-HarvestPlugin.zip
-HarvestPlugin_installer
-HarvestPlugin_installer.pl
+/HarvestPlugin.md5
+/HarvestPlugin.sha1
+/HarvestPlugin.tgz
+/HarvestPlugin.txt
+/HarvestPlugin.zip
+/HarvestPlugin_installer
+/HarvestPlugin_installer.pl
diff --git a/data/System/HarvestPlugin.txt b/data/System/HarvestPlugin.txt
@@ -1,6 +1,6 @@
 %META:TOPICINFO{author="ProjectContributor" comment="" date="1418734826" format="1.1" version="1"}%
-
 ---+!! %TOPIC%
+%FORMFIELD{"Description"}%
 
 This plugin can be used to download and archive external resource, like images or pdfs.
 It expects an url, analyzes it and offers a menu of links to all resources reachable from that page.
@@ -21,30 +21,30 @@ have to modify your local !PatternSkin to add this tool to your site.
 <a href="%ATTACHURLPATH%/HarvestPluginSnap3.png"><img src="%ATTACHURLPATH%/HarvestPluginSnap3.png" width="100"  /></a>
 
 ---++ Installation Instructions
-
 %$INSTALL_INSTRUCTIONS%
 
----++ Info
-<!--
-   * Set SHORTDESCRIPTION = %$SHORTDESCRIPTION%
--->
-
-|  Author(s): | Michael Daum|
-|  Copyright: | &copy; 2012-2015 Michael Daum http://michaeldaumconsulting.com |
-|  License: | [[http://www.gnu.org/licenses/gpl.html][GPL (Gnu General Public License)]] |
-|  Release: | %$RELEASE% |
-|  Version: | %$VERSION% |
-|  Change History: | <!-- versions below in reverse order -->&nbsp; |
+---++ Dependencies
+%$DEPENDENCIES%
+
+---++ Change History
+|  02 Sep 2016 | oo-ify core of plugin; work around broken mime types for images, fix encoding problems and make it work with newer <nop>JsonRpcContrib |
 |  12 Jan 2015 | sort results returned from external web page |
 |  16 Dec 2014 | ignore ssl certificate problems when downloading stuff |
 |  29 Sep 2014 | replaced deprecated jquery.tmpl with jsrendr |
 |  03 Sep 2014 | fixed corruption of downloaded files |
 |  04 Apr 2014 | flag rest handlers that don't require authentication |
 |  12 Dec 2013 | first dot.oh release |
-|  Dependencies: | %$DEPENDENCIES% |
-|  Home page: | Foswiki:Extensions/%TOPIC% |
-|  Support: | Foswiki:Support/%TOPIC% |
 
 %META:FILEATTACHMENT{name="HarvestPluginSnap3.png" attachment="HarvestPluginSnap3.png" attr="" comment="" date="1418734826" size="101095" user="ProjectContributor" version="1"}%
 %META:FILEATTACHMENT{name="HarvestPluginSnap2.png" attachment="HarvestPluginSnap2.png" attr="" comment="" date="1418734826" size="51494" user="ProjectContributor" version="1"}%
 %META:FILEATTACHMENT{name="HarvestPluginSnap1.png" attachment="HarvestPluginSnap1.png" attr="" comment="" date="1418734826" size="43468" user="ProjectContributor" version="1"}%
+%META:FORM{name="PackageForm"}%
+%META:FIELD{name="Author" title="Author" value="Michael Daum"}%
+%META:FIELD{name="Copyright" title="Copyright" value="&copy; 2012-2016 Michael Daum http://michaeldaumconsulting.com"}%
+%META:FIELD{name="Description" title="Description" value="%25$SHORTDESCRIPTION%25"}%
+%META:FIELD{name="Home" title="Home" value="Foswiki:Extensions/%TOPIC%"}%
+%META:FIELD{name="License" title="License" value="[[http://www.gnu.org/licenses/gpl.html][GPL (Gnu General Public License)]]"}%
+%META:FIELD{name="Release" title="Release" value="%$RELEASE%"}%
+%META:FIELD{name="Repository" title="Repository" value="https://github.com/foswiki/%TOPIC%"}%
+%META:FIELD{name="Support" title="Support" value="Foswiki:Support/%TOPIC%"}%
+%META:FIELD{name="Version" title="Version" value="%$VERSION%"}%
diff --git a/lib/Foswiki/Plugins/HarvestPlugin.pm b/lib/Foswiki/Plugins/HarvestPlugin.pm
@@ -1,6 +1,6 @@
 # Plugin for Foswiki - The Free and Open Source Wiki, http://foswiki.org/
 #
-# HarvestPlugin is Copyright (C) 2011-2015 Michael Daum http://michaeldaumconsulting.com
+# HarvestPlugin is Copyright (C) 2011-2016 Michael Daum http://michaeldaumconsulting.com
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -27,12 +27,10 @@ use warnings;
 use Foswiki::Func ();
 use Foswiki::Contrib::JsonRpcContrib ();
 
-our $VERSION = '1.21';
-our $RELEASE = '1.21';
+our $VERSION = '2.00';
+our $RELEASE = '02 Sep 2016';
 our $SHORTDESCRIPTION = 'Download and archive resources from the web';
 our $NO_PREFS_IN_TOPIC = 1;
-our $baseWeb;
-our $baseTopic;
 our $core;
 
 =begin TML
@@ -42,7 +40,6 @@ our $core;
 =cut
 
 sub initPlugin {
-  ($baseTopic, $baseWeb) = @_;
 
   Foswiki::Contrib::JsonRpcContrib::registerMethod('HarvestPlugin', 'analyze', sub { return getCore()->jsonRpcAnalyze(@_); });
   Foswiki::Contrib::JsonRpcContrib::registerMethod('HarvestPlugin', 'attach',  sub { return getCore()->jsonRpcAttach(@_); });
@@ -86,7 +83,7 @@ sub getCore {
 
   unless (defined $core) {
     require Foswiki::Plugins::HarvestPlugin::Core;
-    $core = new Foswiki::Plugins::HarvestPlugin::Core($baseWeb, $baseTopic);
+    $core = Foswiki::Plugins::HarvestPlugin::Core->new();
   }
 
   return $core;

diff --git a/lib/Foswiki/Plugins/HarvestPlugin/Core.pm b/lib/Foswiki/Plugins/HarvestPlugin/Core.pm
@@ -1,6 +1,6 @@
 # Plugin for Foswiki - The Free and Open Source Wiki, http://foswiki.org/
 #
-# HarvestPlugin is Copyright (C) 2011-2015 Michael Daum http://michaeldaumconsulting.com
+# HarvestPlugin is Copyright (C) 2011-2016 Michael Daum http://michaeldaumconsulting.com
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -21,10 +21,11 @@ use Foswiki::Contrib::JsonRpcContrib::Error ();
 use Foswiki::AccessControlException ();
 use Foswiki::Func ();
 use Foswiki::Sandbox ();
+use Foswiki::Plugins ();
 use Digest::MD5 ();
 use Error qw(:try);
-use Encode ();
 use URI ();
+use File::Temp ();
 
 use constant TRACE => 0; # toggle me
 
@@ -59,15 +60,17 @@ constructor for the core
 =cut
 
 sub new {
-  my ($class, $baseWeb, $baseTopic) = @_;
+  my $class= shift;
 
   my $workingDir = Foswiki::Func::getWorkArea('HarvestPlugin');
+  my $session = $Foswiki::Plugins::SESSION;
 
   my $this = bless({
-    baseWeb => $baseWeb,
-    baseTopic => $baseTopic,
+    baseWeb => $session->{webName},
+    baseTopic => $session->{topicName},
     cacheRoot => $workingDir.'/cache',
     cacheExpire => "1 h",
+    @_
   }, $class);
 
   return $this;
@@ -235,20 +238,38 @@ sub jsonRpcAttach {
   }
 
   foreach my $url (@$selection) {
-    require File::Temp;
 
+    $url = URI->new($url);
     writeDebug("url=$url");
 
-    my $content = $this->getExternalResource($url);
+    my ($content, $contentType) = $this->getExternalResource($url);
     next unless $content;
 
     # SMELL: first have to write it to a temp file before being able to attach it
     my $tempFile = new File::Temp(UNLINK => TRACE?0:1);
     print $tempFile $content;
 
-    my $baseFilename = $url;
-    $baseFilename =~ s/^.*[\/\\](.*?)(?:\?.*)?$/$1/;
-    $baseFilename =~ s/%([\da-f]{2})/chr(hex($1))/gei;
+    my $baseFilename;
+    foreach my $segment (reverse $url->path_segments) {
+      if ($segment ne '') {
+        $baseFilename = $segment;
+        $baseFilename =~ s/%([\da-f]{2})/chr(hex($1))/gei;
+
+        # SMELL: just covering a few
+        $baseFilename .= ".jpeg" if $baseFilename !~ /\.jpe?g$/ && $contentType eq 'image/jpeg';
+        $baseFilename .= ".gif" if $baseFilename !~ /\.gif$/ && $contentType eq 'image/gif';
+        $baseFilename .= ".png" if $baseFilename !~ /\.png$/ && $contentType eq 'image/png';
+        $baseFilename .= ".tiff" if $baseFilename !~ /\.tiff$/ && $contentType eq 'image/tiff';
+        $baseFilename .= ".bmp" if $baseFilename !~ /\.bmp$/ && $contentType eq 'image/bmp';
+        $baseFilename .= ".webp" if $baseFilename !~ /\.webp$/ && $contentType eq 'image/webp';
+        last;
+      }
+    }
+
+    unless ($baseFilename) {
+      writeDebug("wasn't able to detect basefile from $url");
+      next;
+    }
 
     my $origName;
 
@@ -288,7 +309,7 @@ sub jsonRpcAnalyze {
   my $include = $request->param("include");
   my $maxDepth = $Foswiki::cfg{HarvestPlugin}{MaxDepth};
   $maxDepth = 1 unless defined $maxDepth;
-  my $timeout = $foswiki::cfg{HarvestPlugin}{TimeOut} || 60;  
+  my $timeout = $Foswiki::cfg{HarvestPlugin}{TimeOut} || 60;  
 
   throw Foswiki::Contrib::JsonRpcContrib::Error(1001, "invalid depth parameter")
     if $depth < 0 || $depth > $maxDepth;
@@ -481,12 +502,6 @@ sub node2record {
     $record->{shorttitle} = $record->{title};
   }
 
-  # encode to utf8...SMELL: shouldn't that be done inside JsonRpcContrib?
-  while (my ($key, $val) = each %$record) {
-    $record->{$key} = Encode::encode_utf8($val);
-    #print STDERR "$key=$val\n";
-  }
-
   # filter some
   return if $record->{width} && $record->{width} eq 1 && $record->{height} && $record->{height} eq 1;
   return $record;
@@ -524,48 +539,53 @@ sub getMimeType {
 
 =begin TML
 
----++ getExternalResource($url) -> $content
-
-wrapper to Foswiki::Net::getExternalResource() which
-adds a cache to it
+---++ getExternalResource($url) -> ($content, $type)
 
 =cut
 
 sub getExternalResource {
   my ($this, $url) = @_;
 
   my $cache = $this->_cache;
+  my $content;
+  my $contentType;
 
   $url =~ s/\/$//;
 
   if ($cache) {
-    my $content = $cache->get(_cache_key($url));
-    writeDebug("found content for $url in cache") if defined $content;
-    return $content if defined $content;
+    my $bucket = $cache->get(_cache_key($url));
+    if (defined $bucket) {
+      $content = $bucket->{content};
+      $contentType = $bucket->{type};
+      writeDebug("found content for $url in cache contentType=$contentType");
+    }
   }
 
-  my $client = $this->client;
-  my $res = $client->get($url);
+  unless (defined $content) { 
+    my $client = $this->client;
+    my $res = $client->get($url);
 
-  throw Foswiki::Contrib::JsonRpcContrib::Error(1002, "error fetching url") 
-    unless $res;
+    throw Foswiki::Contrib::JsonRpcContrib::Error(1002, "error fetching url") 
+      unless $res;
 
-  unless ($res->is_success) {
-    writeDebug("url=$url, http error=".$res->status_line);
-    throw Foswiki::Contrib::JsonRpcContrib::Error(1004, "http error fetching $url: ".$res->code." - ".$res->status_line);
-  }
+    unless ($res->is_success) {
+      writeDebug("url=$url, http error=".$res->status_line);
+      throw Foswiki::Contrib::JsonRpcContrib::Error(1004, "http error fetching $url: ".$res->code." - ".$res->status_line);
+    }
 
-  writeDebug("http status=".$res->status_line);
+    writeDebug("http status=".$res->status_line);
 
-  my $content = $res->decoded_content();
-  my $contentType = $res->header('Content-Type');
-  writeDebug("content type=$contentType");
+    $content = $res->decoded_content();
+    $contentType = $res->header('Content-Type');
+    writeDebug("content type=$contentType");
 
-  if ($cache) {
-    writeDebug("caching content for $url");
-    $cache->set(_cache_key($url), $content);
+    if ($cache) {
+      writeDebug("caching content for $url");
+      $cache->set(_cache_key($url), {content => $content, type => $contentType});
+    }
   }
 
+  return ($content, $contentType) if wantarray;
   return $content;
 }
 
@@ -623,7 +643,7 @@ sub client {
       $proxy .= ':' . $port if $port;
       $ua->proxy([ 'http', 'https' ], $proxy);
 
-      my $proxySkip = $Foswiki::cfg{PROXY}{SkipProxyForDomains};
+      my $proxySkip = $Foswiki::cfg{PROXY}{SkipProxyForDomains} || $Foswiki::cfg{PROXY}{NoProxy};
       if ($proxySkip) {
         my @skipDomains = split(/\s*,\s*/, $proxySkip);
         $ua->no_proxy(@skipDomains);

diff --git a/lib/Foswiki/Plugins/HarvestPlugin/DEPENDENCIES b/lib/Foswiki/Plugins/HarvestPlugin/DEPENDENCIES
@@ -1,3 +1,3 @@
 Foswiki::Plugins::MimeIconPlugin,>=1.1,perl,Required. 
-Foswiki::Contrib::JsonRpcContrib,>=1.1,perl,Required. 
+Foswiki::Contrib::JsonRpcContrib,>=2.23,perl,Required. 
 Foswiki::Plugins::JQueryPlugin,>=6.00,perl,Required. 
diff --git a/lib/Foswiki/Plugins/HarvestPlugin/build.pl b/lib/Foswiki/Plugins/HarvestPlugin/build.pl
@@ -1,24 +1,10 @@
-#!/usr/bin/perl -w
+#!/usr/bin/env perl
 BEGIN { unshift @INC, split( /:/, $ENV{FOSWIKI_LIBS} ); }
 use Foswiki::Contrib::Build;
 
 # Create the build object
 $build = new Foswiki::Contrib::Build('HarvestPlugin');
 
-# (Optional) Set the details of the repository for uploads.
-# This can be any web on any accessible Foswiki installation.
-# These defaults will be used when expanding tokens in .txt
-# files, but be warned, they can be overridden at upload time!
-
-# name of web to upload to
-$build->{UPLOADTARGETWEB} = 'Extensions';
-# Full URL of pub directory
-$build->{UPLOADTARGETPUB} = 'http://foswiki.org/pub';
-# Full URL of bin directory
-$build->{UPLOADTARGETSCRIPT} = 'http://foswiki.org/bin';
-# Script extension
-$build->{UPLOADTARGETSUFFIX} = '';
-
 # Build the target on the command line, or the default target
 $build->build($build->{target});