Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge branch 'master' of github.com:bricoleurs/bricolage

  • Loading branch information...
commit d40aab7ba2ee5c22e8ee9881d6f30fce4dfe07f6 2 parents 5e59259 + 631e289
@theory theory authored
View
10 README.Debian
@@ -51,6 +51,15 @@ There was no Net::SFTP installer from Debian, so proceed with
$ sudo cpan install Net::SFTP
+Amazon S3 Mover
+---------------
+
+If you use the optional Amazon S3 mover, you have to install
+Net::Amazon::S3 by doing the following:
+
+ $ sudo apt-get install libnet-amazon-s3-perl
+
+
Perl module dependencies
------------------------
@@ -150,6 +159,7 @@ Optional, via CPAN:
Devel::Profiler
Net::SFTP
+ Net::Amazon::S3
Math::BigInt::GMP
Text::Aspell
PHP::Interpreter
View
12 conf/bricolage.conf
@@ -175,11 +175,12 @@ PASSWD_LENGTH = 5
# Bricolage cannot ascertain the media type. Defaults to "text/html" if not
# specified.
#
-# Set ENABLE_SFTP_MOVER or ENABLE_WEBDAV_MOVER to 'yes' or 'on' or 1, to tell
-# the API that the distribution engine can distribute resources by SFTP or by
-# the WebDAV protocol. Both are disabled by default. Using either the SFTP or
-# the WebDAV mover requires some manual installation; perldoc Bric::Admin for
-# details.
+# Set ENABLE_SFTP_MOVER, ENABLE_S3_MOVER, or ENABLE_WEBDAV_MOVER to
+# 'yes' or 'on' or 1, to tell the API that the distribution engine can
+# distribute resources by SFTP, by Amazon S3, or by the WebDAV
+# protocol. All of those are disabled by default. Using either the SFTP,
+# the Amazon S3, or the WebDAV mover requires some manual installation;
+# perldoc Bric::Admin for details.
#
# Setting SFTP_MOVER_CIPHER will force the SFTP mover to use that cipher when
# distributing resources via SFTP. Consult the Net::SSH2 documentation
@@ -214,6 +215,7 @@ SFTP_MOVER_CIPHER = 0
SFTP_KEY_TYPE = 0
SFTP_PRIVATE_KEY_FILE = 0
SFTP_PUBLIC_KEY_FILE = 0
+ENABLE_S3_MOVER = No
ENABLE_WEBDAV_MOVER = No
QUEUE_PUBLISH_JOBS = No
FTP_UNLINK_BEFORE_MOVE = No
View
19 inst/upgrade/2.1.0/s3_mover.pl
@@ -0,0 +1,19 @@
+#!/usr/bin/perl -w
+
+use strict;
+use FindBin;
+use lib "$FindBin::Bin/../lib";
+use bric_upgrade qw(:all);
+
+exit if fetch_sql(qq{
+ SELECT 1
+ FROM class
+ WHERE id = 82
+});
+
+do_sql(qq{
+ INSERT INTO class (id, key_name, pkg_name, disp_name, plural_name,
+ description, distributor)
+ VALUES (82, 's3', 'Bric::Util::Trans::S3', 'Amazon S3', 'Amazon S3 Transport',
+ 'Class with methods to move files via Amazon S3.', 1)
+});
View
8 lib/Bric/Admin.pod
@@ -2102,6 +2102,14 @@ entered in the Bricolage interface when setting up the destination.
=item *
+B<ENABLE_S3_MOVER:> This directive indicates whether the Bricolage
+server may distribute resources using the Amazon S3 mover. Setting
+this to "On" or "Yes" or "1" will enable the Amazon S3 mover. In order
+to use the Amazon S3 mover, you must install the Net::Amazon::S3
+module from the CPAN (and its prerequisites).
+
+=item *
+
B<ENABLE_WEBDAV_MOVER:> Bricolage also supports resource distribution through
the WebDAV protocol. Set this directive to "On" or "Yes" or "1" to enable it.
You need to have installed HTTP::DAV from the CPAN, in order to use WebDAV
View
4 lib/Bric/Changes.pod
@@ -16,6 +16,10 @@ This document lists the Changes to Bricolage introduced with each release.
=item *
+Added Amazon S3 mover. [John Durkin and Darren Embry]
+
+=item *
+
Added support for the L<CKeditor WYSIWYG editor|http://ckeditor.com/>.
[Zdravko Balorda]
View
5 lib/Bric/Config.pm
@@ -102,6 +102,7 @@ our @EXPORT_OK = qw(DBD_PACKAGE
MEDIA_URI_ROOT
DEF_MEDIA_TYPE
ENABLE_SFTP_MOVER
+ ENABLE_S3_MOVER
ENABLE_SFTP_V2
SFTP_MOVER_CIPHER
SFTP_KEY_TYPE
@@ -251,6 +252,7 @@ our %EXPORT_TAGS = (all => \@EXPORT_OK,
FTP_UNLINK_BEFORE_MOVE
ENABLE_SFTP_MOVER
ENABLE_SFTP_V2
+ ENABLE_S3_MOVER
SFTP_MOVER_CIPHER
SFTP_KEY_TYPE
SFTP_PRIVATE_KEY_FILE
@@ -472,7 +474,7 @@ require Bric; our $VERSION = Bric->VERSION;
ENABLE_WYSIWYG AUTOGENERATE_SLUG ENABLE_GZIP
MEDIA_UNIQUE_FILENAME LDAP_TLS AUTO_PREVIEW_MEDIA
MASON_STATIC_SOURCE ALLOW_URIS_WITHOUT_CATEGORIES
- EXPIRE_ON_DEACTIVATE SKIP_SSL_REDIRECT))
+ EXPIRE_ON_DEACTIVATE SKIP_SSL_REDIRECT ENABLE_S3_MOVER))
{
my $d = exists $config->{$_} ? lc($config->{$_}) : '0';
$config->{$_} = $d eq 'on' || $d eq 'yes' || $d eq '1' ? 1 : 0;
@@ -588,6 +590,7 @@ require Bric; our $VERSION = Bric->VERSION;
use constant PREVIEW_MASON => $config->{PREVIEW_MASON};
use constant DEF_MEDIA_TYPE => $config->{DEF_MEDIA_TYPE} || 'text/html';
use constant ENABLE_SFTP_MOVER => $config->{ENABLE_SFTP_MOVER};
+ use constant ENABLE_S3_MOVER => $config->{ENABLE_S3_MOVER};
use constant ENABLE_SFTP_V2 => $config->{ENABLE_SFTP_V2};
use constant SFTP_MOVER_CIPHER => $config->{SFTP_MOVER_CIPHER} || 0;
use constant SFTP_KEY_TYPE => $config->{SFTP_KEY_TYPE};
View
9 lib/Bric/Dist/Action/Mover.pm
@@ -40,6 +40,10 @@ if (ENABLE_SFTP_MOVER) {
require Bric::Util::Trans::SFTP;
}
+if (ENABLE_S3_MOVER) {
+ require Bric::Util::Trans::S3;
+}
+
if (ENABLE_WEBDAV_MOVER) {
require Bric::Util::Trans::WebDAV;
}
@@ -152,6 +156,11 @@ sub do_it {
"but SFTP Mover is disabled.");
}
+ unless (ENABLE_S3_MOVER or $class !~ /::S3$/) {
+ throw_ap(error => "Output channel is associated with S3 " .
+ "but S3 Mover is disabled.");
+ }
+
unless (ENABLE_WEBDAV_MOVER or $class !~ /::WebDAV$/) {
throw_ap(error => "Output channel is associated with WebDAV" .
"but WebDAV Mover is disabled.");
View
7 lib/Bric/Dist/ServerType.pm
@@ -561,12 +561,13 @@ B<Notes:> NONE.
=cut
sub list_move_methods {
- my $and_sftp = ENABLE_SFTP_MOVER ? '' : q{AND key_name <> 'sftp'};
- my $and_dav = ENABLE_WEBDAV_MOVER ? '' : q{AND key_name <> 'webdav'};
+ my $and_sftp = ENABLE_SFTP_MOVER ? '' : q{AND key_name <> 'sftp'};
+ my $and_s3 = ENABLE_S3_MOVER ? '' : q{AND key_name <> 's3'};
+ my $and_dav = ENABLE_WEBDAV_MOVER ? '' : q{AND key_name <> 'webdav'};
my $sel = prepare_ca(qq{
SELECT disp_name
FROM class
- WHERE distributor = '1' $and_sftp $and_dav
+ WHERE distributor = '1' $and_sftp $and_s3 $and_dav
ORDER BY disp_name
}, undef);
return wantarray ? @{ col_aref($sel) } : col_aref($sel);
View
1  lib/Bric/Util/Trans/FS.pm
@@ -101,6 +101,7 @@ my $osen = { mac => 'MacOS',
msdos => 'MSDOS',
amiga => 'AmigaOS',
amigaos => 'AmigaOS',
+ awss3 => 'AWSS3',
};
################################################################################
View
258 lib/Bric/Util/Trans/S3.pm
@@ -0,0 +1,258 @@
+package Bric::Util::Trans::S3;
+
+=head1 Name
+
+Bric::Util::Trans::S3 - AWS S3 Client interface for distributing resources.
+
+=cut
+
+# Grab the Version Number.
+require Bric; our $VERSION = Bric->VERSION;
+
+=head1 Synopsis
+
+ use Bric::Util::Trans::S3
+
+=head1 Description
+
+The distribution API uses this class to distribute resources to AWS S3 buckets.
+
+=cut
+
+################################################################################
+# Dependencies
+################################################################################
+# Standard Dependencies
+use strict;
+use Net::Amazon::S3;
+
+################################################################################
+# Programmatic Dependences
+use Bric::Util::Fault qw(throw_gen);
+use Bric::Util::Trans::FS;
+use Bric::Config qw(:dist);
+use Bric::Util::ApacheUtil qw(unescape_uri escape_uri);
+
+################################################################################
+# Inheritance
+################################################################################
+use base qw(Bric);
+
+################################################################################
+# Function and Closure Prototypes
+################################################################################
+
+################################################################################
+# Constants
+################################################################################
+use constant DEBUG => 0;
+
+################################################################################
+# Fields
+################################################################################
+# Public Class Fields
+
+################################################################################
+# Private Class Fields
+my $fs = Bric::Util::Trans::FS->new;
+
+################################################################################
+
+################################################################################
+# Instance Fields
+BEGIN { Bric::register_fields() }
+
+################################################################################
+# Class Methods
+################################################################################
+
+=head1 Interface
+
+=head2 Constructors
+
+NONE.
+
+=head2 Destructors
+
+NONE.
+
+=head2 Public Class Methods
+
+NONE.
+
+=head2 Public Instance Methods
+
+NONE.
+
+=head2 Public Functions
+
+=over
+
+=item my $bool = Bric::Util::Trans::S3->put_res($resources, $st)
+
+Puts the files specified by the $job object into the S3 bucket specified by $st.
+
+B<Throws:>
+
+=over 4
+
+=item *
+
+Unable to connect to AWS API.
+
+=item *
+
+Unable to login to AWS account.
+
+=item *
+
+Unable to set permissions on bucket.
+
+=item *
+
+Unable to put resources into bucket.
+
+=back
+
+B<Side Effects:> NONE.
+
+B<Notes:>
+
+=cut
+
+sub put_res {
+ my ($pkg, $resources, $st) = @_;
+ foreach my $s ($st->get_servers) {
+ # Skip inactive servers.
+ next unless $s->is_active;
+ my $s3 = Net::Amazon::S3->new({
+ aws_access_key_id => $s->get_login,
+ aws_secret_access_key => $s->get_password,
+ retry => 1,
+ }) or
+ throw_gen
+ error => 'fail',
+ payload => 'login: ' . $s->get_login . "\n" . 'password: ' . $s->get_password;
+ # Instantiate the S3 bucket
+ my $bucket = $s3->bucket($s->get_host_name);
+ foreach my $res (@$resources) {
+ # Get the source and destination paths for the resource.
+ my $src = $res->get_tmp_path || $res->get_path;
+ my $dest = $s->get_doc_root . substr $res->get_uri, 1;
+ my $mime = $res->get_media_type;
+ #throw_gen error => 'fail', payload => "src: $src\ndest: $dest\nmime: $mime";
+ # store a file in the bucket
+ $bucket->add_key_filename( $dest, $src,
+ { content_type => $mime, })
+ or
+ throw_gen
+ error => $s3->err,
+ payload => 'src: ' . $src . "\ndest: " . $dest . ' * ' . $s3->errstr;
+ # set the ACL so everyone can see the file
+ $bucket->set_acl({'acl_short' => 'public-read', 'key' => $dest})
+ or throw_gen
+ error => $s3->err,
+ payload => $s3->errstr;
+
+ }
+ return 1;
+ }
+}
+
+################################################################################
+
+=item my $bool = Bric::Util::Trans::S3->del_res($resources, $st)
+
+Deletes the files specified by the $job object from the servers specified by
+$st.
+
+B<Throws:>
+
+=over 4
+
+=item *
+
+Unable to connect to remote server.
+
+=item *
+
+Unable to login to remote server.
+
+=item *
+
+Unable to delete resource from remote server.
+
+=item *
+
+Unable to properly close connection to remote server.
+
+=back
+
+B<Side Effects:> NONE.
+
+B<Notes:> See put_res(), above.
+
+=cut
+
+sub del_res {
+ my ($pkg, $resources, $st) = @_;
+ foreach my $s ($st->get_servers) {
+ # Skip inactive servers.
+ next unless $s->is_active;
+ my $s3 = Net::Amazon::S3->new({
+ aws_access_key_id => $s->get_login,
+ aws_secret_access_key => $s->get_password,
+ retry => 1,
+ });
+ # Instantiate the S3 bucket
+ my $bucket = $s3->bucket($s->get_host_name);
+ foreach my $res (@$resources) {
+ # Get the name of the file to be deleted.
+ my $target = $s->get_doc_root . $res->get_uri;
+ $bucket->delete_key($target)
+ or throw_gen
+ error => $s3->err,
+ payload => $s3->errstr;
+
+ }
+ }
+ return 1;
+}
+
+=back
+
+=head1 Private
+
+=head2 Private Class Methods
+
+NONE.
+
+=head2 Private Instance Methods
+
+NONE.
+
+=head2 Private Functions
+
+NONE.
+
+=cut
+
+1;
+__END__
+
+=head1 Notes
+
+NONE.
+
+=head1 Author
+
+John Durkin E<lt>john@j2d3.comE<gt>
+
+=head1 See Also
+
+L<Bric|Bric>,
+L<Bric::Dist::Action|Bric::Dist::Action>,
+L<Bric::Dist::Action::Mover|Bric::Dist::Action::Mover>,
+L<Bric::Util::Trans::FS|Bric::Util::Trans::FS>
+
+=cut
View
4 sql/Pg/Bric/Util/Class.val
@@ -256,3 +256,7 @@ VALUES ('80', 'pub_job', 'Bric::Util::Job::Pub', 'Publication Job', 'Publication
INSERT INTO class (id, key_name, pkg_name, disp_name, plural_name, description, distributor)
VALUES ('81', 'element', 'Bric::Biz::Element', 'Element', 'Elements', 'Element objects.', '0');
+
+INSERT INTO class (id, key_name, pkg_name, disp_name, plural_name, description, distributor)
+VALUES ('82', 's3', 'Bric::Util::Trans::S3', 'Amazon S3', 'Amazon S3 Transport', 'Class with methods to move files via Amazon S3.', '1');
+
View
4 sql/mysql/Bric/Util/Class.val
@@ -256,3 +256,7 @@ VALUES ('80', 'pub_job', 'Bric::Util::Job::Pub', 'Publication Job', 'Publication
INSERT INTO class (id, key_name, pkg_name, disp_name, plural_name, description, distributor)
VALUES ('81', 'element', 'Bric::Biz::Element', 'Element', 'Elements', 'Element objects.', '0');
+
+INSERT INTO class (id, key_name, pkg_name, disp_name, plural_name, description, distributor)
+VALUES ('82', 's3', 'Bric::Util::Trans::S3', 'Amazon S3', 'Amazon S3 Transport', 'Class with methods to move files via Amazon S3.', '1');
+
View
23 t/Bric/Util/Trans/S3/Test.pm
@@ -0,0 +1,23 @@
+package Bric::Util::Trans::S3::Test;
+use strict;
+use warnings;
+use base qw(Bric::Test::Base);
+use Test::More;
+
+##############################################################################
+# Test class loading.
+##############################################################################
+sub _test_load : Test(3) {
+ eval { require Net::Amazon::S3; 1 } or
+ return 'Net::Amazon::S3 not installed';
+ use_ok('Bric::Util::Trans::S3');
+ isa_ok 'Bric::Util::Trans::S3', 'Bric';
+ can_ok 'Bric::Util::Trans::S3', qw(
+ new
+ put_res
+ del_res
+ );
+}
+
+1;
+__END__
Please sign in to comment.
Something went wrong with that request. Please try again.