Skip to content

Commit

Permalink
Merge pull request #38 from hindessm/more-s3-nodes
Browse files Browse the repository at this point in the history 
More s3 nodes
  • Loading branch information
@knolleary
knolleary committed Oct 21, 2014
2 parents f120d3b + e819829 commit 49cfa86
Show file tree
Hide file tree
Showing 4 changed files with 228 additions and 5 deletions.
Binary file added aws/icons/amazon.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
92 changes: 92 additions & 0 deletions aws/s3.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,96 @@
limitations under the License.
-->

<script type="text/x-red" data-template-name="amazon s3 in">
<div class="form-row">
<label for="node-input-aws"><i class="fa fa-user"></i> AWS</label>
<input type="text" id="node-input-aws">
</div>
<div class="form-row">
<label for="node-input-bucket"><i class="fa fa-folder"></i> Bucket</label>
<input type="text" id="node-input-bucket" placeholder="bucket name">
</div>
<div class="form-row node-input-filepattern">
<label for="node-input-filepattern"><i class="fa fa-file"></i> Filename Pattern</label>
<input type="text" id="node-input-filepattern" placeholder="Filepattern">
</div>
<div class="form-row">
<label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
<input type="text" id="node-input-name" placeholder="Name">
</div>
</script>

<script type="text/x-red" data-help-name="amazon s3 in">
<p>Amazon S3 watch node. Watches for file events on Box. By default all
file events are reported, but the filename pattern can be supplied
to limit the events to files which have full filenames that match
the glob pattern. The event messages consist of the full filename
in <b>msg.payload</b> property, the filename in <b>msg.file</b>,
the event type in <b>msg.event</b>.</p>
</script>

<script type="text/javascript">
RED.nodes.registerType('amazon s3 in',{
category: 'storage-input',
color:"#C0DEED",
defaults: {
aws: {type:"aws-config",required:true},
bucket: {required:true},
filepattern: {value:""},
name: {value:""},
},
inputs:0,
outputs:1,
icon: "amazon.png",
label: function() {
return this.bucket ? "s3 "+this.bucket : "s3";
}
});
</script>

<script type="text/x-red" data-template-name="amazon s3">
<div class="form-row">
<label for="node-input-aws"><i class="fa fa-user"></i> AWS</label>
<input type="text" id="node-input-aws">
</div>
<div class="form-row">
<label for="node-input-bucket"><i class="fa fa-folder"></i> Bucket</label>
<input type="text" id="node-input-bucket" placeholder="bucket name">
</div>
<div class="form-row node-input-filename">
<label for="node-input-filename"><i class="fa fa-file"></i> Filename</label>
<input type="text" id="node-input-filename" placeholder="Filename">
</div>
<div class="form-row">
<label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
<input type="text" id="node-input-name" placeholder="Name">
</div>
</script>

<script type="text/x-red" data-help-name="amazon s3">
<p>Amazon S3 input node. Downloads content from an Amazon S3 bucket. The bucket name can be specified in the node <b>bucket</b> property or in the <b>msg.bucket</b> property. The name of the file to download is taken from the node <b>filename</b> property or the <b>msg.filename</b> property. The downloaded content is sent as <b>msg.payload</b> property. If the download fails <b>msg.error</b> will contain an error object.</p>
</script>

<script type="text/javascript">
RED.nodes.registerType('amazon s3',{
category: 'storage-output',
color:"#C0DEED",
defaults: {
aws: {type:"aws-config",required:true},
bucket: {required:true},
filename: {value:""},
name: {value:""},
},
inputs:1,
outputs:1,
icon: "amazon.png",
align: "right",
label: function() {
return this.bucket ? "s3 "+this.bucket : "s3";
}
});
</script>

<script type="text/x-red" data-template-name="amazon s3 out">
<div class="form-row">
<label for="node-input-aws"><i class="fa fa-user"></i> AWS</label>
Expand Down Expand Up @@ -50,9 +140,11 @@
bucket: {required:true},
filename: {value:""},
localFilename: {value:""},
name: {value:""},
},
inputs:1,
outputs:0,
icon: "amazon.png",
align: "right",
label: function() {
return this.bucket ? "s3 "+this.bucket : "s3";
Expand Down
140 changes: 135 additions & 5 deletions aws/s3.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,132 @@
module.exports = function(RED) {
"use strict";
var fs = require('fs');
var minimatch = require("minimatch");

function AmazonS3InNode(n) {
RED.nodes.createNode(this,n);
this.awsConfig = RED.nodes.getNode(n.aws);
this.region = n.region;
this.bucket = n.bucket;
this.filepattern = n.filepattern || "";
var node = this;
var AWS = this.awsConfig ? this.awsConfig.AWS : null;
if (!AWS) {
node.warn("Missing AWS credentials");
return;
}
var s3 = new AWS.S3();
node.status({fill:"blue",shape:"dot",text:"initializing"});
s3.listObjects({ Bucket: node.bucket }, function(err, data) {
if (err) {
node.error("failed to fetch S3 state: " + err);
node.status({fill:"red",shape:"ring",text:"error"});
return;
}
node.state = data.Contents.filter(function (e) {
return !node.filepattern || minimatch(e, node.filepattern);
}).map(function (e) {
return e.Key;
});
node.status({});
node.on("input", function(msg) {
node.status({fill:"blue",shape:"dot",text:"checking for changes"});
s3.listObjects({ Bucket: node.bucket }, function(err, data) {
if (err) {
node.warn("failed to fetch S3 state: " + err);
node.status({});
return;
}
node.status({});
var newState = data.Contents.filter(function (e) {
return !node.filepattern ||
minimatch(e, node.filepattern);
}).map(function (e) {
return e.Key;
});
var seen = {};
var i;
for (i = 0; i < node.state.length; i++) {
seen[node.state[i]] = true;
}
for (i = 0; i < newState.length; i++) {
if (seen[newState[i]]) {
delete seen[newState[i]];
} else {
msg.payload = newState[i];
msg.file = newState[i].substring(newState[i].lastIndexOf('/') + 1);
msg.event = 'add';
node.send(msg);
}
}
for (var k in seen) {
if (seen.hasOwnProperty(k)) {
msg.payload = k;
msg.file = k.substring(k.lastIndexOf('/') + 1);
msg.event = 'delete';
node.send(msg);
}
}
node.state = newState;
});
});
var interval = setInterval(function() {
node.emit("input", {});
}, 900000); // 15 minutes
node.on("close", function() {
if (interval !== null) {
clearInterval(interval);
}
});
});
}
RED.nodes.registerType("amazon s3 in", AmazonS3InNode);

function AmazonS3QueryNode(n) {
RED.nodes.createNode(this,n);
this.awsConfig = RED.nodes.getNode(n.aws);
this.region = n.region;
this.bucket = n.bucket;
this.filename = n.filename || "";
var node = this;
var AWS = this.awsConfig ? this.awsConfig.AWS : null;
if (!AWS) {
node.warn("Missing AWS credentials");
return;
}
var s3 = new AWS.S3();
node.on("input", function(msg) {
var bucket = node.bucket || msg.bucket;
if (bucket === "") {
node.warn("No bucket specified");
return;
}
var filename = node.filename || msg.filename;
if (filename === "") {
node.warn("No filename specified");
return;
}
msg.bucket = bucket;
msg.filename = filename;
node.status({fill:"blue",shape:"dot",text:"downloading"});
s3.getObject({
Bucket: bucket,
Key: filename,
}, function(err, data) {
if (err) {
node.warn("download failed " + err.toString());
delete msg.payload;
msg.error = err;
} else {
msg.payload = data.Body;
delete msg.error;
}
node.status({});
node.send(msg);
});
});
}
RED.nodes.registerType("amazon s3", AmazonS3QueryNode);

function AmazonS3OutNode(n) {
RED.nodes.createNode(this,n);
Expand All @@ -26,29 +152,33 @@ module.exports = function(RED) {
this.filename = n.filename || "";
this.localFilename = n.localFilename || "";
var node = this;
var AWS = this.awsConfig.AWS;
var AWS = this.awsConfig ? this.awsConfig.AWS : null;
if (!AWS) {
node.warn("Missing AWS credentials");
return;
}
if (AWS) {
var s3 = new AWS.S3();
node.status({fill:"blue",shape:"dot",text:"checking credentials"});
s3.listObjects({ Bucket: this.bucket }, function(err) {
s3.listObjects({ Bucket: node.bucket }, function(err) {
if (err) {
node.error("AWS S3 error: " + err);
node.status({fill:"red",shape:"ring",text:"error"});
return;
}
node.status({});
node.on("input", function(msg) {
var bucket = this.bucket || msg.bucket;
var bucket = node.bucket || msg.bucket;
if (bucket === "") {
node.warn("No bucket specified");
return;
}
var filename = this.filename || msg.filename;
var filename = node.filename || msg.filename;
if (filename === "") {
node.warn("No filename specified");
return;
}
var localFilename = this.localFilename || msg.localFilename;
var localFilename = node.localFilename || msg.localFilename;
if (localFilename) {
// TODO: use chunked upload for large files
node.status({fill:"blue",shape:"dot",text:"uploading"});
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"dependencies" : {
"aws-sdk": "^2.0.15",
"dropbox":"^0.10.3",
"minimatch": "^1.0.0",
"oauth":"~0.9.11",
"request":"~2.40.0",
"instagram-node":"0.5.1",
Expand Down

0 comments on commit 49cfa86

Please sign in to comment.