Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

More s3 nodes #38

Merged
merged 4 commits into from
Oct 21, 2014
Merged

More s3 nodes #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5a72bad
Fix missing credentials error case.
hindessm Oct 15, 2014
fd5011e
Add amazon icon.
hindessm Oct 9, 2014
08388ac
Add Amazon S3 query node.
hindessm Oct 13, 2014
e819829
Add S3 watch node.
hindessm Oct 13, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added aws/icons/amazon.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
92 changes: 92 additions & 0 deletions aws/s3.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,96 @@
limitations under the License.
-->

<script type="text/x-red" data-template-name="amazon s3 in">
<div class="form-row">
<label for="node-input-aws"><i class="fa fa-user"></i> AWS</label>
<input type="text" id="node-input-aws">
</div>
<div class="form-row">
<label for="node-input-bucket"><i class="fa fa-folder"></i> Bucket</label>
<input type="text" id="node-input-bucket" placeholder="bucket name">
</div>
<div class="form-row node-input-filepattern">
<label for="node-input-filepattern"><i class="fa fa-file"></i> Filename Pattern</label>
<input type="text" id="node-input-filepattern" placeholder="Filepattern">
</div>
<div class="form-row">
<label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
<input type="text" id="node-input-name" placeholder="Name">
</div>
</script>

<script type="text/x-red" data-help-name="amazon s3 in">
<p>Amazon S3 watch node. Watches for file events on Box. By default all
file events are reported, but the filename pattern can be supplied
to limit the events to files which have full filenames that match
the glob pattern. The event messages consist of the full filename
in <b>msg.payload</b> property, the filename in <b>msg.file</b>,
the event type in <b>msg.event</b>.</p>
</script>

<script type="text/javascript">
RED.nodes.registerType('amazon s3 in',{
category: 'storage-input',
color:"#C0DEED",
defaults: {
aws: {type:"aws-config",required:true},
bucket: {required:true},
filepattern: {value:""},
name: {value:""},
},
inputs:0,
outputs:1,
icon: "amazon.png",
label: function() {
return this.bucket ? "s3 "+this.bucket : "s3";
}
});
</script>

<script type="text/x-red" data-template-name="amazon s3">
<div class="form-row">
<label for="node-input-aws"><i class="fa fa-user"></i> AWS</label>
<input type="text" id="node-input-aws">
</div>
<div class="form-row">
<label for="node-input-bucket"><i class="fa fa-folder"></i> Bucket</label>
<input type="text" id="node-input-bucket" placeholder="bucket name">
</div>
<div class="form-row node-input-filename">
<label for="node-input-filename"><i class="fa fa-file"></i> Filename</label>
<input type="text" id="node-input-filename" placeholder="Filename">
</div>
<div class="form-row">
<label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
<input type="text" id="node-input-name" placeholder="Name">
</div>
</script>

<script type="text/x-red" data-help-name="amazon s3">
<p>Amazon S3 input node. Downloads content from an Amazon S3 bucket. The bucket name can be specified in the node <b>bucket</b> property or in the <b>msg.bucket</b> property. The name of the file to download is taken from the node <b>filename</b> property or the <b>msg.filename</b> property. The downloaded content is sent as <b>msg.payload</b> property. If the download fails <b>msg.error</b> will contain an error object.</p>
</script>

<script type="text/javascript">
RED.nodes.registerType('amazon s3',{
category: 'storage-output',
color:"#C0DEED",
defaults: {
aws: {type:"aws-config",required:true},
bucket: {required:true},
filename: {value:""},
name: {value:""},
},
inputs:1,
outputs:1,
icon: "amazon.png",
align: "right",
label: function() {
return this.bucket ? "s3 "+this.bucket : "s3";
}
});
</script>

<script type="text/x-red" data-template-name="amazon s3 out">
<div class="form-row">
<label for="node-input-aws"><i class="fa fa-user"></i> AWS</label>
Expand Down Expand Up @@ -50,9 +140,11 @@
bucket: {required:true},
filename: {value:""},
localFilename: {value:""},
name: {value:""},
},
inputs:1,
outputs:0,
icon: "amazon.png",
align: "right",
label: function() {
return this.bucket ? "s3 "+this.bucket : "s3";
Expand Down
140 changes: 135 additions & 5 deletions aws/s3.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,132 @@
module.exports = function(RED) {
"use strict";
var fs = require('fs');
var minimatch = require("minimatch");

function AmazonS3InNode(n) {
RED.nodes.createNode(this,n);
this.awsConfig = RED.nodes.getNode(n.aws);
this.region = n.region;
this.bucket = n.bucket;
this.filepattern = n.filepattern || "";
var node = this;
var AWS = this.awsConfig ? this.awsConfig.AWS : null;
if (!AWS) {
node.warn("Missing AWS credentials");
return;
}
var s3 = new AWS.S3();
node.status({fill:"blue",shape:"dot",text:"initializing"});
s3.listObjects({ Bucket: node.bucket }, function(err, data) {
if (err) {
node.error("failed to fetch S3 state: " + err);
node.status({fill:"red",shape:"ring",text:"error"});
return;
}
node.state = data.Contents.filter(function (e) {
return !node.filepattern || minimatch(e, node.filepattern);
}).map(function (e) {
return e.Key;
});
node.status({});
node.on("input", function(msg) {
node.status({fill:"blue",shape:"dot",text:"checking for changes"});
s3.listObjects({ Bucket: node.bucket }, function(err, data) {
if (err) {
node.warn("failed to fetch S3 state: " + err);
node.status({});
return;
}
node.status({});
var newState = data.Contents.filter(function (e) {
return !node.filepattern ||
minimatch(e, node.filepattern);
}).map(function (e) {
return e.Key;
});
var seen = {};
var i;
for (i = 0; i < node.state.length; i++) {
seen[node.state[i]] = true;
}
for (i = 0; i < newState.length; i++) {
if (seen[newState[i]]) {
delete seen[newState[i]];
} else {
msg.payload = newState[i];
msg.file = newState[i].substring(newState[i].lastIndexOf('/') + 1);
msg.event = 'add';
node.send(msg);
}
}
for (var k in seen) {
if (seen.hasOwnProperty(k)) {
msg.payload = k;
msg.file = k.substring(k.lastIndexOf('/') + 1);
msg.event = 'delete';
node.send(msg);
}
}
node.state = newState;
});
});
var interval = setInterval(function() {
node.emit("input", {});
}, 900000); // 15 minutes
node.on("close", function() {
if (interval !== null) {
clearInterval(interval);
}
});
});
}
RED.nodes.registerType("amazon s3 in", AmazonS3InNode);

function AmazonS3QueryNode(n) {
RED.nodes.createNode(this,n);
this.awsConfig = RED.nodes.getNode(n.aws);
this.region = n.region;
this.bucket = n.bucket;
this.filename = n.filename || "";
var node = this;
var AWS = this.awsConfig ? this.awsConfig.AWS : null;
if (!AWS) {
node.warn("Missing AWS credentials");
return;
}
var s3 = new AWS.S3();
node.on("input", function(msg) {
var bucket = node.bucket || msg.bucket;
if (bucket === "") {
node.warn("No bucket specified");
return;
}
var filename = node.filename || msg.filename;
if (filename === "") {
node.warn("No filename specified");
return;
}
msg.bucket = bucket;
msg.filename = filename;
node.status({fill:"blue",shape:"dot",text:"downloading"});
s3.getObject({
Bucket: bucket,
Key: filename,
}, function(err, data) {
if (err) {
node.warn("download failed " + err.toString());
delete msg.payload;
msg.error = err;
} else {
msg.payload = data.Body;
delete msg.error;
}
node.status({});
node.send(msg);
});
});
}
RED.nodes.registerType("amazon s3", AmazonS3QueryNode);

function AmazonS3OutNode(n) {
RED.nodes.createNode(this,n);
Expand All @@ -26,29 +152,33 @@ module.exports = function(RED) {
this.filename = n.filename || "";
this.localFilename = n.localFilename || "";
var node = this;
var AWS = this.awsConfig.AWS;
var AWS = this.awsConfig ? this.awsConfig.AWS : null;
if (!AWS) {
node.warn("Missing AWS credentials");
return;
}
if (AWS) {
var s3 = new AWS.S3();
node.status({fill:"blue",shape:"dot",text:"checking credentials"});
s3.listObjects({ Bucket: this.bucket }, function(err) {
s3.listObjects({ Bucket: node.bucket }, function(err) {
if (err) {
node.error("AWS S3 error: " + err);
node.status({fill:"red",shape:"ring",text:"error"});
return;
}
node.status({});
node.on("input", function(msg) {
var bucket = this.bucket || msg.bucket;
var bucket = node.bucket || msg.bucket;
if (bucket === "") {
node.warn("No bucket specified");
return;
}
var filename = this.filename || msg.filename;
var filename = node.filename || msg.filename;
if (filename === "") {
node.warn("No filename specified");
return;
}
var localFilename = this.localFilename || msg.localFilename;
var localFilename = node.localFilename || msg.localFilename;
if (localFilename) {
// TODO: use chunked upload for large files
node.status({fill:"blue",shape:"dot",text:"uploading"});
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"dependencies" : {
"aws-sdk": "^2.0.15",
"dropbox":"^0.10.3",
"minimatch": "^1.0.0",
"oauth":"~0.9.11",
"request":"~2.40.0",
"instagram-node":"0.5.1",
Expand Down