-
Notifications
You must be signed in to change notification settings - Fork 2
/
S3 Download.js
187 lines (168 loc) · 6.15 KB
/
S3 Download.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
function timerFired( s : Switch )
{
// Get flow element properties
var intervalSeconds = s.getPropertyValue("checkIntervalMinutes")*60;
var targetBucket = s.getPropertyValue("targetBucket");
var leaveOriginals = s.getPropertyValue("leaveOriginals");
var region = s.getPropertyValue('region');
var namedProfile = s.getPropertyValue('namedProfile');
var prefix = s.getPropertyValue('prefix');
var CliPathPrefix = s.getPropertyValue('CliPathPrefix');
var debug = s.getPropertyValue("debug");
// Set the log level
var logLevel = 2;
// Set the timerInterval on start
if (s.getTimerInterval() == 0){
s.setTimerInterval(intervalSeconds);
}
// Log some stuff
if(debug == "Yes"){
s.log(logLevel, "Debug output");
s.log(logLevel, "intervalSeconds: "+intervalSeconds);
s.log(logLevel, "targetBucket: "+targetBucket);
s.log(logLevel, "leaveOriginals: "+leaveOriginals);
}
// Function for adding optional params
var addOptionalParameters = function(cmd, restrictTo){
if(region) cmd += " --region "+region;
if(namedProfile) cmd += " --profile "+namedProfile;
if(prefix && restrictTo == 'list-objects'){
cmd += " --prefix "+prefix;
}
// Booleans
return cmd;
}
// Function for explicitly calling Python
var addCliPathPrefix = function(cmd, CliPathPrefix)
{
fixedCmd = CliPathPrefix + cmd;
return fixedCmd;
}
// Function to see if AWS CLI is installed
var verifyAwsCli = function()
{
cmd = addCliPathPrefix("aws --version", CliPathPrefix);
Process.execute(cmd);
var awsVersionError = Process.stderr;
var awsVersionResponse = Process.stdout;
if(debug == 'Yes'){
s.log(logLevel, "aws version response: "+awsVersionResponse);
s.log(logLevel, "aws version error: "+awsVersionError);
}
if(!awsVersionError){
s.log(3, "AWS CLI does not appear to be installed!");
s.log(3, "You may have to set (or unset) the CLI Path Prefix element property so Switch may execute 'aws' commands.");
} else {
return true;
}
}
// Function to see if an S3 bucket exists and is accessible
var verifyS3Bucket = function(bucketName)
{
Process.execute(addCliPathPrefix("aws s3api head-bucket --bucket "+bucketName, CliPathPrefix));
var awsHeadBucketResponse = Process.stderr;
if(awsHeadBucketResponse){
s.log(3, awsHeadBucketResponse);
return false;
} else {
if(debug == 'Yes') s.log(logLevel, "Bucket '"+bucketName+"' exists and is available to you.");
return true;
}
}
// Function to list all objects in an S3 bucket
var listS3Objects = function(bucketName)
{
// Get list of S3 objects
list_cmd = addOptionalParameters(addCliPathPrefix("aws s3api list-objects --bucket "+targetBucket+" --output json", CliPathPrefix), 'list-objects');
if(debug == "Yes") s.log(2, 'list_cmd: '+list_cmd);
Process.execute(list_cmd);
var listObjectsResponse = Process.stdout;
// Evaluate response
var parsedObject = eval("(" + listObjectsResponse + ")");
var objects = parsedObject.Contents;
// Return
return objects;
}
// Function to get objects
var getS3Objects = function(inputObjects)
{
var i = null;
for(i = 0; i < inputObjects.length; i++){
key = inputObjects[i].Key;
size = inputObjects[i].Size;
basename_key = key.replace(/^.*[\\\/]/, '');
// Check for empty object AWS sometimes returns to maintain the "folder" structure on the S3 console
if(key == prefix || size === 0){
if(debug == "Yes"){ s.log(logLevel, "Empty 'folder' object found and ignored: "+key) };
continue;
};
// Log some stuff
if(debug == "Yes"){
s.log(logLevel, "Key: "+key);
s.log(logLevel, "Basename key: "+basename_key);
}
// Create a new job container
job = s.createNewJob(targetBucket+"_"+key);
fn = job.createPathWithName(basename_key, false);
// Get object metadata for private data writing
head_object_cmd = addOptionalParameters(addCliPathPrefix("aws s3api head-object --bucket "+targetBucket+" --key \""+key+"\" --output json", CliPathPrefix), 'No');
if(debug == "Yes") s.log(2, 'head_object_cmd: '+head_object_cmd);
Process.execute(head_object_cmd);
awsHeadBucketResponse = Process.stdout;
if(debug == "Yes") s.log(2, 'awsHeadBucketResponse: ' + awsHeadBucketResponse);
parsedHeadObject = eval("(" + awsHeadBucketResponse + ")");
metadata = parsedHeadObject.Metadata;
if(debug == "Yes") s.log(2, 'metadata found: '+metadata.toString());
// Set privat data for each metadata found
for (prop in metadata) {
job.setPrivateData("aws-metadata-" + prop, metadata[prop]);
}
// Invoke AWS CLI
download_cmd = addOptionalParameters(addCliPathPrefix("aws s3api get-object --bucket "+targetBucket+" --key \""+key+"\" \""+fn+"\" --output json", CliPathPrefix), 'No');
if(debug == "Yes") s.log(2, 'download_cmd: '+download_cmd);
Process.execute(download_cmd);
downloadError = Process.stderr;
// Evaluate response
if(downloadError){
s.log(3, "AWS CLI S3 download error: "+downloadError);
} else {
// Log some stuff
if(debug == "Yes") s.log(logLevel, "Object successfully downloaded : "+key);
// Complete job
job.sendToSingle(fn);
// Delete object
if(leaveOriginals == 'No'){
// Invoke
delete_cmd = addOptionalParameters(addCliPathPrefix("aws s3api delete-object --bucket "+targetBucket+" --key \""+key+"\" --output json", CliPathPrefix), 'No');
if(debug == "Yes") s.log(2, 'delete_cmd: '+delete_cmd);
Process.execute(delete_cmd);
deleteError = Process.stderr;
deleteResponse = Process.stdout;
// Log some stuff
if(debug == "Yes") s.log(logLevel, "deleteResponse : "+deleteResponse);
// Check for errors
if(deleteError){
s.log(3, "AWS CLI S3 delete error: "+deleteError);
} else {
if(debug == "Yes") s.log(logLevel, "File successfully removed from S3.");
}
}
}
}
}
// Ensure AWS CLI is installed
if(verifyAwsCli()){
// Verify S3 bucket
if(verifyS3Bucket(targetBucket)){
// List objects
var objects = listS3Objects(targetBucket);
// Check to see if any objects exist
if(objects.length > 0){
// Log some stuff
if(debug == "Yes")s.log(logLevel, "num of objects found: "+ objects.length);
// Get all objects
getS3Objects(objects);
}
}
}
}