-
Notifications
You must be signed in to change notification settings - Fork 0
/
test.go
66 lines (48 loc) · 1.27 KB
/
test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
package main
import (
"encoding/csv"
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"fmt"
"log"
"os"
"strings"
)
func main() {
// FETCH FILE FROM AWS ########################
// Uses example from https://stackoverflow.com/a/49270166
test_bucket := "pennsieve-prod-discover-publish-use1"
test_item := "2/2/metadata/records/Disease.csv"
sess, _ := session.NewSession(&aws.Config{
Region: aws.String("us-east-1")},
)
downloader := s3manager.NewDownloader(sess)
file, err := os.Create(test_item)
numBytes, err := downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(test_bucket),
Key: aws.String(test_item),
})
if err != nil {
log.Fatalf("Unable to download item %q, %v", test_item, err)
}
fmt.Println("Downloaded", file.Name(), numBytes, "bytes")
// READ CSV FILE ############################
r := csv.NewReader(strings.NewReader(file.Name()))
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
fmt.Println(record)
}
// PROCESS CSV ##############################
// DELIVER PROCESSED DATA ###################
// CLEAN UP LAMBDA ENVIRONMENT ##############
}