forked from gruntwork-io/terratest
-
Notifications
You must be signed in to change notification settings - Fork 0
/
ec2-files.go
234 lines (198 loc) · 10.8 KB
/
ec2-files.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
package aws
import (
"os"
"path/filepath"
"testing"
"github.com/gruntwork-io/terratest/modules/customerrors"
"github.com/gruntwork-io/terratest/modules/files"
"github.com/gruntwork-io/terratest/modules/ssh"
)
type RemoteFileSpecification struct {
AsgNames []string //ASGs where our instances will be
RemotePathToFileFilter map[string][]string //A map of the files to fetch, where the keys are directories on the remote host and the values are filters for what files to fetch from the directory. The filters support bash-style wildcards.
UseSudo bool
SshUser string
KeyPair *Ec2Keypair
LocalDestinationDir string //base path where to store downloaded artifacts locally. The final path of each resource will include the ip of the host and the name of the immediate parent folder.
}
// FetchContentsOfFileFromInstance looks up the public IP address of the EC2 Instance with the given ID, connects to
// the Instance via SSH using the given username and Key Pair, fetches the contents of the file at the given path
// (using sudo if useSudo is true), and returns the contents of that file as a string.
func FetchContentsOfFileFromInstance(t *testing.T, awsRegion string, sshUserName string, keyPair *Ec2Keypair, instanceID string, useSudo bool, filePath string) string {
out, err := FetchContentsOfFileFromInstanceE(t, awsRegion, sshUserName, keyPair, instanceID, useSudo, filePath)
if err != nil {
t.Fatal(err)
}
return out
}
// FetchContentsOfFileFromInstanceE looks up the public IP address of the EC2 Instance with the given ID, connects to
// the Instance via SSH using the given username and Key Pair, fetches the contents of the file at the given path
// (using sudo if useSudo is true), and returns the contents of that file as a string.
func FetchContentsOfFileFromInstanceE(t *testing.T, awsRegion string, sshUserName string, keyPair *Ec2Keypair, instanceID string, useSudo bool, filePath string) (string, error) {
publicIp, err := GetPublicIpOfEc2InstanceE(t, instanceID, awsRegion)
if err != nil {
return "", err
}
host := ssh.Host{
SshUserName: sshUserName,
SshKeyPair: keyPair.KeyPair,
Hostname: publicIp,
}
return ssh.FetchContentsOfFileE(t, host, useSudo, filePath)
}
// FetchContentsOfFilesFromInstance looks up the public IP address of the EC2 Instance with the given ID, connects to
// the Instance via SSH using the given username and Key Pair, fetches the contents of the files at the given paths
// (using sudo if useSudo is true), and returns a map from file path to the contents of that file as a string.
func FetchContentsOfFilesFromInstance(t *testing.T, awsRegion string, sshUserName string, keyPair *Ec2Keypair, instanceID string, useSudo bool, filePaths ...string) map[string]string {
out, err := FetchContentsOfFilesFromInstanceE(t, awsRegion, sshUserName, keyPair, instanceID, useSudo, filePaths...)
if err != nil {
t.Fatal(err)
}
return out
}
// FetchContentsOfFilesFromInstanceE looks up the public IP address of the EC2 Instance with the given ID, connects to
// the Instance via SSH using the given username and Key Pair, fetches the contents of the files at the given paths
// (using sudo if useSudo is true), and returns a map from file path to the contents of that file as a string.
func FetchContentsOfFilesFromInstanceE(t *testing.T, awsRegion string, sshUserName string, keyPair *Ec2Keypair, instanceID string, useSudo bool, filePaths ...string) (map[string]string, error) {
publicIp, err := GetPublicIpOfEc2InstanceE(t, instanceID, awsRegion)
if err != nil {
return nil, err
}
host := ssh.Host{
SshUserName: sshUserName,
SshKeyPair: keyPair.KeyPair,
Hostname: publicIp,
}
return ssh.FetchContentsOfFilesE(t, host, useSudo, filePaths...)
}
// FetchContentsOfFileFromAsg looks up the EC2 Instances in the given ASG, looks up the public IPs of those EC2
// Instances, connects to each Instance via SSH using the given username and Key Pair, fetches the contents of the file
// at the given path (using sudo if useSudo is true), and returns a map from Instance ID to the contents of that file
// as a string.
func FetchContentsOfFileFromAsg(t *testing.T, awsRegion string, sshUserName string, keyPair *Ec2Keypair, asgName string, useSudo bool, filePath string) map[string]string {
out, err := FetchContentsOfFileFromAsgE(t, awsRegion, sshUserName, keyPair, asgName, useSudo, filePath)
if err != nil {
t.Fatal(err)
}
return out
}
// FetchContentsOfFileFromAsgE looks up the EC2 Instances in the given ASG, looks up the public IPs of those EC2
// Instances, connects to each Instance via SSH using the given username and Key Pair, fetches the contents of the file
// at the given path (using sudo if useSudo is true), and returns a map from Instance ID to the contents of that file
// as a string.
func FetchContentsOfFileFromAsgE(t *testing.T, awsRegion string, sshUserName string, keyPair *Ec2Keypair, asgName string, useSudo bool, filePath string) (map[string]string, error) {
instanceIDs, err := GetInstanceIdsForAsgE(t, asgName, awsRegion)
if err != nil {
return nil, err
}
instanceIdToContents := map[string]string{}
for _, instanceID := range instanceIDs {
contents, err := FetchContentsOfFileFromInstanceE(t, awsRegion, sshUserName, keyPair, instanceID, useSudo, filePath)
if err != nil {
return nil, err
}
instanceIdToContents[instanceID] = contents
}
return instanceIdToContents, err
}
// FetchContentsOfFilesFromAsg looks up the EC2 Instances in the given ASG, looks up the public IPs of those EC2
// Instances, connects to each Instance via SSH using the given username and Key Pair, fetches the contents of the files
// at the given paths (using sudo if useSudo is true), and returns a map from Instance ID to a map of file path to the
// contents of that file as a string.
func FetchContentsOfFilesFromAsg(t *testing.T, awsRegion string, sshUserName string, keyPair *Ec2Keypair, asgName string, useSudo bool, filePaths ...string) map[string]map[string]string {
out, err := FetchContentsOfFilesFromAsgE(t, awsRegion, sshUserName, keyPair, asgName, useSudo, filePaths...)
if err != nil {
t.Fatal(err)
}
return out
}
// FetchContentsOfFilesFromAsgE looks up the EC2 Instances in the given ASG, looks up the public IPs of those EC2
// Instances, connects to each Instance via SSH using the given username and Key Pair, fetches the contents of the files
// at the given paths (using sudo if useSudo is true), and returns a map from Instance ID to a map of file path to the
// contents of that file as a string.
func FetchContentsOfFilesFromAsgE(t *testing.T, awsRegion string, sshUserName string, keyPair *Ec2Keypair, asgName string, useSudo bool, filePaths ...string) (map[string]map[string]string, error) {
instanceIDs, err := GetInstanceIdsForAsgE(t, asgName, awsRegion)
if err != nil {
return nil, err
}
instanceIdToFilePathToContents := map[string]map[string]string{}
for _, instanceID := range instanceIDs {
contents, err := FetchContentsOfFilesFromInstanceE(t, awsRegion, sshUserName, keyPair, instanceID, useSudo, filePaths...)
if err != nil {
return nil, err
}
instanceIdToFilePathToContents[instanceID] = contents
}
return instanceIdToFilePathToContents, err
}
// FetchFilesFromInstance looks up the EC2 Instances in the given ASG, looks up the public IPs of those EC2
// Instances, connects to each Instance via SSH using the given username and Key Pair, downloads the files
// matching filenameFilters at the given remoteDirectory (using sudo if useSudo is true), and stores the files locally
// at localDirectory/<publicip>/<remoteFolderName>
func FetchFilesFromInstance(t *testing.T, awsRegion string, sshUserName string, keyPair *Ec2Keypair, instanceID string, useSudo bool, remoteDirectory string, localDirectory string, filenameFilters []string) {
err := FetchFilesFromInstanceE(t, awsRegion, sshUserName, keyPair, instanceID, useSudo, remoteDirectory, localDirectory, filenameFilters)
if err != nil {
t.Fatal(err)
}
}
// FetchFilesFromInstanceE looks up the EC2 Instances in the given ASG, looks up the public IPs of those EC2
// Instances, connects to each Instance via SSH using the given username and Key Pair, downloads the files
// matching filenameFilters at the given remoteDirectory (using sudo if useSudo is true), and stores the files locally
// at localDirectory/<publicip>/<remoteFolderName>
func FetchFilesFromInstanceE(t *testing.T, awsRegion string, sshUserName string, keyPair *Ec2Keypair, instanceID string, useSudo bool, remoteDirectory string, localDirectory string, filenameFilters []string) error {
publicIp, err := GetPublicIpOfEc2InstanceE(t, instanceID, awsRegion)
if err != nil {
return err
}
host := ssh.Host{
Hostname: publicIp,
SshUserName: sshUserName,
SshKeyPair: keyPair.KeyPair,
}
finalLocalDestDir := filepath.Join(localDirectory, publicIp, filepath.Base(remoteDirectory))
if !files.FileExists(finalLocalDestDir) {
os.MkdirAll(finalLocalDestDir, 0755)
}
scpOptions := ssh.ScpDownloadOptions{
RemoteHost: host,
RemoteDir: remoteDirectory,
LocalDir: finalLocalDestDir,
FileNameFilters: filenameFilters,
}
return ssh.ScpDirFromE(t, scpOptions, useSudo)
}
// FetchFilesFromAsgs looks up the EC2 Instances in all the ASGs given in the RemoteFileSpecification,
// looks up the public IPs of those EC2 Instances, connects to each Instance via SSH using the given
// username and Key Pair, downloads the files matching filenameFilters at the given
// remoteDirectory (using sudo if useSudo is true), and stores the files locally at
// localDirectory/<publicip>/<remoteFolderName>
func FetchFilesFromAsgs(t *testing.T, awsRegion string, spec RemoteFileSpecification) {
err := FetchFilesFromAsgsE(t, awsRegion, spec)
if err != nil {
t.Fatal(err)
}
}
// FetchFilesFromAsgsE looks up the EC2 Instances in all the ASGs given in the RemoteFileSpecification,
// looks up the public IPs of those EC2 Instances, connects to each Instance via SSH using the given
// username and Key Pair, downloads the files matching filenameFilters at the given
// remoteDirectory (using sudo if useSudo is true), and stores the files locally at
// localDirectory/<publicip>/<remoteFolderName>
func FetchFilesFromAsgsE(t *testing.T, awsRegion string, spec RemoteFileSpecification) error {
errorsOccurred := []error{}
for _, curAsg := range spec.AsgNames {
for curRemoteDir, fileFilters := range spec.RemotePathToFileFilter {
instanceIDs, err := GetInstanceIdsForAsgE(t, curAsg, awsRegion)
if err != nil {
errorsOccurred = append(errorsOccurred, err)
} else {
for _, instanceID := range instanceIDs {
err = FetchFilesFromInstanceE(t, awsRegion, spec.SshUser, spec.KeyPair, instanceID, spec.UseSudo, curRemoteDir, spec.LocalDestinationDir, fileFilters)
if err != nil {
errorsOccurred = append(errorsOccurred, err)
}
}
}
}
}
return customerrors.NewMultiError(errorsOccurred...)
}