diff --git a/.gitignore b/.gitignore index 6b328f8f06f..b47ed3d919e 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ __snapshots__ /bin/callgraph /bin/chamber +/bin/ecs-service-logs /bin/soda /bin/golint /bin/swagger diff --git a/Gopkg.lock b/Gopkg.lock index 2f83b01872b..5b0f53ba225 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -17,6 +17,25 @@ pruneopts = "" revision = "d81462e38c2145023f9ecf5414fc84d45d5bfe82" +[[projects]] + branch = "master" + digest = "1:4caf528cc62b22f8902f9be351f2f506d8f1563990bbe75dabaf1339c2005cd4" + name = "github.com/99designs/aws-vault" + packages = [ + "prompt", + "vault", + ] + pruneopts = "" + revision = "305bcd142e1d76a9cffcd3bc7ebf14c265e6c871" + +[[projects]] + branch = "master" + digest = "1:a4608ef1abb93291a7c4e8360af19d5f493da1bcbac2c099a6f388d19ad2b1a5" + name = "github.com/99designs/keyring" + packages = ["."] + pruneopts = "" + revision = "82da6802f65f1ac7963cfc3b7c62ae12dab8ee5d" + [[projects]] digest = "1:e4b30804a381d7603b8a344009987c1ba351c26043501b23b8c7ce21f0b67474" name = "github.com/BurntSushi/toml" @@ -49,6 +68,14 @@ revision = "ccb8e960c48f04d6935e72476ae4a51028f9e22f" version = "v9" +[[projects]] + branch = "master" + digest = "1:70156739d3ae5ba381a7941aff2e8d1daf4a5e50719a569f2c6de6a6a81c9bd4" + name = "github.com/aulanov/go.dbus" + packages = ["."] + pruneopts = "" + revision = "25c3068a42a0b50b877953fb249dbcffc6bd1bca" + [[projects]] digest = "1:28d643e5dedba6b2c4d8b954fb60d4004cf1a097baa01d3645f7b958fc706fd3" name = "github.com/aws/aws-sdk-go" @@ -87,6 +114,9 @@ "private/protocol/rest", "private/protocol/restxml", "private/protocol/xml/xmlutil", + "service/cloudwatchlogs", + "service/ecs", + "service/iam", "service/s3", "service/s3/s3iface", "service/s3/s3manager", @@ -135,6 +165,14 @@ pruneopts = "" revision = "cafe2ce98974a3dcca6b92ce393a91a0b58b8133" +[[projects]] + digest = "1:a0a137c32515be7033c09be792a79b20d8c69e1891bf3f797e8ba83ecbb4b1fe" + name = "github.com/danieljoos/wincred" + packages = ["."] + pruneopts = "" + revision = "412b574fb496839b312a75fba146bd32a89001cf" + version = "v1.0.1" + [[projects]] digest = "1:0deddd908b6b4b768cfc272c16ee61e7088a60f7fe2f06c547bd3d8e1f8b8e77" name = "github.com/davecgh/go-spew" @@ -167,6 +205,23 @@ revision = "9f541cc9db5d55bce703bd99987c9d5cb8eea45e" version = "v1.0.0" +[[projects]] + digest = "1:ea315bb1ba6750dbec90b5f7bb6dab3c581904a4227b6a20d6b4b1dfc8285505" + name = "github.com/dvsekhvalnov/jose2go" + packages = [ + ".", + "aes", + "arrays", + "base64url", + "compact", + "kdf", + "keys/ecc", + "padding", + ] + pruneopts = "" + revision = "f21a8cedbbae609f623613ec8f81125c243212e6" + version = "v1.3" + [[projects]] branch = "master" digest = "1:88b37f144a80737f9e5cd50c887c00c2f3c7211257f884b8b80ce97e61ed1ccb" @@ -242,6 +297,14 @@ revision = "41f3572897373c5538c50a2402db15db079fa4fd" version = "2.0.0" +[[projects]] + digest = "1:ed30e45cd59986b6f98b4d191157a54050f5dd550e2cec47ad032b6230e4af08" + name = "github.com/go-ini/ini" + packages = ["."] + pruneopts = "" + revision = "6ed8d5f64cd79a498d1f3fab5880cc376ce41bbe" + version = "v1.41.0" + [[projects]] branch = "master" digest = "1:48f9a43c330434d61098a65bdf4d8cbfbdc0eecf30ab35b1b88405868df6a42b" @@ -302,6 +365,7 @@ name = "github.com/go-openapi/runtime" packages = [ ".", + "flagext", "logger", "middleware", "middleware/denco", @@ -581,6 +645,14 @@ pruneopts = "" revision = "14085ca3e1a995a72ac03700ee3e6b56706bda8e" +[[projects]] + digest = "1:cc1255e2fef3819bfab3540277001e602892dd431ef9ab5499bcdbc425923d64" + name = "github.com/godbus/dbus" + packages = ["."] + pruneopts = "" + revision = "2ff6f7ffd60f0f2410b3105864bdd12c7894f844" + version = "v5.0.1" + [[projects]] digest = "1:141cc9fc6279592458b304038bd16a05ef477d125c6dad281216345a11746fd7" name = "github.com/gofrs/uuid" @@ -648,6 +720,14 @@ revision = "e59506cc896acb7f7bf732d4fdf5e25f7ccd8983" version = "v1.1.1" +[[projects]] + branch = "master" + digest = "1:36f5bd9b89fcd728488eee3a0cf8787448579ac381fded60edd6e3ac01fe1a38" + name = "github.com/gsterjov/go-libsecret" + packages = ["."] + pruneopts = "" + revision = "a6f4afe4910cad8688db3e0e9b9ac92ad22d54e1" + [[projects]] digest = "1:8e3bd93036b4a925fe2250d3e4f38f21cadb8ef623561cd80c3c50c114b13201" name = "github.com/hashicorp/errwrap" @@ -796,6 +876,14 @@ pruneopts = "" revision = "95032a82bc518f77982ea72343cc1ade730072f0" +[[projects]] + branch = "master" + digest = "1:0f38f10cf33188908afc6541797c271b110a5d65385951b2c741380bdab56439" + name = "github.com/keybase/go-keychain" + packages = ["."] + pruneopts = "" + revision = "f1daa725cce4049b1715f1e97d6a51880e401e70" + [[projects]] digest = "1:765270f95ea68ad2150f6143eb8b9c0c17b038a7e2255b46580674471af00e27" name = "github.com/kisielk/gotool" @@ -948,6 +1036,14 @@ revision = "506f3da9b7c86d737e91f16b7431df8635871552" version = "v1.0.2" +[[projects]] + digest = "1:6dbb0eb72090871f2e58d1e37973fe3cb8c0f45f49459398d3fc740cb30e13bd" + name = "github.com/mitchellh/go-homedir" + packages = ["."] + pruneopts = "" + revision = "af06845cf3004701891bf4fdb884bfe4920b3727" + version = "v1.1.0" + [[projects]] digest = "1:bcc46a0fbd9e933087bef394871256b5c60269575bb661935874729c65bbbf60" name = "github.com/mitchellh/mapstructure" @@ -1321,6 +1417,7 @@ "html", "html/atom", "idna", + "netutil", ] pruneopts = "" revision = "d26f9f9a57f3fab6a695bec0d84433c2c50f8bbf" @@ -1471,9 +1568,15 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ + "github.com/99designs/aws-vault/prompt", + "github.com/99designs/aws-vault/vault", + "github.com/99designs/keyring", "github.com/aws/aws-sdk-go/aws", + "github.com/aws/aws-sdk-go/aws/credentials", "github.com/aws/aws-sdk-go/aws/endpoints", "github.com/aws/aws-sdk-go/aws/session", + "github.com/aws/aws-sdk-go/service/cloudwatchlogs", + "github.com/aws/aws-sdk-go/service/ecs", "github.com/aws/aws-sdk-go/service/s3", "github.com/aws/aws-sdk-go/service/s3/s3manager", "github.com/aws/aws-sdk-go/service/ses", @@ -1485,11 +1588,16 @@ "github.com/facebookgo/clock", "github.com/felixge/httpsnoop", "github.com/go-gomail/gomail", + "github.com/go-openapi/errors", "github.com/go-openapi/loads", "github.com/go-openapi/runtime", + "github.com/go-openapi/runtime/flagext", "github.com/go-openapi/runtime/middleware", + "github.com/go-openapi/runtime/security", + "github.com/go-openapi/spec", "github.com/go-openapi/strfmt", "github.com/go-openapi/swag", + "github.com/go-openapi/validate", "github.com/go-swagger/go-swagger/cmd/swagger", "github.com/gobuffalo/pop", "github.com/gobuffalo/pop/soda", @@ -1502,6 +1610,7 @@ "github.com/honeycombio/beeline-go", "github.com/honeycombio/beeline-go/wrappers/hnynethttp", "github.com/imdario/mergo", + "github.com/jessevdk/go-flags", "github.com/jmoiron/sqlx", "github.com/jung-kurt/gofpdf", "github.com/markbates/goth", @@ -1514,7 +1623,6 @@ "github.com/spf13/afero", "github.com/spf13/pflag", "github.com/spf13/viper", - "github.com/stretchr/objx", "github.com/stretchr/testify/assert", "github.com/stretchr/testify/mock", "github.com/stretchr/testify/suite", @@ -1527,6 +1635,7 @@ "goji.io", "goji.io/pat", "golang.org/x/crypto/bcrypt", + "golang.org/x/net/netutil", "golang.org/x/text/language", "golang.org/x/text/message", ] diff --git a/Gopkg.toml b/Gopkg.toml index 751101b788b..8bd40ac16cc 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -31,3 +31,7 @@ required = [ [[constraint]] name = "github.com/stretchr/objx" version = "0.1.1" + +[[constraint]] + name = "github.com/99designs/aws-vault" + branch = "master" diff --git a/Makefile b/Makefile index f218801d75d..de19b8bde6e 100644 --- a/Makefile +++ b/Makefile @@ -170,6 +170,7 @@ build_tools: server_deps server_generate go build -i -ldflags "$(LDFLAGS)" -o bin/make-tsp-user ./cmd/make_tsp_user go build -i -ldflags "$(LDFLAGS)" -o bin/paperwork ./cmd/paperwork go build -i -ldflags "$(LDFLAGS)" -o bin/tsp-award-queue ./cmd/tsp_award_queue + go build -i -ldflags "$(LDFLAGS)" -o bin/ecs-service-logs ./cmd/ecs-service-logs tsp_run: build_tools db_dev_run ./bin/tsp-award-queue diff --git a/bin/ecs-show-service-logs b/bin/ecs-show-service-logs index 982f72d854b..ed16c11cf13 100755 --- a/bin/ecs-show-service-logs +++ b/bin/ecs-show-service-logs @@ -3,6 +3,8 @@ # Show logs from the containers running for the named service. # set -eo pipefail +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +readonly DIR usage() { echo "$0 | less" @@ -11,27 +13,6 @@ usage() { [[ -z $1 || -z $2 ]] && usage set -u -readonly prefix="app" -readonly name=$1 -readonly environment=$2 -readonly cluster=$prefix-$environment +[[ -f "$DIR/ecs-service-logs" ]] || (echo "Missing bin/ecs-service-logs. Run make build_tools" && exit 1) -# awslogs-stream-prefix settings -readonly container=$name-$environment -readonly log_group_name=ecs-tasks-$name-$environment - -# Get list of running tasks -for task_arn in $(aws ecs list-tasks --cluster "$cluster" --service-name "$name" --query 'taskArns' | jq -r '.[]'); do - [[ -z $task_arn ]] && { echo "Missing task ARN"; exit 1; } - - # Parse out the task ID - task_id=$(echo "$task_arn" | perl -ne 'm|^arn:aws:ecs:([^:]+:){2}task/([\S]+)|; print "$2\n";') - [[ -z $task_id ]] && { echo "Couldn't parse task ID: $task_arn"; exit 1; } - - # Display logs for this task - log_stream_name=$prefix/$container/$task_id - echo "Task $task_id" - echo "-----------------------------------------" - aws logs get-log-events --log-group-name "$log_group_name" --log-stream-name "$log_stream_name" --query 'events[].message' | jq -r '.[]' || true - echo -done +"$DIR/ecs-service-logs" --cluster "app-${2}" --service "${1}" --status "RUNNING" --verbose diff --git a/bin/ecs-show-service-stopped-logs b/bin/ecs-show-service-stopped-logs index 87938d3f0f6..2af187c9b43 100755 --- a/bin/ecs-show-service-stopped-logs +++ b/bin/ecs-show-service-stopped-logs @@ -3,6 +3,8 @@ # Show logs from the most recently stopped app tasks. # set -eo pipefail +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +readonly DIR readonly LIMIT=${LIMIT:-25} usage() { @@ -12,23 +14,6 @@ usage() { [[ -z $1 || -z $2 ]] && usage set -u -readonly name=$1 -readonly environment=$2 -readonly cluster=app-$environment +[[ -f "$DIR/ecs-service-logs" ]] || (echo "Missing bin/ecs-service-logs. Run make build_tools" && exit 1) -# awslogs-stream-prefix settings -readonly prefix=$name -readonly container=$name-$environment -readonly log_group_name=ecs-tasks-$name-$environment - -# Get list of recently stopped tasks -for task_id in $(aws ecs describe-services --cluster "$cluster" --services "$name" --query 'services[].events[].message' | grep stopped | grep -o 'task [0-9a-f-]*' | cut -f 2 -d ' '); do - [[ -z $task_id ]] && { echo "Missing task ID"; exit 1; } - - # Display logs for this task - log_stream_name=$prefix/$container/$task_id - echo "Task $task_id" - echo "-----------------------------------------" - aws logs get-log-events --limit "$LIMIT" --log-group-name "$log_group_name" --log-stream-name "$log_stream_name" --query 'events[].message' | jq -r '.[]' || true - echo -done +"$DIR/ecs-service-logs" --cluster "app-${2}" --service "${1}" --environment "${2}" --status "STOPPED" --verbose diff --git a/cmd/ecs-service-logs/main.go b/cmd/ecs-service-logs/main.go new file mode 100644 index 00000000000..d746ca5b5c6 --- /dev/null +++ b/cmd/ecs-service-logs/main.go @@ -0,0 +1,475 @@ +// +// ecs-service-logs is a simple program to print ECS Service logs to stdout. +// +// Usage of ecs-service-logs: +// --aws-profile string The aws-vault profile +// --aws-region string The AWS Region (default "us-west-2") +// --aws-vault-keychain-name string The aws-vault keychain name +// --cluster string The cluster name +// --environment string The environment name +// --limit int The log limit. If 1 and above, will limit the results returned by each log stream. (default -1) +// --service string The service name +// --status string The task status: RUNNING, STOPPED +// --verbose Print section lines +// +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "regexp" + "strings" + + "github.com/99designs/aws-vault/prompt" + "github.com/99designs/aws-vault/vault" + "github.com/99designs/keyring" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/endpoints" + awssession "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/pkg/errors" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +// The ECS ARN format is changing as explained +// https://aws.amazon.com/ecs/faqs/#Transition_to_new_ARN_and_ID_format +var regexpTaskArnNew = regexp.MustCompile("^arn:aws:ecs:([^:]+?):([^:]+?):task/([^/]+?)/(.+)$") +var regexpTaskArnOld = regexp.MustCompile("^arn:aws:ecs:([^:]+?):([^:]+?):task/(.+)$") + +// We need to use regex to extract tasks ids from service events, +// because stopped tasks are only returned by ecs.ListTasks for up to an hour after stopped. +// - https://docs.aws.amazon.com/sdk-for-go/api/service/ecs/#ECS.ListTasks +var regexpServiceEventStoppedTask = regexp.MustCompile("^[(]service ([0-9a-zA-Z_-]+)[)] has stopped (\\d+) running tasks:\\s+(.+)[.]") +var regexpServiceEventStoppedTaskID = regexp.MustCompile("[(]task ([0-9a-z-]+)[)]") + +var environments = []string{"prod", "staging", "experimental"} +var ecsTaskStatuses = []string{"RUNNING", "STOPPED"} + +func parseTaskID(taskArn string) string { + + // Each match will include a slice of strings starting with + // (0) the full match, then + // (1) the region, + // (2) the account name, + // (3) (the cluster name if a new arn), and then + // (4) the task id. + + if matches := regexpTaskArnNew.FindStringSubmatch(taskArn); len(matches) > 0 { + return matches[4] // returns the task id that was parsed from the new format + } + + if matches := regexpTaskArnOld.FindStringSubmatch(taskArn); len(matches) > 0 { + return matches[3] // returns the task id that was parse from the old format + } + + return "" +} + +func parseStoppedTaskEvent(message string) []string { + if matches := regexpServiceEventStoppedTask.FindStringSubmatch(message); len(matches) > 0 { + if tasks := regexpServiceEventStoppedTaskID.FindAllStringSubmatch(matches[3], -1); len(tasks) > 0 { + taskIds := make([]string, 0, len(tasks)) + for _, task := range tasks { + taskIds = append(taskIds, task[1]) + } + return taskIds + } + } + return make([]string, 0) +} + +type errInvalidRegion struct { + Region string +} + +func (e *errInvalidRegion) Error() string { + return fmt.Sprintf("invalid region %q", e.Region) +} + +type errInvalidEnvironment struct { + Environment string +} + +func (e *errInvalidEnvironment) Error() string { + return fmt.Sprintf("invalid environment %q, expecting one of %q", e.Environment, environments) +} + +type errInvalidTaskStatus struct { + Status string +} + +func (e *errInvalidTaskStatus) Error() string { + return fmt.Sprintf("invalid status %q, expecting one of %q", e.Status, ecsTaskStatuses) +} + +type errInvalidCluster struct { + Cluster string +} + +func (e *errInvalidCluster) Error() string { + return fmt.Sprintf("invalid cluster %q", e.Cluster) +} + +type errInvalidService struct { + Service string +} + +func (e *errInvalidService) Error() string { + return fmt.Sprintf("invalid service %q", e.Service) +} + +func initFlags(flag *pflag.FlagSet) { + flag.String("aws-region", "us-west-2", "The AWS Region") + flag.String("aws-profile", "", "The aws-vault profile") + flag.String("aws-vault-keychain-name", "", "The aws-vault keychain name") + flag.String("cluster", "", "The cluster name") + flag.String("environment", "", "The environment name") + flag.String("service", "", "The service name") + flag.String("status", "", "The task status: "+strings.Join(ecsTaskStatuses, ", ")) + flag.Int("limit", -1, "The log limit. If 1 and above, will limit the results returned by each log stream.") + flag.BoolP("verbose", "v", false, "Print section lines") +} + +func checkConfig(v *viper.Viper) error { + clusterName := v.GetString("cluster") + + if len(clusterName) == 0 { + return &errInvalidCluster{Cluster: clusterName} + } + + regions, ok := endpoints.RegionsForService(endpoints.DefaultPartitions(), endpoints.AwsPartitionID, endpoints.EcsServiceID) + if !ok { + return fmt.Errorf("could not find regions for service %q", endpoints.EcsServiceID) + } + + region := v.GetString("aws-region") + if len(region) == 0 { + return errors.Wrap(&errInvalidRegion{Region: region}, fmt.Sprintf("%q is invalid", "aws-region")) + } + + if _, ok := regions[region]; !ok { + return errors.Wrap(&errInvalidRegion{Region: region}, fmt.Sprintf("%q is invalid", "aws-region")) + } + + status := v.GetString("status") + if len(status) > 0 { + valid := false + for _, str := range ecsTaskStatuses { + if status == str { + valid = true + break + } + } + if !valid { + return errors.Wrap(&errInvalidTaskStatus{Status: status}, fmt.Sprintf("%q is invalid", "status")) + } + + if status == "STOPPED" { + environment := v.GetString("environment") + if len(environment) == 0 { + return errors.New("when status is set to STOPPED then environment must be set") + } + valid := false + for _, str := range environments { + if environment == str { + valid = true + break + } + } + if !valid { + return errors.Wrap(&errInvalidEnvironment{Environment: environment}, fmt.Sprintf("%q is invalid", "environment")) + } + } + } + + return nil +} + +func quit(logger *log.Logger, flag *pflag.FlagSet, err error) { + logger.Println(err.Error()) + fmt.Println("Usage of ecs-service-logs:") + flag.PrintDefaults() + os.Exit(1) +} + +// Job is struct linking a task id to a given CloudWatch Log Stream. +type Job struct { + TaskID string + GetLogEventsInput *cloudwatchlogs.GetLogEventsInput +} + +// getAWSCredentials uses aws-vault to return AWS credentials +func getAWSCredentials(keychainName string, keychainProfile string) (*credentials.Credentials, error) { + + // Open the keyring which holds the credentials + ring, _ := keyring.Open(keyring.Config{ + ServiceName: "aws-vault", + AllowedBackends: []keyring.BackendType{keyring.KeychainBackend}, + KeychainName: keychainName, + KeychainTrustApplication: true, + }) + + // Prepare options for the vault before creating the provider + vConfig, err := vault.LoadConfigFromEnv() + if err != nil { + return nil, errors.Wrap(err, "Unable to load AWS config from environment") + } + vOptions := vault.VaultOptions{ + Config: vConfig, + MfaPrompt: prompt.Method("terminal"), + } + vOptions = vOptions.ApplyDefaults() + err = vOptions.Validate() + if err != nil { + return nil, errors.Wrap(err, "Unable to validate aws-vault options") + } + + // Get a new provider to retrieve the credentials + provider, err := vault.NewVaultProvider(ring, keychainProfile, vOptions) + if err != nil { + return nil, errors.Wrap(err, "Unable to create aws-vault provider") + } + credVals, err := provider.Retrieve() + if err != nil { + return nil, errors.Wrap(err, "Unable to retrieve aws credentials from aws-vault") + } + return credentials.NewStaticCredentialsFromCreds(credVals), nil +} + +func main() { + flag := pflag.CommandLine + initFlags(flag) + flag.Parse(os.Args[1:]) + + v := viper.New() + v.BindPFlags(flag) + v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + v.AutomaticEnv() + + // Create the logger + // Remove the prefix and any datetime data + logger := log.New(os.Stdout, "", log.LstdFlags) + + if !v.GetBool("verbose") { + // Disable any logging that isn't attached to the logger unless using the verbose flag + log.SetOutput(ioutil.Discard) + log.SetFlags(0) + + // Remove the flags for the logger + logger.SetFlags(0) + } + + err := checkConfig(v) + if err != nil { + quit(logger, flag, err) + } + + awsRegion := v.GetString("aws-region") + + awsConfig := &aws.Config{ + Region: aws.String(awsRegion), + } + + verbose := v.GetBool("verbose") + keychainName := v.GetString("aws-vault-keychain-name") + keychainProfile := v.GetString("aws-profile") + + if len(keychainName) > 0 && len(keychainProfile) > 0 { + creds, err := getAWSCredentials(keychainName, keychainProfile) + if err != nil { + quit(logger, flag, errors.Wrap(err, fmt.Sprintf("Unable to get AWS credentials from the keychain %s and profile %s", keychainName, keychainProfile))) + } + awsConfig.CredentialsChainVerboseErrors = aws.Bool(verbose) + awsConfig.Credentials = creds + } + + sess, err := awssession.NewSession(awsConfig) + if err != nil { + quit(logger, flag, errors.Wrap(err, "failed to create AWS session")) + } + + serviceECS := ecs.New(sess) + + serviceCloudWatchLogs := cloudwatchlogs.New(sess) + + clusterName := v.GetString("cluster") + serviceName := v.GetString("service") + status := v.GetString("status") + limit := v.GetInt("limit") + environment := v.GetString("environment") + + jobs := make([]Job, 0) + + if status == "STOPPED" { + stoppedTaskIds := make([]string, 0) + describeServicesInput := &ecs.DescribeServicesInput{ + Cluster: aws.String(clusterName), + } + if len(serviceName) > 0 { + describeServicesInput.Services = []*string{aws.String(serviceName)} + } + describeServicesOutput, err := serviceECS.DescribeServices(describeServicesInput) + if err != nil { + quit(logger, flag, err) + } + for _, service := range describeServicesOutput.Services { + for _, event := range service.Events { + message := aws.StringValue(event.Message) + if len(message) > 0 { + taskIds := parseStoppedTaskEvent(message) + if len(taskIds) > 0 { + stoppedTaskIds = append(stoppedTaskIds, taskIds...) + } + } + } + } + + // If there are no tasks returned from the query then simply exit. + if len(stoppedTaskIds) == 0 { + return + } + + for _, taskID := range stoppedTaskIds { + + logGroupName := fmt.Sprintf("ecs-tasks-%s-%s", serviceName, environment) + logStreamName := fmt.Sprintf("app/%s-%s/%s", serviceName, environment, taskID) + + getLogEventsInput := &cloudwatchlogs.GetLogEventsInput{ + LogGroupName: aws.String(logGroupName), + LogStreamName: aws.String(logStreamName), + } + if limit > 0 { + getLogEventsInput.Limit = aws.Int64(int64(limit)) + } + jobs = append(jobs, Job{ + TaskID: taskID, + GetLogEventsInput: getLogEventsInput, + }) + } + + } else { + taskArns := make([]*string, 0) + var nextToken *string + for { + + listTasksInput := &ecs.ListTasksInput{ + Cluster: aws.String(clusterName), + NextToken: nextToken, + } + if len(serviceName) > 0 { + listTasksInput.ServiceName = aws.String(serviceName) + } + listTasksOutput, err := serviceECS.ListTasks(listTasksInput) + if err != nil { + quit(logger, flag, err) + } + taskArns = append(taskArns, listTasksOutput.TaskArns...) + + if listTasksOutput.NextToken == nil { + break + } + nextToken = listTasksOutput.NextToken + } + + // If there are no tasks returned from the query then simply exit. + if len(taskArns) == 0 { + return + } + + describeTasksOutput, err := serviceECS.DescribeTasks(&ecs.DescribeTasksInput{ + Cluster: aws.String(clusterName), + Tasks: taskArns, + }) + if err != nil { + quit(logger, flag, err) + } + + tasks := describeTasksOutput.Tasks + + taskDefinitionArns := map[string]struct{}{} + for _, task := range tasks { + taskDefinitionArns[*task.TaskDefinitionArn] = struct{}{} + } + + taskDefinitions := map[string]*ecs.TaskDefinition{} + for taskDefinitionArn := range taskDefinitionArns { + describeTaskDefinitionOutput, err := serviceECS.DescribeTaskDefinition(&ecs.DescribeTaskDefinitionInput{ + TaskDefinition: aws.String(taskDefinitionArn), + }) + if err != nil { + panic(err) + } + taskDefinitions[taskDefinitionArn] = describeTaskDefinitionOutput.TaskDefinition + } + + for _, task := range tasks { + + if status != "" && status != *task.LastStatus { + continue + } + + taskID := parseTaskID(*task.TaskArn) + + taskDefinition, ok := taskDefinitions[*task.TaskDefinitionArn] + if !ok { + quit(logger, flag, fmt.Errorf("missing task definition with arn %s for task %s", *task.TaskDefinitionArn, *task.TaskArn)) + } + + for _, containerDefinition := range taskDefinition.ContainerDefinitions { + + containerName := *containerDefinition.Name + + logDriver := *containerDefinition.LogConfiguration.LogDriver + if logDriver != "awslogs" { + quit(logger, flag, fmt.Errorf("found log driver %s, expecting %s", logDriver, "awslogs")) + } + + logConfigurationOptions := containerDefinition.LogConfiguration.Options + if len(logConfigurationOptions) == 0 { + quit(logger, flag, fmt.Errorf("log configuration options is empty")) + } + + logGroupName := logConfigurationOptions["awslogs-group"] + //logRegion := *logConfigurationOptions["awslogs-region"] + streamPrefix := *logConfigurationOptions["awslogs-stream-prefix"] + + logStreamName := fmt.Sprintf("%s/%s/%s", streamPrefix, containerName, taskID) + + getLogEventsInput := &cloudwatchlogs.GetLogEventsInput{ + LogGroupName: logGroupName, + LogStreamName: aws.String(logStreamName), + } + if limit > 0 { + getLogEventsInput.Limit = aws.Int64(int64(limit)) + } + jobs = append(jobs, Job{ + TaskID: taskID, + GetLogEventsInput: getLogEventsInput, + }) + } + + } + } + + for _, job := range jobs { + + if verbose { + fmt.Println(fmt.Sprintf("Task %s", job.TaskID)) + fmt.Println("-----------------------------------------") + } + + getLogEventsOutput, err := serviceCloudWatchLogs.GetLogEvents(job.GetLogEventsInput) + if err != nil { + quit(logger, flag, errors.Wrap(err, "error retrieving log events")) + } + for _, event := range getLogEventsOutput.Events { + fmt.Println(*event.Message) + } + } + +}