Skip to content

Commit

Permalink
feat(stack): add Firelens sidecar deployment support
Browse files Browse the repository at this point in the history
  • Loading branch information
@iamhopaul123
iamhopaul123 committed Jun 4, 2020
1 parent dd65a89 commit 5378de4
Show file tree
Hide file tree
Showing 21 changed files with 200 additions and 128 deletions.
3 changes: 3 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,14 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
Expand Down Expand Up @@ -487,6 +489,7 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
18 changes: 6 additions & 12 deletions internal/pkg/aws/cloudwatchlogs/cloudwatchlogs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import (
const (
// SleepDuration is the sleep time for making the next request for log events.
SleepDuration = 1 * time.Second

logStreamNamePrefix = "copilot/"
)

var (
Expand Down Expand Up @@ -124,12 +126,8 @@ func (c *CloudWatchLogs) TaskLogEvents(logGroupName string, streamLastEventTime
}

for _, event := range resp.Events {
taskID, err := parseTaskID(*logStreamName)
if err != nil {
return nil, err
}
log := &Event{
TaskID: taskID,
LogStreamName: trimLogStreamName(*logStreamName),
IngestionTime: aws.Int64Value(event.IngestionTime),
Message: aws.StringValue(event.Message),
Timestamp: aws.Int64Value(event.Timestamp),
Expand Down Expand Up @@ -171,11 +169,7 @@ func (c *CloudWatchLogs) LogGroupExists(logGroupName string) (bool, error) {
return false, nil
}

func parseTaskID(logStreamName string) (string, error) {
// logStreamName example: ecs/{name}/1cc0685ad01d4d0f8e4e2c00d1775c56
logStreamNameSplit := strings.Split(logStreamName, "/")
if len(logStreamNameSplit) != 3 {
return "", fmt.Errorf("cannot parse task ID from log stream name: %s", logStreamName)
}
return logStreamNameSplit[2], nil
func trimLogStreamName(logStreamName string) string {
// logStreamName example: copilot/{name}/1cc0685ad01d4d0f8e4e2c00d1775c56
return strings.TrimPrefix(logStreamName, logStreamNamePrefix)
}
78 changes: 39 additions & 39 deletions internal/pkg/aws/cloudwatchlogs/cloudwatchlogs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,11 @@ func TestLogEvents(t *testing.T) {
OrderBy: aws.String("LastEventTime"),
}).Return(&cloudwatchlogs.DescribeLogStreamsOutput{
LogStreams: []*cloudwatchlogs.LogStream{
&cloudwatchlogs.LogStream{
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream1"),
{
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream1"),
},
&cloudwatchlogs.LogStream{
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream2"),
{
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream2"),
},
},
}, nil)
Expand All @@ -57,10 +57,10 @@ func TestLogEvents(t *testing.T) {
EndTime: aws.Int64(1234568),
Limit: aws.Int64(10),
LogGroupName: aws.String("mockLogGroup"),
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream1"),
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream1"),
}).Return(&cloudwatchlogs.GetLogEventsOutput{
Events: []*cloudwatchlogs.OutputLogEvent{
&cloudwatchlogs.OutputLogEvent{
{
Message: aws.String("some log"),
Timestamp: aws.Int64(1),
},
Expand All @@ -72,10 +72,10 @@ func TestLogEvents(t *testing.T) {
EndTime: aws.Int64(1234568),
Limit: aws.Int64(10),
LogGroupName: aws.String("mockLogGroup"),
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream2"),
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream2"),
}).Return(&cloudwatchlogs.GetLogEventsOutput{
Events: []*cloudwatchlogs.OutputLogEvent{
&cloudwatchlogs.OutputLogEvent{
{
Message: aws.String("other log"),
Timestamp: aws.Int64(0),
},
Expand All @@ -84,20 +84,20 @@ func TestLogEvents(t *testing.T) {
},

wantLogEvents: []*Event{
&Event{
TaskID: "mockLogStream2",
Message: "other log",
Timestamp: 0,
{
LogStreamName: "mockLogGroup/mockLogStream2",
Message: "other log",
Timestamp: 0,
},
&Event{
TaskID: "mockLogStream1",
Message: "some log",
Timestamp: 1,
{
LogStreamName: "mockLogGroup/mockLogStream1",
Message: "some log",
Timestamp: 1,
},
},
wantLastEventTime: map[string]int64{
"ecs/mockLogGroup/mockLogStream1": 1,
"ecs/mockLogGroup/mockLogStream2": 0,
"copilot/mockLogGroup/mockLogStream1": 1,
"copilot/mockLogGroup/mockLogStream2": 0,
},
wantErr: nil,
},
Expand All @@ -107,7 +107,7 @@ func TestLogEvents(t *testing.T) {
endTime: 1234999,
limit: 10,
lastEventTime: map[string]int64{
"ecs/mockLogGroup/mockLogStream": 1234890,
"copilot/mockLogGroup/mockLogStream": 1234890,
},
mockcloudwatchlogsClient: func(m *mocks.Mockapi) {
m.EXPECT().DescribeLogStreams(&cloudwatchlogs.DescribeLogStreamsInput{
Expand All @@ -116,8 +116,8 @@ func TestLogEvents(t *testing.T) {
OrderBy: aws.String("LastEventTime"),
}).Return(&cloudwatchlogs.DescribeLogStreamsOutput{
LogStreams: []*cloudwatchlogs.LogStream{
&cloudwatchlogs.LogStream{
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream"),
{
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream"),
},
},
}, nil)
Expand All @@ -127,10 +127,10 @@ func TestLogEvents(t *testing.T) {
Limit: aws.Int64(10),
EndTime: aws.Int64(1234999),
LogGroupName: aws.String("mockLogGroup"),
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream"),
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream"),
}).Return(&cloudwatchlogs.GetLogEventsOutput{
Events: []*cloudwatchlogs.OutputLogEvent{
&cloudwatchlogs.OutputLogEvent{
{
Message: aws.String("some log"),
Timestamp: aws.Int64(1234892),
},
Expand All @@ -139,14 +139,14 @@ func TestLogEvents(t *testing.T) {
},

wantLogEvents: []*Event{
&Event{
TaskID: "mockLogStream",
Message: "some log",
Timestamp: 1234892,
{
LogStreamName: "mockLogGroup/mockLogStream",
Message: "some log",
Timestamp: 1234892,
},
},
wantLastEventTime: map[string]int64{
"ecs/mockLogGroup/mockLogStream": 1234892,
"copilot/mockLogGroup/mockLogStream": 1234892,
},
wantErr: nil,
},
Expand All @@ -163,8 +163,8 @@ func TestLogEvents(t *testing.T) {
OrderBy: aws.String("LastEventTime"),
}).Return(&cloudwatchlogs.DescribeLogStreamsOutput{
LogStreams: []*cloudwatchlogs.LogStream{
&cloudwatchlogs.LogStream{
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream"),
{
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream"),
},
},
}, nil)
Expand All @@ -174,14 +174,14 @@ func TestLogEvents(t *testing.T) {
EndTime: aws.Int64(1234568),
Limit: aws.Int64(1),
LogGroupName: aws.String("mockLogGroup"),
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream"),
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream"),
}).Return(&cloudwatchlogs.GetLogEventsOutput{
Events: []*cloudwatchlogs.OutputLogEvent{
&cloudwatchlogs.OutputLogEvent{
{
Message: aws.String("some log"),
Timestamp: aws.Int64(0),
},
&cloudwatchlogs.OutputLogEvent{
{
Message: aws.String("other log"),
Timestamp: aws.Int64(1),
},
Expand All @@ -190,14 +190,14 @@ func TestLogEvents(t *testing.T) {
},

wantLogEvents: []*Event{
&Event{
TaskID: "mockLogStream",
Message: "other log",
Timestamp: 1,
{
LogStreamName: "mockLogGroup/mockLogStream",
Message: "other log",
Timestamp: 1,
},
},
wantLastEventTime: map[string]int64{
"ecs/mockLogGroup/mockLogStream": 1,
"copilot/mockLogGroup/mockLogStream": 1,
},
wantErr: nil,
},
Expand Down Expand Up @@ -250,7 +250,7 @@ func TestLogEvents(t *testing.T) {
OrderBy: aws.String("LastEventTime"),
}).Return(&cloudwatchlogs.DescribeLogStreamsOutput{
LogStreams: []*cloudwatchlogs.LogStream{
&cloudwatchlogs.LogStream{
{
LogStreamName: aws.String("mockLogStream"),
},
},
Expand Down
14 changes: 7 additions & 7 deletions internal/pkg/aws/cloudwatchlogs/event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ import (
)

const (
shortTaskIDLength = 7
shortLogStreamNameLength = 25
)

// Event represents a log event.
type Event struct {
TaskID string `json:"taskID"`
LogStreamName string `json:"logStreamName"`
IngestionTime int64 `json:"ingestionTime"`
Message string `json:"message"`
Timestamp int64 `json:"timestamp"`
Expand All @@ -41,12 +41,12 @@ func (l *Event) HumanString() string {
for _, code := range warningCodes {
l.Message = strings.ReplaceAll(l.Message, code, color.Yellow.Sprint(code))
}
return fmt.Sprintf("%s %s\n", color.Grey.Sprint(l.shortTaskID()), l.Message)
return fmt.Sprintf("%s %s\n", color.Grey.Sprint(l.shortLogStreamName()), l.Message)
}

func (l *Event) shortTaskID() string {
if len(l.TaskID) < shortTaskIDLength {
return l.TaskID
func (l *Event) shortLogStreamName() string {
if len(l.LogStreamName) < shortLogStreamNameLength {
return l.LogStreamName
}
return l.TaskID[0:shortTaskIDLength]
return l.LogStreamName[0:shortLogStreamNameLength]
}
32 changes: 16 additions & 16 deletions internal/pkg/cli/svc_logs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,29 +573,29 @@ func TestSvcLogs_Execute(t *testing.T) {
}
logEvents := []*cloudwatchlogs.Event{
{
TaskID: "123456789",
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -`,
LogStreamName: "firelens_log_router/fcfe4ab8043841c08162318e5ad805f1",
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -`,
},
{
TaskID: "123456789",
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -`,
LogStreamName: "firelens_log_router/fcfe4ab8043841c08162318e5ad805f1",
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -`,
},
{
TaskID: "123456789",
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -`,
LogStreamName: "firelens_log_router/fcfe4ab8043841c08162318e5ad805f1",
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -`,
},
}
moreLogEvents := []*cloudwatchlogs.Event{
{
TaskID: "123456789",
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 404 -`,
LogStreamName: "firelens_log_router/fcfe4ab8043841c08162318e5ad805f1",
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 404 -`,
},
}
logEventsHumanString := `1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -
1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -
1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -
logEventsHumanString := `firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -
firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -
firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -
`
logEventsJSONString := "{\"taskID\":\"123456789\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"GET / HTTP/1.1\\\" 200 -\",\"timestamp\":0}\n{\"taskID\":\"123456789\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"FATA some error\\\" - -\",\"timestamp\":0}\n{\"taskID\":\"123456789\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"WARN some warning\\\" - -\",\"timestamp\":0}\n"
logEventsJSONString := "{\"logStreamName\":\"firelens_log_router/fcfe4ab8043841c08162318e5ad805f1\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"GET / HTTP/1.1\\\" 200 -\",\"timestamp\":0}\n{\"logStreamName\":\"firelens_log_router/fcfe4ab8043841c08162318e5ad805f1\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"FATA some error\\\" - -\",\"timestamp\":0}\n{\"logStreamName\":\"firelens_log_router/fcfe4ab8043841c08162318e5ad805f1\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"WARN some warning\\\" - -\",\"timestamp\":0}\n"
testCases := map[string]struct {
inputApp string
inputSvc string
Expand Down Expand Up @@ -671,10 +671,10 @@ func TestSvcLogs_Execute(t *testing.T) {
},

wantedError: nil,
wantedContent: `1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -
1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -
1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -
1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 404 -
wantedContent: `firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -
firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -
firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -
firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 404 -
`,
},
"returns error if fail to get event logs": {
Expand Down
3 changes: 2 additions & 1 deletion internal/pkg/deploy/cloudformation/stack/backend_svc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,15 @@ func (s *BackendService) Template() (string, error) {
}
sidecars, err := s.manifest.Sidecar.SidecarsOpts()
if err != nil {
return "", fmt.Errorf("converts the sidecar configuration for service %s: %w", s.name, err)
return "", fmt.Errorf("convert the sidecar configuration for service %s: %w", s.name, err)
}
content, err := s.parser.ParseBackendService(template.ServiceOpts{
Variables: s.manifest.BackendServiceConfig.Variables,
Secrets: s.manifest.BackendServiceConfig.Secrets,
NestedStack: outputs,
Sidecars: sidecars,
HealthCheck: s.manifest.BackendServiceConfig.Image.HealthCheckOpts(),
LogConfig: s.manifest.LogConfigOpts(),
})
if err != nil {
return "", fmt.Errorf("parse backend service template: %w", err)
Expand Down
2 changes: 1 addition & 1 deletion internal/pkg/deploy/cloudformation/stack/backend_svc_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func TestBackendService_Template(t *testing.T) {
Value: hello`,
}
},
wantedErr: fmt.Errorf("converts the sidecar configuration for service frontend: %w", errors.New("cannot parse port mapping from 80/80/80")),
wantedErr: fmt.Errorf("convert the sidecar configuration for service frontend: %w", errors.New("cannot parse port mapping from 80/80/80")),
},
"failed parsing svc template": {
manifest: testBackendSvcManifest,
Expand Down
3 changes: 2 additions & 1 deletion internal/pkg/deploy/cloudformation/stack/lb_web_svc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,13 +95,14 @@ func (s *LoadBalancedWebService) Template() (string, error) {
}
sidecars, err := s.manifest.Sidecar.SidecarsOpts()
if err != nil {
return "", fmt.Errorf("converts the sidecar configuration for service %s: %w", s.name, err)
return "", fmt.Errorf("convert the sidecar configuration for service %s: %w", s.name, err)
}
content, err := s.parser.ParseLoadBalancedWebService(template.ServiceOpts{
Variables: s.manifest.Variables,
Secrets: s.manifest.Secrets,
NestedStack: outputs,
Sidecars: sidecars,
LogConfig: s.manifest.LogConfigOpts(),
RulePriorityLambda: rulePriorityLambda.String(),
})
if err != nil {
Expand Down
10 changes: 9 additions & 1 deletion internal/pkg/manifest/backend_svc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,18 @@ type BackendService struct {
type BackendServiceConfig struct {
Image imageWithPortAndHealthcheck `yaml:",flow"`
TaskConfig `yaml:",inline"`
LogConfig `yaml:"logging,flow"`
*LogConfig `yaml:"logging,flow"`
Sidecar `yaml:",inline"`
}

// LogConfigOpts converts the service's Firelens configuration into a format parsable by the templates pkg.
func (bc *BackendServiceConfig) LogConfigOpts() *template.LogConfigOpts {
if bc.LogConfig == nil {
return nil
}
return bc.logConfigOpts()
}

type imageWithPortAndHealthcheck struct {
ServiceImageWithPort `yaml:",inline"`
HealthCheck *ContainerHealthCheck `yaml:"healthcheck"`
Expand Down

0 comments on commit 5378de4

Please sign in to comment.