Skip to content

Commit 5d99671

Browse files
jefchienjefchien
authored
Add file tailing support to opentelemetry section (#2206)
1 parent 23e78d9 commit 5d99671

36 files changed

Lines changed: 2190 additions & 171 deletions

‎internal/util/timestamp/timestamp.go‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package timestamp
5+
6+
import "strings"
7+
8+
/*
9+
Strftime-to-regex and strftime-to-Go-layout mappings for timestamp parsing.
10+
11+
Go's reference time is: "Mon Jan 2 15:04:05 MST 2006"
12+
Based on https://golang.org/src/time/format.go and http://strftime.org/
13+
14+
Directive | Go layout | Regex | Meaning
15+
----------|-------------|--------------------|---------
16+
%B | January | \w{7} | Full month name
17+
%b | Jan | \w{3} | Abbreviated month name
18+
%-m | 1 | \s{0,1}\d{1,2} | Month (no leading zero)
19+
%m | 01 | \s{0,1}\d{1,2} | Month (zero-padded)
20+
%A | Monday | \w{6,9} | Full weekday name
21+
%a | Mon | \w{3} | Abbreviated weekday name
22+
%-d | _2 | \s{0,1}\d{1,2} | Day (no leading zero)
23+
%d | _2 | \s{0,1}\d{1,2} | Day (zero-padded)
24+
%H | 15 | \d{2} | Hour (24h, zero-padded)
25+
%-I | 3 | \d{1,2} | Hour (12h, no leading zero)
26+
%I | 03 | \d{2} | Hour (12h, zero-padded)
27+
%-M | 4 | \d{1,2} | Minute (no leading zero)
28+
%M | 04 | \d{2} | Minute (zero-padded)
29+
%-S | 5 | \d{1,2} | Second (no leading zero)
30+
%S | 05 | \d{2} | Second (zero-padded)
31+
%Y | 2006 | \d{4} | Year (4 digit)
32+
%y | 06 | \d{2} | Year (2 digit)
33+
%p | PM | \w{2} | AM/PM
34+
%Z | MST | \w{3} | Timezone name
35+
%z | -0700 | [+-]\d{4} | Timezone offset
36+
%f | .000 | \d{1,9} | Fractional seconds
37+
*/
38+
39+
// FormatRegexMap maps strftime directives to regex patterns for timestamp extraction.
40+
var FormatRegexMap = map[string]string{
41+
"%B": `\w{7}`,
42+
"%b": `\w{3}`,
43+
"%-m": `\s{0,1}\d{1,2}`,
44+
"%m": `\s{0,1}\d{1,2}`,
45+
"%A": `\w{6,9}`,
46+
"%a": `\w{3}`,
47+
"%-d": `\s{0,1}\d{1,2}`,
48+
"%d": `\s{0,1}\d{1,2}`,
49+
"%H": `\d{2}`,
50+
"%-I": `\d{1,2}`,
51+
"%I": `\d{2}`,
52+
"%-M": `\d{1,2}`,
53+
"%M": `\d{2}`,
54+
"%-S": `\d{1,2}`,
55+
"%S": `\d{2}`,
56+
"%Y": `\d{4}`,
57+
"%y": `\d{2}`,
58+
"%p": `\w{2}`,
59+
"%Z": `\w{3}`,
60+
"%z": `[+-]\d{4}`,
61+
"%f": `(\d{1,9})`,
62+
}
63+
64+
// FormatLayoutMap maps strftime directives to Go reference time layout strings.
65+
var FormatLayoutMap = map[string]string{
66+
"%B": "January",
67+
"%b": "Jan",
68+
"%-m": "1",
69+
"%m": "01",
70+
"%A": "Monday",
71+
"%a": "Mon",
72+
"%-d": "_2",
73+
"%d": "_2",
74+
"%H": "15",
75+
"%-I": "3",
76+
"%I": "03",
77+
"%-M": "4",
78+
"%M": "04",
79+
"%-S": "5",
80+
"%S": "05",
81+
"%Y": "2006",
82+
"%y": "06",
83+
"%p": "PM",
84+
"%Z": "MST",
85+
"%z": "-0700",
86+
"%f": ".000",
87+
}
88+
89+
// RegexEscapeMap escapes characters that are special in regex but normal in the
90+
// user's timestamp format string.
91+
var RegexEscapeMap = map[string]string{
92+
"^": `\^`,
93+
".": `\.`,
94+
"*": `\*`,
95+
"?": `\?`,
96+
"+": `\+`,
97+
"|": `\|`,
98+
"[": `\[`,
99+
"]": `\]`,
100+
"(": `\(`,
101+
")": `\)`,
102+
"{": `\{`,
103+
"}": `\}`,
104+
"$": `\$`,
105+
}
106+
107+
// BuildRegexWithNamedCaptureGroup converts a strftime format string to a regex with a named capture group (?P<timestamp>...).
108+
func BuildRegexWithNamedCaptureGroup(format string) string {
109+
return `(?P<timestamp>` + BuildRegex(format) + `)`
110+
}
111+
112+
// BuildRegex converts a strftime format string to a regex pattern.
113+
// If the format starts with "%-m" or "%-d", the leading \s{0,1} is stripped because:
114+
//
115+
// "%-m %-d %H:%M:%S" would produce regex "(\s{0,1}\d{1,2} \s{0,1}\d{1,2} \d{2}:\d{2}:\d{2})"
116+
// and layout "1 _2 15:04:05". The timestamp " 2 1 07:10:06" matches the regex but not the
117+
// layout. Stripping the prefix makes the regex and layout consistent.
118+
func BuildRegex(format string) string {
119+
res := ReplaceAll(format, RegexEscapeMap)
120+
res = ReplaceAll(res, FormatRegexMap)
121+
res = strings.TrimPrefix(res, `\s{0,1}`)
122+
return res
123+
}
124+
125+
// BuildLayout converts a strftime format string to a Go time layout string.
126+
func BuildLayout(format string) string {
127+
res := format
128+
// %f needs variable-width fractional seconds (".999999999") instead of FormatLayoutMap's
129+
// fixed ".000". Handle both ".%f" and "%f" since %f includes the dot separator.
130+
res = strings.ReplaceAll(res, ".%f", ".999999999")
131+
res = strings.ReplaceAll(res, "%f", ".999999999")
132+
// Use lenient layout "1" for month (accepts both "1" and "01"), avoiding v1's
133+
// two-layout approach where both %m ("01") and %-m ("1") variants were emitted.
134+
// %-m already maps to "1" in FormatLayoutMap so only %m needs overriding.
135+
res = strings.ReplaceAll(res, "%m", "%-m")
136+
return ReplaceAll(res, FormatLayoutMap)
137+
}
138+
139+
// ReplaceAll replaces all occurrences of keys in the replacements map with their values.
140+
func ReplaceAll(input string, replacements map[string]string) string {
141+
res := input
142+
for k, v := range replacements {
143+
if strings.Contains(res, k) {
144+
res = strings.ReplaceAll(res, k, v)
145+
}
146+
}
147+
return res
148+
}

‎internal/util/timestamp/timestamp_test.go‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package timestamp
5+
6+
import (
7+
"testing"
8+
"time"
9+
10+
"github.com/stretchr/testify/assert"
11+
"github.com/stretchr/testify/require"
12+
)
13+
14+
func TestBuildRegexWithNamedCaptureGroup(t *testing.T) {
15+
tests := []struct {
16+
name string
17+
format string
18+
expected string
19+
}{
20+
{
21+
name: "simple date time",
22+
format: "%Y-%m-%d %H:%M:%S",
23+
expected: `(?P<timestamp>\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2} \d{2}:\d{2}:\d{2})`,
24+
},
25+
{
26+
name: "date with milliseconds",
27+
format: "%Y-%m-%d %H:%M:%S.%f",
28+
expected: `(?P<timestamp>\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2} \d{2}:\d{2}:\d{2}\.(\d{1,9}))`,
29+
},
30+
{
31+
name: "syslog format",
32+
format: "%b %d %H:%M:%S",
33+
expected: `(?P<timestamp>\w{3} \s{0,1}\d{1,2} \d{2}:\d{2}:\d{2})`,
34+
},
35+
{
36+
name: "with timezone",
37+
format: "%Y-%m-%d %H:%M:%S %Z",
38+
expected: `(?P<timestamp>\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2} \d{2}:\d{2}:\d{2} \w{3})`,
39+
},
40+
{
41+
name: "12 hour format",
42+
format: "%Y-%m-%d %I:%M:%S %p",
43+
expected: `(?P<timestamp>\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2} \d{2}:\d{2}:\d{2} \w{2})`,
44+
},
45+
{
46+
name: "format starting with month",
47+
format: "%-m/%-d/%Y %H:%M:%S",
48+
expected: `(?P<timestamp>\d{1,2}/\s{0,1}\d{1,2}/\d{4} \d{2}:\d{2}:\d{2})`,
49+
},
50+
}
51+
52+
for _, tt := range tests {
53+
t.Run(tt.name, func(t *testing.T) {
54+
result := BuildRegexWithNamedCaptureGroup(tt.format)
55+
assert.Equal(t, tt.expected, result)
56+
})
57+
}
58+
}
59+
60+
func TestBuildRegex(t *testing.T) {
61+
result := BuildRegex("%Y-%m-%d %H:%M:%S")
62+
assert.Equal(t, `\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2} \d{2}:\d{2}:\d{2}`, result)
63+
}
64+
65+
func TestBuildRegex_EscapesSpecialChars(t *testing.T) {
66+
result := BuildRegex("%Y.%m.%d")
67+
assert.Equal(t, `\d{4}\.\s{0,1}\d{1,2}\.\s{0,1}\d{1,2}`, result)
68+
}
69+
70+
func TestBuildLayout(t *testing.T) {
71+
tests := []struct {
72+
name string
73+
format string
74+
expected string
75+
}{
76+
{
77+
name: "simple date time",
78+
format: "%Y-%m-%d %H:%M:%S",
79+
expected: "2006-1-_2 15:04:05",
80+
},
81+
{
82+
name: "non-padded month and day",
83+
format: "%-m/%-d/%Y %H:%M:%S",
84+
expected: "1/_2/2006 15:04:05",
85+
},
86+
{
87+
name: "with fractional seconds (dot in format)",
88+
format: "%Y-%m-%d %H:%M:%S.%f",
89+
expected: "2006-1-_2 15:04:05.999999999",
90+
},
91+
{
92+
name: "with fractional seconds (no dot in format)",
93+
format: "%Y-%m-%d %H:%M:%S%f",
94+
expected: "2006-1-_2 15:04:05.999999999",
95+
},
96+
{
97+
name: "12 hour format",
98+
format: "%Y-%m-%d %I:%M:%S %p",
99+
expected: "2006-1-_2 03:04:05 PM",
100+
},
101+
{
102+
name: "ISO8601 with T",
103+
format: "%Y-%m-%dT%H:%M:%S",
104+
expected: "2006-1-_2T15:04:05",
105+
},
106+
}
107+
108+
for _, tt := range tests {
109+
t.Run(tt.name, func(t *testing.T) {
110+
result := BuildLayout(tt.format)
111+
assert.Equal(t, tt.expected, result)
112+
})
113+
}
114+
}
115+
116+
func TestBuildLayout_ParsesTimestamps(t *testing.T) {
117+
tests := []struct {
118+
name string
119+
format string
120+
inputs []string
121+
want time.Time
122+
}{
123+
{
124+
name: "padded month parses",
125+
format: "%Y-%m-%d %H:%M:%S",
126+
inputs: []string{"2024-01-02 07:10:06", "2024-1-02 07:10:06"},
127+
want: time.Date(2024, 1, 2, 7, 10, 6, 0, time.UTC),
128+
},
129+
{
130+
name: "non-padded directives",
131+
format: "%-m/%-d/%Y %H:%M:%S",
132+
inputs: []string{"1/2/2024 07:10:06", "01/02/2024 07:10:06"},
133+
want: time.Date(2024, 1, 2, 7, 10, 6, 0, time.UTC),
134+
},
135+
{
136+
name: "ISO8601",
137+
format: "%Y-%m-%dT%H:%M:%S",
138+
inputs: []string{"2024-12-31T23:59:59"},
139+
want: time.Date(2024, 12, 31, 23, 59, 59, 0, time.UTC),
140+
},
141+
{
142+
name: "fractional seconds 3 digits",
143+
format: "%Y-%m-%d %H:%M:%S.%f",
144+
inputs: []string{"2024-01-02 07:10:06.123"},
145+
want: time.Date(2024, 1, 2, 7, 10, 6, 123000000, time.UTC),
146+
},
147+
{
148+
name: "fractional seconds 6 digits",
149+
format: "%Y-%m-%d %H:%M:%S.%f",
150+
inputs: []string{"2024-01-02 07:10:06.123456"},
151+
want: time.Date(2024, 1, 2, 7, 10, 6, 123456000, time.UTC),
152+
},
153+
{
154+
name: "fractional seconds 9 digits",
155+
format: "%Y-%m-%d %H:%M:%S.%f",
156+
inputs: []string{"2024-01-02 07:10:06.123456789"},
157+
want: time.Date(2024, 1, 2, 7, 10, 6, 123456789, time.UTC),
158+
},
159+
}
160+
161+
for _, tt := range tests {
162+
t.Run(tt.name, func(t *testing.T) {
163+
layout := BuildLayout(tt.format)
164+
for _, input := range tt.inputs {
165+
parsed, err := time.Parse(layout, input)
166+
require.NoError(t, err, "layout=%q input=%q", layout, input)
167+
assert.Equal(t, tt.want, parsed)
168+
}
169+
})
170+
}
171+
}

‎translator/config/schema.json‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1971,6 +1971,59 @@
19711971
},
19721972
"required": ["collect_list"],
19731973
"additionalProperties": false
1974+
},
1975+
"files": {
1976+
"type": "object",
1977+
"properties": {
1978+
"collect_list": {
1979+
"type": "array",
1980+
"minItems": 1,
1981+
"items": {
1982+
"type": "object",
1983+
"properties": {
1984+
"file_path": {
1985+
"description": "File path or glob pattern to tail",
1986+
"type": "string",
1987+
"minLength": 1,
1988+
"maxLength": 4096
1989+
},
1990+
"log_group_name": {
1991+
"description": "CloudWatch Logs group name override",
1992+
"type": "string",
1993+
"minLength": 1,
1994+
"maxLength": 512
1995+
},
1996+
"log_stream_name": {
1997+
"description": "CloudWatch Logs stream name override",
1998+
"type": "string",
1999+
"minLength": 1,
2000+
"maxLength": 512
2001+
},
2002+
"multi_line_start_pattern": {
2003+
"description": "Regex pattern that marks the start of a new log entry. Use {timestamp_format} to derive from timestamp_format.",
2004+
"type": "string"
2005+
},
2006+
"timestamp_format": {
2007+
"description": "Strftime format for parsing timestamps from log lines (e.g. %Y-%m-%d %H:%M:%S)",
2008+
"type": "string"
2009+
},
2010+
"timezone": {
2011+
"description": "Timezone for timestamp parsing",
2012+
"type": "string",
2013+
"enum": ["UTC", "Local", "LOCAL"]
2014+
},
2015+
"encoding": {
2016+
"description": "File encoding (default: utf-8)",
2017+
"type": "string"
2018+
}
2019+
},
2020+
"required": ["file_path"],
2021+
"additionalProperties": false
2022+
}
2023+
}
2024+
},
2025+
"required": ["collect_list"],
2026+
"additionalProperties": false
19742027
}
19752028
},
19762029
"additionalProperties": false

‎translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,8 @@ processors:
966966
statements:
967967
- set(resource.attributes["aws.log.group.name"], Concat(["/aws/cwagent", resource.attributes["aws.log.source"]], "/")) where resource.attributes["aws.log.group.name"] == nil and resource.attributes["aws.log.source"] != nil
968968
- set(resource.attributes["aws.log.group.name"], "/aws/cwagent/default") where resource.attributes["aws.log.group.name"] == nil
969+
- set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["log.file.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["aws.log.source"] == "files" and resource.attributes["host.id"] != nil and resource.attributes["log.file.name"] != nil
970+
- set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["log.file.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["aws.log.source"] == "files" and resource.attributes["host.name"] != nil and resource.attributes["log.file.name"] != nil
969971
- set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["service.namespace"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil and resource.attributes["service.namespace"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service"
970972
- set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service"
971973
- set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["service.namespace"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil and resource.attributes["service.namespace"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service"
@@ -1589,8 +1591,8 @@ service:
15891591
- ec2tagger
15901592
- awsentity/service/telegraf
15911593
receivers:
1592-
- telegraf_statsd
15931594
- telegraf_socket_listener
1595+
- telegraf_statsd
15941596
metrics/hostDeltaMetrics:
15951597
exporters:
15961598
- awscloudwatch

0 commit comments

Comments
 (0)