Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Timestamp fractions pattern #15911

Merged
merged 5 commits into from Jan 31, 2020
Merged

Timestamp fractions pattern #15911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4f5db1b
Add support for `f` in date patterns
Jan 28, 2020
9055c73
Use `f` pattern in timestamp encoder
Jan 28, 2020
19c641e
update check
Jan 28, 2020
33ddbeb
Add godoc
Jan 31, 2020
cd6478a
typo
Jan 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 22 additions & 2 deletions libbeat/common/dtfmt/builder.go
View file
Open in desktop
Expand Up @@ -109,6 +109,26 @@ func (b *builder) nanoOfSecond(digits int) {
}
}

func (b *builder) fractNanoOfSecond(digits int) {
const fractDigits = 3

if digits <= 0 {
return
}

// cap number of digits at 9, as we do not support higher precision and
// would remove trailing zeroes anyway.
if digits > 9 {
digits = 9
}

minDigits := fractDigits
if digits < minDigits {
minDigits = digits
}
b.add(paddedNumber{ftNanoOfSecond, 9 - digits, minDigits, digits, fractDigits, false})
}

func (b *builder) secondOfMinute(digits int) {
b.appendDecimal(ftSecondOfMinute, digits, 2)
}
Expand Down Expand Up @@ -223,12 +243,12 @@ func (b *builder) appendDecimalValue(ft fieldType, minDigits, maxDigits int, sig
if minDigits <= 1 {
b.add(unpaddedNumber{ft, maxDigits, signed})
} else {
b.add(paddedNumber{ft, 0, minDigits, maxDigits, signed})
b.add(paddedNumber{ft, 0, minDigits, maxDigits, 0, signed})
}
}

func (b *builder) appendExtDecimal(ft fieldType, divExp, minDigits, maxDigits int) {
b.add(paddedNumber{ft, divExp, minDigits, maxDigits, false})
b.add(paddedNumber{ft, divExp, minDigits, maxDigits, 0, false})
}

func (b *builder) appendDecimal(ft fieldType, minDigits, maxDigits int) {
Expand Down
48 changes: 25 additions & 23 deletions libbeat/common/dtfmt/doc.go
View file
Open in desktop
Expand Up @@ -22,29 +22,31 @@
//
// Symbol Meaning Type Supported Examples
// ------ ------- ------- --------- -------
// G era text no AD
// C century of era (&gt;=0) number no 20
// Y year of era (&gt;=0) year yes 1996
//
// x weekyear year yes 1996
// w week of weekyear number yes 27
// e day of week number yes 2
// E day of week text yes Tuesday; Tue
//
// y year year yes 1996
// D day of year number yes 189
// M month of year month yes July; Jul; 07
// d day of month number yes 10
//
// a halfday of day text yes PM
// K hour of halfday (0~11) number yes 0
// h clockhour of halfday (1~12) number yes 12
//
// H hour of day (0~23) number yes 0
// k clockhour of day (1~24) number yes 24
// m minute of hour number yes 30
// s second of minute number yes 55
// S fraction of second millis no 978
// G era text no AD
// C century of era (&gt;=0) number no 20
// Y year of era (&gt;=0) year yes 1996
//
// x weekyear year yes 1996
// w week of weekyear number yes 27
// e day of week number yes 2
// E day of week text yes Tuesday; Tue
//
// y year year yes 1996
// D day of year number yes 189
// M month of year month yes July; Jul; 07
// d day of month number yes 10
//
// a halfday of day text yes PM
// K hour of halfday (0~11) number yes 0
// h clockhour of halfday (1~12) number yes 12
//
// H hour of day (0~23) number yes 0
// k clockhour of day (1~24) number yes 24
// m minute of hour number yes 30
// s second of minute number yes 55
// S fraction of second nanoseconds yes 978000
// f fraction of seconds nanoseconds yes 123456789
// multiple of 3
//
// z time zone text no Pacific Standard Time; PST
// Z time zone offset/id zone no -0800; -08:00; America/Los_Angeles
Expand Down
66 changes: 48 additions & 18 deletions libbeat/common/dtfmt/dtfmt_test.go
View file
Open in desktop
Expand Up @@ -82,10 +82,37 @@ func TestFormat(t *testing.T) {
{mkTime(8, 5, 24, 0), "kk:mm:ss aa", "09:05:24 AM"},
{mkTime(20, 5, 24, 0), "k:m:s a", "21:5:24 PM"},
{mkTime(20, 5, 24, 0), "kk:mm:ss aa", "21:05:24 PM"},
{mkTime(1, 2, 3, 123), "S", "1"},
{mkTime(1, 2, 3, 123), "SS", "12"},
{mkTime(1, 2, 3, 123), "SSS", "123"},
{mkTime(1, 2, 3, 123), "SSSS", "1230"},
{mkTime(1, 2, 3, 123*time.Millisecond), "S", "1"},
{mkTime(1, 2, 3, 123*time.Millisecond), "SS", "12"},
{mkTime(1, 2, 3, 123*time.Millisecond), "SSS", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "SSSS", "1230"},
{mkTime(1, 2, 3, 123*time.Millisecond), "f", "1"},
{mkTime(1, 2, 3, 123*time.Millisecond), "ff", "12"},
{mkTime(1, 2, 3, 123*time.Millisecond), "fff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "ffff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "fffff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "ffffff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "fffffff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "ffffffff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "fffffffff", "123"},
{mkTime(1, 2, 3, 123*time.Microsecond), "f", "0"},
{mkTime(1, 2, 3, 123*time.Microsecond), "ff", "00"},
{mkTime(1, 2, 3, 123*time.Microsecond), "fff", "000"},
{mkTime(1, 2, 3, 123*time.Microsecond), "ffff", "0001"},
{mkTime(1, 2, 3, 123*time.Microsecond), "fffff", "00012"},
{mkTime(1, 2, 3, 123*time.Microsecond), "ffffff", "000123"},
{mkTime(1, 2, 3, 123*time.Microsecond), "fffffff", "000123"},
{mkTime(1, 2, 3, 123*time.Microsecond), "ffffffff", "000123"},
{mkTime(1, 2, 3, 123*time.Microsecond), "fffffffff", "000123"},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just curious: Why are the trailing zeros removed? As a user, I would expect to see as many digits as I configured in the format string.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have multiple formatters for fractions of a second: S, n, and f:

  • S (fraction of second): print as many digits as set in format string
  • n (nanoseconds): always print 9 digits independent of the number of n in the format string
  • f (precision aware fraction of second): if timestamp precision is millisecond we print 3, if microsecond we print 6, and if nanosecond we print 9 digits.

By using f in Beats we would keep the original timestamp precision in case we parse a timestamp from a JSON file.

{mkTime(1, 2, 3, 123*time.Nanosecond), "f", "0"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "ff", "00"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "fff", "000"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "ffff", "000"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "fffff", "000"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "ffffff", "000"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "fffffff", "0000001"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "ffffffff", "00000012"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "fffffffff", "000000123"},

// literals
{time.Now(), "--=++,_!/?\\[]{}@#$%^&*()", "--=++,_!/?\\[]{}@#$%^&*()"},
Expand All @@ -94,24 +121,27 @@ func TestFormat(t *testing.T) {
{time.Now(), "'plain' '' 'text'", "plain ' text"},
{time.Now(), "'plain '' text'", "plain ' text"},

// beats timestamp
{mkDateTime(2017, 1, 2, 4, 6, 7, 123),
// timestamps with microseconds precision only
{mkDateTime(2017, 1, 2, 4, 6, 7, 123*time.Millisecond),
"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
"2017-01-02T04:06:07.123Z"},
{mkDateTime(2017, 1, 2, 4, 6, 7, 123456*time.Microsecond),
"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
"2017-01-02T04:06:07.123Z"},

// beats timestamp
{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123, time.FixedZone("PST", -8*60*60)),
"yyyy-MM-dd'T'HH:mm:ss.SSSz",
{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123*time.Millisecond, time.FixedZone("PST", -8*60*60)),
"yyyy-MM-dd'T'HH:mm:ss.fffffffffz",
"2017-01-02T04:06:07.123-08:00"},

// beats nanoseconds timestamp
{mkDateTime(2017, 1, 2, 4, 6, 7, 123),
"yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnn'Z'",
"2017-01-02T04:06:07.123000000Z"},
{mkDateTime(2017, 1, 2, 4, 6, 7, 123*time.Nanosecond),
"yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'",
"2017-01-02T04:06:07.000000123Z"},

{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123, time.FixedZone("PST", -8*60*60)),
"yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnnz",
"2017-01-02T04:06:07.123000000-08:00"},
{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123*time.Millisecond, time.FixedZone("PST", -8*60*60)),
"yyyy-MM-dd'T'HH:mm:ss.fffffffffz",
"2017-01-02T04:06:07.123-08:00"},
}

for i, test := range tests {
Expand All @@ -132,14 +162,14 @@ func mkDate(y, m, d int) time.Time {
return mkDateTime(y, m, d, 0, 0, 0, 0)
}

func mkTime(h, m, s, S int) time.Time {
func mkTime(h, m, s int, S time.Duration) time.Time {
return mkDateTime(2000, 1, 1, h, m, s, S)
}

func mkDateTime(y, M, d, h, m, s, S int) time.Time {
func mkDateTime(y, M, d, h, m, s int, S time.Duration) time.Time {
return mkDateTimeWithLocation(y, M, d, h, m, s, S, time.UTC)
}

func mkDateTimeWithLocation(y, M, d, h, m, s, S int, l *time.Location) time.Time {
return time.Date(y, time.Month(M), d, h, m, s, S*1000000, l)
func mkDateTimeWithLocation(y, M, d, h, m, s int, S time.Duration, l *time.Location) time.Time {
return time.Date(y, time.Month(M), d, h, m, s, int(S), l)
}
16 changes: 10 additions & 6 deletions libbeat/common/dtfmt/elems.go
View file
Open in desktop
Expand Up @@ -44,10 +44,10 @@ type unpaddedNumber struct {
}

type paddedNumber struct {
ft fieldType
divExp int
minDigits, maxDigits int
signed bool
ft fieldType
divExp int
minDigits, maxDigits, fractDigits int
signed bool
}

type textField struct {
Expand Down Expand Up @@ -188,10 +188,14 @@ func (n unpaddedNumber) compile() (prog, error) {
}

func (n paddedNumber) compile() (prog, error) {
if n.divExp == 0 {
switch {
case n.fractDigits != 0:
return makeProg(opExtNumFractPadded, byte(n.ft), byte(n.divExp), byte(n.maxDigits), byte(n.fractDigits))
case n.divExp == 0:
return makeProg(opNumPadded, byte(n.ft), byte(n.maxDigits))
default:
return makeProg(opExtNumPadded, byte(n.ft), byte(n.divExp), byte(n.maxDigits))
}
return makeProg(opExtNumPadded, byte(n.ft), byte(n.divExp), byte(n.maxDigits))
}

func (n twoDigitYear) compile() (prog, error) {
Expand Down
11 changes: 5 additions & 6 deletions libbeat/common/dtfmt/fmt.go
View file
Open in desktop
Expand Up @@ -59,8 +59,6 @@ func releaseCtx(c *ctx) {
// If pattern is invalid an error is returned.
func NewFormatter(pattern string) (*Formatter, error) {
b := newBuilder()

// pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
err := parsePatternTo(b, pattern)
if err != nil {
return nil, err
Expand Down Expand Up @@ -136,7 +134,6 @@ func (f *Formatter) Format(t time.Time) (string, error) {
}

func parsePatternTo(b *builder, pattern string) error {
// pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
for i := 0; i < len(pattern); {
tok, tokText, err := parseToken(pattern, &i)
if err != nil {
Expand Down Expand Up @@ -213,8 +210,8 @@ func parsePatternTo(b *builder, pattern string) error {
case 'S': // fraction of second
b.nanoOfSecond(tokLen)

case 'z': // timezone offset
b.timeZoneOffsetText()
case 'f': // faction of second (without zeros)
b.fractNanoOfSecond(tokLen)

case 'n': // nano second
// if timestamp layout use `n`, it always return 9 digits nanoseconds.
Expand All @@ -223,6 +220,9 @@ func parsePatternTo(b *builder, pattern string) error {
}
b.nanoOfSecond(tokLen)

case 'z': // timezone offset
b.timeZoneOffsetText()

case '\'': // literal
if tokLen == 1 {
b.appendRune(rune(tokText[0]))
Expand All @@ -243,7 +243,6 @@ func parseToken(pattern string, i *int) (rune, string, error) {
start := *i
idx := start
length := len(pattern)
// pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
r, w := utf8.DecodeRuneInString(pattern[idx:])
idx += w
if ('A' <= r && r <= 'Z') || ('a' <= r && r <= 'z') {
Expand Down
38 changes: 24 additions & 14 deletions libbeat/common/dtfmt/prog.go
View file
Open in desktop
Expand Up @@ -27,20 +27,21 @@ type prog struct {
}

const (
opNone byte = iota
opCopy1 // copy next byte
opCopy2 // copy next 2 bytes
opCopy3 // copy next 3 bytes
opCopy4 // copy next 4 bytes
opCopyShort // [op, len, content[len]]
opCopyLong // [op, len1, len, content[len1<<8 + len]]
opNum // [op, ft]
opNumPadded // [op, ft, digits]
opExtNumPadded // [op, ft, divExp, digits]
opZeros // [op, count]
opTwoDigit // [op, ft]
opTextShort // [op, ft]
opTextLong // [op, ft]
opNone byte = iota
opCopy1 // copy next byte
opCopy2 // copy next 2 bytes
opCopy3 // copy next 3 bytes
opCopy4 // copy next 4 bytes
opCopyShort // [op, len, content[len]]
opCopyLong // [op, len1, len, content[len1<<8 + len]]
opNum // [op, ft]
opNumPadded // [op, ft, digits]
opExtNumPadded // [op, ft, divExp, digits]
opExtNumFractPadded // [op, ft, divExp, digits, fractDigits]
opZeros // [op, count]
opTwoDigit // [op, ft]
opTextShort // [op, ft]
opTextLong // [op, ft]
)

var pow10Table [10]int
Expand Down Expand Up @@ -108,6 +109,15 @@ func (p prog) eval(bytes []byte, ctx *ctx, t time.Time) ([]byte, error) {
return bytes, err
}
bytes = appendPadded(bytes, v/div, digits)
case opExtNumFractPadded:
ft, divExp, digits, fractDigits := fieldType(p.p[i]), int(p.p[i+1]), int(p.p[i+2]), int(p.p[i+3])
div := pow10Table[divExp]
i += 4
v, err := getIntField(ft, ctx, t)
if err != nil {
return bytes, err
}
bytes = appendFractPadded(bytes, v/div, digits, fractDigits)
case opZeros:
digits := int(p.p[i])
i++
Expand Down