-
-
Notifications
You must be signed in to change notification settings - Fork 34
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #73 from tkuchiki/add-count-subcommand
Add count subcommand
- Loading branch information
Showing
13 changed files
with
559 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| package cmd | ||
|
|
||
| import ( | ||
| "os" | ||
|
|
||
| "github.com/spf13/cobra" | ||
| "github.com/tkuchiki/alp/counter" | ||
| "github.com/tkuchiki/alp/helpers" | ||
| "github.com/tkuchiki/alp/options" | ||
| "github.com/tkuchiki/alp/parsers" | ||
| ) | ||
|
|
||
| func NewCountCmd() *cobra.Command { | ||
| var countCmd = &cobra.Command{ | ||
| Use: "count", | ||
| Short: "Count by log entries", | ||
| Long: `Count by log entries`, | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| file, err := cmd.PersistentFlags().GetString("file") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| reverse, err := cmd.PersistentFlags().GetBool("reverse") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| pattern, err := cmd.PersistentFlags().GetString("pattern") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| opts := options.NewOptions() | ||
|
|
||
| opts = options.SetOptions(opts, | ||
| options.File(file), | ||
| options.Reverse(reverse), | ||
| options.Pattern(pattern), | ||
| ) | ||
|
|
||
| format, err := cmd.PersistentFlags().GetString("format") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| keysStr, err := cmd.PersistentFlags().GetString("keys") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| keys := helpers.SplitCSV(keysStr) | ||
|
|
||
| cnter := counter.NewCounter(os.Stdout, os.Stderr, opts.Reverse) | ||
|
|
||
| f, err := cnter.Open(opts.File) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer f.Close() | ||
|
|
||
| var parser parsers.Parser | ||
| switch format { | ||
| case "json": | ||
| jsonKeys := parsers.NewJSONKeys(opts.JSON.UriKey, opts.JSON.MethodKey, opts.JSON.TimeKey, | ||
| opts.JSON.ResponseTimeKey, opts.JSON.RequestTimeKey, opts.JSON.BodyBytesKey, opts.JSON.StatusKey) | ||
| parser = parsers.NewJSONParser(f, jsonKeys, opts.QueryString, opts.QueryStringIgnoreValues) | ||
| case "ltsv": | ||
| label := parsers.NewLTSVLabel(opts.LTSV.UriLabel, opts.LTSV.MethodLabel, opts.LTSV.TimeLabel, | ||
| opts.LTSV.ApptimeLabel, opts.LTSV.ReqtimeLabel, opts.LTSV.SizeLabel, opts.LTSV.StatusLabel, | ||
| ) | ||
| parser = parsers.NewLTSVParser(f, label, opts.QueryString, opts.QueryStringIgnoreValues) | ||
| case "regexp": | ||
| names := parsers.NewSubexpNames(opts.Regexp.UriSubexp, opts.Regexp.MethodSubexp, opts.Regexp.TimeSubexp, | ||
| opts.Regexp.ResponseTimeSubexp, opts.Regexp.RequestTimeSubexp, opts.Regexp.BodyBytesSubexp, opts.Regexp.StatusSubexp) | ||
| parser, err = parsers.NewRegexpParser(f, opts.Regexp.Pattern, names, opts.QueryString, opts.QueryStringIgnoreValues) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| cnter.SetParser(parser) | ||
|
|
||
| err = cnter.CountAndPrint(keys) | ||
|
|
||
| return err | ||
| }, | ||
| } | ||
|
|
||
| countCmd.PersistentFlags().StringP("format", "", "json", "Log format (json,ltsv,regexp)") | ||
| countCmd.PersistentFlags().StringP("pattern", "", options.DefaultPatternOption, "Regular expressions pattern matching the log (only use with --format=regexp)") | ||
| countCmd.PersistentFlags().StringP("file", "", "", "The access log file") | ||
| countCmd.PersistentFlags().BoolP("reverse", "r", false, "Sort results in reverse order") | ||
| countCmd.PersistentFlags().StringP("keys", "", "", "Log key names (comma separated)") | ||
| countCmd.MarkPersistentFlagRequired("keys") | ||
|
|
||
| return countCmd | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,178 @@ | ||
| package counter | ||
|
|
||
| import ( | ||
| "io" | ||
| "os" | ||
| "sort" | ||
| "strings" | ||
| "sync" | ||
|
|
||
| "github.com/tkuchiki/alp/errors" | ||
| "github.com/tkuchiki/alp/parsers" | ||
| ) | ||
|
|
||
| type Counter struct { | ||
| outWriter io.Writer | ||
| errWriter io.Writer | ||
| inReader *os.File | ||
| parser parsers.Parser | ||
| groups *groups | ||
| printer *Printer | ||
| reverse bool | ||
| } | ||
|
|
||
| func NewCounter(outw, errw io.Writer, reverse bool) *Counter { | ||
| return &Counter{ | ||
| outWriter: outw, | ||
| errWriter: errw, | ||
| inReader: os.Stdin, | ||
| printer: NewPrinter(outw), | ||
| groups: newGroups(), | ||
| reverse: reverse, | ||
| } | ||
| } | ||
|
|
||
| func (c *Counter) SetParser(parser parsers.Parser) { | ||
| c.parser = parser | ||
| } | ||
|
|
||
| func (c *Counter) SetInReader(f *os.File) { | ||
| c.inReader = f | ||
| } | ||
|
|
||
| func (c *Counter) Open(filename string) (*os.File, error) { | ||
| var f *os.File | ||
| var err error | ||
|
|
||
| if filename != "" { | ||
| f, err = os.Open(filename) | ||
| } else { | ||
| f = c.inReader | ||
| } | ||
|
|
||
| return f, err | ||
| } | ||
|
|
||
| func (c *Counter) Count(keys []string) error { | ||
| c.groups.keys = keys | ||
|
|
||
| Loop: | ||
| for { | ||
| s, err := c.parser.Parse() | ||
| if err != nil { | ||
| if err == io.EOF { | ||
| break | ||
| } else if err == errors.SkipReadLineErr { | ||
| continue Loop | ||
| } | ||
|
|
||
| return err | ||
| } | ||
|
|
||
| entries := make([]string, len(keys)) | ||
| group := newGroup() | ||
| for _, key := range keys { | ||
| val, ok := s.Entries[key] | ||
| if !ok { | ||
| continue Loop | ||
| } | ||
| entries = append(entries, val) | ||
| group.values[key] = val | ||
| } | ||
|
|
||
| concatenatedKey := strings.Join(entries, "_") | ||
| idx := c.groups.hints.loadOrStore(concatenatedKey) | ||
|
|
||
| if idx >= len(c.groups.groups) { | ||
| c.groups.groups = append(c.groups.groups, group) | ||
| } | ||
|
|
||
| c.groups.groups[idx].count++ | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (c *Counter) Sort() { | ||
| if c.reverse { | ||
| sort.Slice(c.groups.groups, func(i, j int) bool { | ||
| return c.groups.groups[i].count > c.groups.groups[j].count | ||
| }) | ||
| } else { | ||
| sort.Slice(c.groups.groups, func(i, j int) bool { | ||
| return c.groups.groups[i].count < c.groups.groups[j].count | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func (c *Counter) Print() { | ||
| c.Sort() | ||
| c.printer.Print(c.groups) | ||
| } | ||
|
|
||
| func (c *Counter) CountAndPrint(keys []string) error { | ||
| err := c.Count(keys) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| c.Print() | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| type hints struct { | ||
| values map[string]int | ||
| len int | ||
| mu sync.RWMutex | ||
| } | ||
|
|
||
| func newHints() *hints { | ||
| return &hints{ | ||
| values: make(map[string]int), | ||
| } | ||
| } | ||
|
|
||
| func (h *hints) loadOrStore(key string) int { | ||
| h.mu.Lock() | ||
| defer h.mu.Unlock() | ||
| _, ok := h.values[key] | ||
| if !ok { | ||
| h.values[key] = h.len | ||
| h.len++ | ||
| } | ||
|
|
||
| return h.values[key] | ||
| } | ||
|
|
||
| type groups struct { | ||
| keys []string | ||
| groups []*group | ||
| hints *hints | ||
| } | ||
|
|
||
| func newGroups() *groups { | ||
| return &groups{ | ||
| hints: newHints(), | ||
| } | ||
| } | ||
|
|
||
| type group struct { | ||
| values map[string]string | ||
| count int64 | ||
| } | ||
|
|
||
| func newGroup() *group { | ||
| return &group{ | ||
| values: make(map[string]string), | ||
| count: 0, | ||
| } | ||
| } | ||
|
|
||
| func (g *group) incr() { | ||
| g.count++ | ||
| } | ||
|
|
||
| func (g *group) getCount() int64 { | ||
| return g.count | ||
| } |
Oops, something went wrong.