Skip to content

Commit

Permalink
Merge pull request #12 from lupinthe14th/develop
Browse files Browse the repository at this point in the history
improve: change logger package from logrus to zerolog
  • Loading branch information
lupinthe14th committed Jan 26, 2021
2 parents e045804 + 4951dd8 commit 2930c7a
Show file tree
Hide file tree
Showing 7 changed files with 141 additions and 53 deletions.
2 changes: 2 additions & 0 deletions client.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"net/http"

"github.com/elastic/go-elasticsearch/v8"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
)

Expand All @@ -24,6 +25,7 @@ func newClient(c *cli.Context) (*elasticsearch.Client, error) {
Username: username,
Password: password,
Transport: tp,
Logger: &CustomLogger{log.Logger},
}
es, err := elasticsearch.NewClient(cfg)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ require (
github.com/cheggaaa/pb/v3 v3.0.5
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/elastic/go-elasticsearch/v8 v8.0.0-20201216152027-57cb70149147
github.com/rs/zerolog v1.20.0
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sirupsen/logrus v1.7.0
github.com/tidwall/gjson v1.6.7
github.com/urfave/cli/v2 v2.3.0
golang.org/x/sys v0.0.0-20201223074533-0d417f636930 // indirect
Expand Down
21 changes: 12 additions & 9 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo=
github.com/cheggaaa/pb/v3 v3.0.5 h1:lmZOti7CraK9RSjzExsY53+WWfub9Qv13B5m4ptEoPE=
github.com/cheggaaa/pb/v3 v3.0.5/go.mod h1:X1L61/+36nz9bjIsrDU52qHKOQukUQe2Ge+YvGuquCw=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/elastic/go-elasticsearch/v8 v8.0.0-20201216152027-57cb70149147 h1:lT/ru7eO08Q5o7r9UFyNWUpUTKYTTDDik0m0R3AngEY=
github.com/elastic/go-elasticsearch/v8 v8.0.0-20201216152027-57cb70149147/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
Expand All @@ -21,18 +19,18 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs=
github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/tidwall/gjson v1.6.7 h1:Mb1M9HZCRWEcXQ8ieJo7auYyyiSux6w9XN3AdTpxJrE=
github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI=
github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
Expand All @@ -41,11 +39,16 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo=
golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
71 changes: 71 additions & 0 deletions logger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package main

import (
"io"
"io/ioutil"
"net/http"
"time"

"github.com/rs/zerolog"
)

// CustomLogger implements the estransport.Logger interface.
type CustomLogger struct {
zerolog.Logger
}

// LogRoundTrip prints the information about request and response.
func (l *CustomLogger) LogRoundTrip(
req *http.Request,
res *http.Response,
err error,
start time.Time,
dur time.Duration,
) error {
var (
e *zerolog.Event
nReq int64
nRes int64
)

// Set error level.
//
switch {
case err != nil:
e = l.Error()
case res != nil && res.StatusCode > 0 && res.StatusCode < 300:
e = l.Debug()
case res != nil && res.StatusCode > 299 && res.StatusCode < 500:
e = l.Warn()
case res != nil && res.StatusCode > 499:
e = l.Error()
default:
e = l.Error()
}

// Count number of bytes in request and response.
//
if req != nil && req.Body != nil && req.Body != http.NoBody {
nReq, _ = io.Copy(ioutil.Discard, req.Body)
}
if res != nil && res.Body != nil && res.Body != http.NoBody {
nRes, _ = io.Copy(ioutil.Discard, res.Body)
}

// Log event.
//
e.Str("method", req.Method).
Int("status_code", res.StatusCode).
Dur("duration", dur).
Int64("req_bytes", nReq).
Int64("res_bytes", nRes).
Msg(req.URL.String())

return nil
}

// RequestBodyEnabled makes the client pass request body to logger
func (l *CustomLogger) RequestBodyEnabled() bool { return true }

// ResponseBodyEnabled makes the client pass response body to logger
func (l *CustomLogger) ResponseBodyEnabled() bool { return true }
12 changes: 6 additions & 6 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@ import (
"strings"

"github.com/lupinthe14th/escli/pkg/version"
"github.com/sirupsen/logrus"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
)

func main() {
if err := newApp().Run(os.Args); err != nil {
logrus.Fatal(err)
log.Fatal().Err(err)
}
}

Expand Down Expand Up @@ -51,12 +52,11 @@ func newApp() *cli.App {
},
}
app.Before = func(c *cli.Context) error {
zerolog.SetGlobalLevel(zerolog.InfoLevel)
if debug {
logrus.SetLevel(logrus.DebugLevel)
zerolog.SetGlobalLevel(zerolog.DebugLevel)
}
logrus.SetFormatter(&logrus.TextFormatter{
FullTimestamp: true,
})
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
return nil
}

Expand Down
71 changes: 40 additions & 31 deletions search.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
"time"

"github.com/cheggaaa/pb/v3"
"github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
"github.com/tidwall/gjson"
"github.com/urfave/cli/v2"
)
Expand Down Expand Up @@ -42,9 +42,9 @@ var searchCommand = &cli.Command{
Usage: "Show match all",
},
&cli.StringFlag{
Name: "query",
Name: "filename",
Value: "",
Aliases: []string{"q"},
Aliases: []string{"f"},
Usage: "Specify query json file",
},
&cli.StringFlag{
Expand Down Expand Up @@ -88,7 +88,7 @@ func searchAction(c *cli.Context) error {

m, _ := time.ParseDuration("5m")
query, err := buildQuery(c)
logrus.Debugf("query: %s", query)
log.Debug().Msgf("query: %s", query)
if err != nil {
return err
}
Expand Down Expand Up @@ -125,12 +125,12 @@ func searchAction(c *cli.Context) error {
b.ReadFrom(res.Body)
total := gjson.GetBytes(b.Bytes(), "hits.total.value").Int()
bar := pb.Start64(total)
logrus.Infof("total hits: %v", total)
log.Debug().Msgf("total hits: %v", total)
hits := int64(len(gjson.GetBytes(b.Bytes(), "hits.hits").Array()))
logrus.Debugf("hits: %v", hits)
log.Debug().Msgf("hits: %v", hits)
took := gjson.GetBytes(b.Bytes(), "took").Int()
sid := gjson.GetBytes(b.Bytes(), "_scroll_id").String()
logrus.Debugf("sid: %v", sid)
log.Debug().Msgf("sid: %v", sid)

amplitudeIDs := make([]AmplitudeID, 0, hits)

Expand Down Expand Up @@ -196,35 +196,38 @@ func searchAction(c *cli.Context) error {
}
hits = int64(len(gjson.GetBytes(b.Bytes(), "hits.hits").Array()))
took += gjson.GetBytes(b.Bytes(), "took").Int()
logrus.Debugf("hits: %v", hits)
logrus.Debugf("amplitude Id: %v", len(amplitudeIDs))
log.Debug().Msgf("hits: %v", hits)
log.Debug().Msgf("amplitude Id: %v", len(amplitudeIDs))
// in any case, only the most recently received _scroll_id should be used.
// See: https://www.elastic.co/guide/en/elasticsearch/reference/master/paginate-search-results.html#scroll-search-results
sid = gjson.GetBytes(b.Bytes(), "_scroll_id").String()
logrus.Debugf("sid: %v", sid)
log.Debug().Msgf("sid: %v", sid)
}
}
bar.Finish()
out, err := json.Marshal(&amplitudeIDs)
if err != nil {
return err
if len(amplitudeIDs) > 0 {
out, err := json.Marshal(&amplitudeIDs)
if err != nil {
return err
}
fmt.Fprintf(w, "%v\n", string(out))
}
fmt.Fprintf(w, "%v\n", string(out))

logrus.Infof("amplitude Id count: %v", len(amplitudeIDs))
logrus.Infof(
"[%s] %d hits; took: %dms\n",
log.Debug().Msgf("amplitude Id count: %v", len(amplitudeIDs))
log.Debug().Msgf(
"[%s] %d hits; took: %dms",
res.Status(),
total,
took,
)
if c.Bool("print") {
printAmplitudeIDSummary(amplitudeIDs)
printAmplitudeIDSummary(c, amplitudeIDs)
}
return nil
}

func printAmplitudeIDSummary(amplitudeIDs []AmplitudeID) {
func printAmplitudeIDSummary(c *cli.Context, amplitudeIDs []AmplitudeID) {
w := c.App.Writer
memo := make(map[string]int)
for _, v := range amplitudeIDs {
memo[v.UserID]++
Expand All @@ -244,7 +247,7 @@ func printAmplitudeIDSummary(amplitudeIDs []AmplitudeID) {
return userIDs[i].count < userIDs[j].count
})
for i, userID := range userIDs {
logrus.Infof("%v: %v: %v", i+1, userID.uuid, userID.count)
fmt.Fprintf(w, "%v: %v: %v\n", i+1, userID.uuid, userID.count)
}
}

Expand Down Expand Up @@ -279,33 +282,35 @@ func trimNextEqual(s string) string {
}

func buildQuery(c *cli.Context) (io.Reader, error) {
filename := c.String("query")
logrus.Debugf("filename: %s", filename)
filename := c.String("filename")
log.Debug().Msgf("filename: %s", filename)
since := c.Timestamp("since").Format(time.RFC3339Nano)
logrus.Debugf("since: %s", since)
log.Debug().Msgf("since: %s", since)
until := c.Timestamp("until").Format(time.RFC3339Nano)
logrus.Debugf("until: %s", until)
log.Debug().Msgf("until: %s", until)
if filename == "" {
var b strings.Builder
switch c.String("rule") {
case "AmazonIpReputation":
b.WriteString(fmt.Sprintf(AMAZON_IP_REPUTATION_QUERY, since, until))
b.WriteString(fmt.Sprintf(AmazonIPReputationQuery, since, until))
case "AnonymousIP":
b.WriteString(fmt.Sprintf(ANONYMOUS_IP_QUERY, since, until))
b.WriteString(fmt.Sprintf(AnonymousIPQuery, since, until))
default:
b.WriteString(fmt.Sprintf(MATCH_ALL_QUERY, since, until))
b.WriteString(fmt.Sprintf(MatchAllQuery, since, until))
}
return strings.NewReader(b.String()), nil
}
query, err := ioutil.ReadFile(filename)
logrus.Debugf("query: %v", query)
log.Debug().Msgf("query: %v", query)
if err != nil {
return nil, err
}
return bytes.NewReader(query), nil
}

const AMAZON_IP_REPUTATION_QUERY = `{
// AmazonIPReputationQuery is a query string that match Amazon IP Reputation List Rule Group.
// If the time is not specified, the number of matches becomes large, so it is specified separately.
const AmazonIPReputationQuery = `{
"query": {
"bool": {
"must": [],
Expand Down Expand Up @@ -387,7 +392,9 @@ const AMAZON_IP_REPUTATION_QUERY = `{
}
}`

const ANONYMOUS_IP_QUERY = `{
// AnonymousIPQuery is a query string that match Anonymous IP List Rule Group.
// If the time is not specified, the number of matches becomes large, so it is specified separately.
const AnonymousIPQuery = `{
"query": {
"bool": {
"must": [
Expand Down Expand Up @@ -473,7 +480,9 @@ const ANONYMOUS_IP_QUERY = `{
}
}`

const MATCH_ALL_QUERY = `{
// MatchAllQuery is a query string that matches all.
// If the time is not specified, the number of matches becomes large, so it is specified separately.
const MatchAllQuery = `{
"query": {
"bool": {
"must": [
Expand Down

0 comments on commit 2930c7a

Please sign in to comment.