From c4bc92bf40a1ff40b56b6809fe88d4bbb8133c8c Mon Sep 17 00:00:00 2001 From: Ivan Nikolov Date: Wed, 8 Jan 2020 14:51:32 +0200 Subject: [PATCH 1/4] Bump service to v2 Apply go mod tidy Start using go-logger/v2 in main.go --- go.mod | 20 +++++++------------- go.sum | 47 ++++++++++++++++++++++++++--------------------- main.go | 44 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index 74c244a..b855766 100644 --- a/go.mod +++ b/go.mod @@ -1,33 +1,27 @@ -module github.com/Financial-Times/generic-rw-s3 +module github.com/Financial-Times/generic-rw-s3/v2 go 1.13 require ( github.com/Financial-Times/go-fthealth v0.0.0-20171204124831-1b007e2b37b7 github.com/Financial-Times/go-logger v0.0.0-20171020134212-ec38b420999c + github.com/Financial-Times/go-logger/v2 v2.0.1 github.com/Financial-Times/http-handlers-go v0.0.0-20170809121007-229ac16f1d9e github.com/Financial-Times/message-queue-gonsumer v0.0.0-20170622111749-6f96a5cb1e34 github.com/Financial-Times/service-status-go v0.0.0-20160323111542-3f5199736a3d github.com/Financial-Times/transactionid-utils-go v0.2.0 github.com/aws/aws-sdk-go v1.12.34-0.20171122234130-9365c0c6f327 - github.com/davecgh/go-spew v1.1.1-0.20171005155431-ecdeabc65495 // indirect - github.com/go-ini/ini v1.32.0 // indirect + github.com/go-ini/ini v1.51.1 // indirect github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.6.1-0.20171109035402-2d5fef06b891 - github.com/hashicorp/go-version v0.0.0-20170914154128-fc61389e27c7 // indirect + github.com/hashicorp/go-version v1.2.0 // indirect github.com/jawher/mow.cli v1.0.4-0.20171111121841-3ff64ca21987 - github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect + github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452 - github.com/onsi/ginkgo v1.10.3 // indirect - github.com/onsi/gomega v1.7.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 - github.com/sirupsen/logrus v1.0.4-0.20171118124223-95cd2b9c79aa + github.com/sirupsen/logrus v1.0.5 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/stretchr/testify v1.1.5-0.20171018052257-2aa2c176b9da - golang.org/x/text v0.3.2 // indirect - gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect - gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect - gopkg.in/ini.v1 v1.51.0 // indirect + gopkg.in/ini.v1 v1.51.1 // indirect ) diff --git a/go.sum b/go.sum index 2166ace..f3ac835 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/Financial-Times/go-fthealth v0.0.0-20171204124831-1b007e2b37b7 h1:dkf github.com/Financial-Times/go-fthealth v0.0.0-20171204124831-1b007e2b37b7/go.mod h1:gpAzq6W5rCheYlY32JOIxS/VjVcYHbC2PkMzQngHT9c= github.com/Financial-Times/go-logger v0.0.0-20171020134212-ec38b420999c h1:A7iKRlkrf64XqWTTky2pwrdI61xws2pTpOfy6hXhbDY= github.com/Financial-Times/go-logger v0.0.0-20171020134212-ec38b420999c/go.mod h1:NI4Dg39A21H57YC2nG8C42C6ENz/YVsI0jMQWngJzR0= +github.com/Financial-Times/go-logger/v2 v2.0.1 h1:iekEfSsUtlkg+YkXTZo+/fIN2VbZ2/3Hl9yolP3z5X8= +github.com/Financial-Times/go-logger/v2 v2.0.1/go.mod h1:Jpky5JYSX7xjGUClfA9hEMDmn40tUbfQQITjVIFGQiM= github.com/Financial-Times/http-handlers-go v0.0.0-20170809121007-229ac16f1d9e h1:/Y2wrSfkueFmdOIyQSABebfEe5P+yFyxBnmtnx1C0HM= github.com/Financial-Times/http-handlers-go v0.0.0-20170809121007-229ac16f1d9e/go.mod h1:sAkXv1oPYgNTYBYsYs83HwpYp7R50mvgBGGcsOlJtOw= github.com/Financial-Times/message-queue-gonsumer v0.0.0-20170622111749-6f96a5cb1e34 h1:c9jv/62SBTFAADwgx5S8uOZdPDkbPFTX031mIC0v6Kg= @@ -12,12 +14,12 @@ github.com/Financial-Times/transactionid-utils-go v0.2.0 h1:YcET5Hd1fUGWWpQSVszY github.com/Financial-Times/transactionid-utils-go v0.2.0/go.mod h1:tPAcAFs/dR6Q7hBDGNyUyixHRvg/n9NW/JTq8C58oZ0= github.com/aws/aws-sdk-go v1.12.34-0.20171122234130-9365c0c6f327 h1:Pzr+dv1XD7TVwRKTMI6k3ARQZZ4LIj/cjfeq/U6IuIY= github.com/aws/aws-sdk-go v1.12.34-0.20171122234130-9365c0c6f327/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= -github.com/davecgh/go-spew v1.1.1-0.20171005155431-ecdeabc65495 h1:b2hEFhj0PgDc77eCeDUSKXynIoXJRt6yTZ8aMk2cPoI= -github.com/davecgh/go-spew v1.1.1-0.20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v0.0.0-20170829195320-a47672248388 h1:xOYbryI96Npr2YM3ar+j8HTeiuA+vzxhiwaw+kLCruk= +github.com/davecgh/go-spew v0.0.0-20170829195320-a47672248388/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-ini/ini v1.32.0 h1:/MArBHSS0TFR28yPPDK1vPIjt4wUnPBfb81i6iiyKvA= -github.com/go-ini/ini v1.32.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ini/ini v1.51.1 h1:/QG3cj23k5V8mOl4JnNzUNhc1kr/jzMiNsNuWKcx8gM= +github.com/go-ini/ini v1.51.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -28,49 +30,52 @@ github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6Ylu github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.1-0.20171109035402-2d5fef06b891 h1:+Bw2euuJhuRflg4Z6ihh7u8GEFN/SS6T4tpI7fyEZrw= github.com/gorilla/mux v1.6.1-0.20171109035402-2d5fef06b891/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/hashicorp/go-version v0.0.0-20170914154128-fc61389e27c7 h1:Tijq+ZHupzK8WfomfH2s5dpKkpZd2TcN2i1LDbzWbwk= -github.com/hashicorp/go-version v0.0.0-20170914154128-fc61389e27c7/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jawher/mow.cli v1.0.4-0.20171111121841-3ff64ca21987 h1:r+pKutOAt5+z98xacKlh+4jiIQAtGF1iyuwupYObKcE= github.com/jawher/mow.cli v1.0.4-0.20171111121841-3ff64ca21987/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452 h1:hOY53G+kBFhbYFpRVxHl5eS7laP6B1+Cq+Z9Dry1iMU= github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc= +github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M= +github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 h1:gwcdIpH6NU2iF8CmcqD+CP6+1CkRBOhHaPR+iu6raBY= github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/sirupsen/logrus v1.0.4-0.20171118124223-95cd2b9c79aa h1:yTf7trBncZgwdV3qVVDwpppNGLF85oPIcypVwVBBZNk= -github.com/sirupsen/logrus v1.0.4-0.20171118124223-95cd2b9c79aa/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I= +github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.1.5-0.20171018052257-2aa2c176b9da h1:glZmY4mCDpnJuNJ4z+wbu5y2Qir8LgfkvYgv5as+LBY= github.com/stretchr/testify v1.1.5-0.20171018052257-2aa2c176b9da/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20170825220121-81e90905daef h1:R8ubLIilYRXIXpgjOg2l/ECVs3HzVKIjJEhxSsQ91u4= +golang.org/x/crypto v0.0.0-20170825220121-81e90905daef/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= @@ -80,9 +85,9 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= +gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index a5dbcd0..1a1467e 100644 --- a/main.go +++ b/main.go @@ -6,14 +6,15 @@ import ( "os" "time" - "github.com/Financial-Times/generic-rw-s3/service" + "github.com/Financial-Times/generic-rw-s3/v2/service" "github.com/Financial-Times/go-logger" + log "github.com/Financial-Times/go-logger/v2" "github.com/Financial-Times/message-queue-gonsumer/consumer" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/gorilla/mux" - "github.com/jawher/mow.cli" + cli "github.com/jawher/mow.cli" ) const ( @@ -121,21 +122,46 @@ func main() { app.Action = func() { + logger.InitLogger(*appName, *logLevel) + + logConfig := log.KeyNamesConfig{KeyTime: "@time"} + uppLogger := log.NewUPPLogger(*appName, *logLevel, logConfig) + uppLogger.Infof("Application started with args %s", os.Args) + qConf := consumer.QueueConfig{ Addrs: *sourceAddresses, Group: *sourceGroup, Topic: *sourceTopic, ConcurrentProcessing: *sourceConcurrentProcessing, } - runServer(*appName, *port, *resourcePath, *awsRegion, *bucketName, *bucketPrefix, *wrkSize, qConf, *onlyUpdatesEnabled, *requestLoggingEnabled) + runServer(*appName, + *port, + *resourcePath, + *awsRegion, + *bucketName, + *bucketPrefix, + *wrkSize, + qConf, + *onlyUpdatesEnabled, + *requestLoggingEnabled, + uppLogger) } - logger.InitLogger(*appName, *logLevel) - logger.Infof("Application started with args %s", os.Args) app.Run(os.Args) } -func runServer(appName string, port string, resourcePath string, awsRegion string, bucketName string, bucketPrefix string, wrks int, qConf consumer.QueueConfig, onlyUpdatesEnabled bool, requestLoggingEnabled bool) { +func runServer(appName string, + port string, + resourcePath string, + awsRegion string, + bucketName string, + bucketPrefix string, + wrks int, + qConf consumer.QueueConfig, + onlyUpdatesEnabled bool, + requestLoggingEnabled bool, + uppLogger *log.UPPLogger) { + hc := &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -157,7 +183,7 @@ func runServer(appName string, port string, resourcePath string, awsRegion strin HTTPClient: hc, }) if err != nil { - logger.Fatalf("Failed to create AWS session: %v", err) + uppLogger.Fatalf("Failed to create AWS session: %v", err) } svc := s3.New(sess) @@ -174,7 +200,7 @@ func runServer(appName string, port string, resourcePath string, awsRegion strin qp := service.NewQProcessor(w) - logger.Infof("listening on %v", port) + uppLogger.Infof("listening on %v", port) if qConf.Topic != "" { c := consumer.NewConsumer(qConf, qp.ProcessMsg, hc) @@ -182,7 +208,7 @@ func runServer(appName string, port string, resourcePath string, awsRegion strin defer c.Stop() } if err := http.ListenAndServe(":"+port, nil); err != nil { - logger.Fatalf("Unable to start server: %v", err) + uppLogger.Fatalf("Unable to start server: %v", err) } } From 756f32a9b3c50e4a5b2bdc9276117628d0011257 Mon Sep 17 00:00:00 2001 From: Ivan Nikolov Date: Wed, 8 Jan 2020 15:13:19 +0200 Subject: [PATCH 2/4] Move to v1.0.0 of message-queue-gosumer --- go.mod | 5 ++++- go.sum | 16 ++++++++++++++-- .../app-configs/generic-rw-s3_delivery.yaml | 2 +- main.go | 4 ++-- service/processor.go | 2 +- service/processor_test.go | 2 +- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index b855766..cefa7d6 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Financial-Times/go-logger v0.0.0-20171020134212-ec38b420999c github.com/Financial-Times/go-logger/v2 v2.0.1 github.com/Financial-Times/http-handlers-go v0.0.0-20170809121007-229ac16f1d9e - github.com/Financial-Times/message-queue-gonsumer v0.0.0-20170622111749-6f96a5cb1e34 + github.com/Financial-Times/message-queue-gonsumer v1.0.0 github.com/Financial-Times/service-status-go v0.0.0-20160323111542-3f5199736a3d github.com/Financial-Times/transactionid-utils-go v0.2.0 github.com/aws/aws-sdk-go v1.12.34-0.20171122234130-9365c0c6f327 @@ -19,9 +19,12 @@ require ( github.com/jawher/mow.cli v1.0.4-0.20171111121841-3ff64ca21987 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452 + github.com/onsi/ginkgo v1.10.3 // indirect + github.com/onsi/gomega v1.7.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 github.com/sirupsen/logrus v1.0.5 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/stretchr/testify v1.1.5-0.20171018052257-2aa2c176b9da + golang.org/x/text v0.3.2 // indirect gopkg.in/ini.v1 v1.51.1 // indirect ) diff --git a/go.sum b/go.sum index f3ac835..5b24e53 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/Financial-Times/go-logger/v2 v2.0.1 h1:iekEfSsUtlkg+YkXTZo+/fIN2VbZ2/ github.com/Financial-Times/go-logger/v2 v2.0.1/go.mod h1:Jpky5JYSX7xjGUClfA9hEMDmn40tUbfQQITjVIFGQiM= github.com/Financial-Times/http-handlers-go v0.0.0-20170809121007-229ac16f1d9e h1:/Y2wrSfkueFmdOIyQSABebfEe5P+yFyxBnmtnx1C0HM= github.com/Financial-Times/http-handlers-go v0.0.0-20170809121007-229ac16f1d9e/go.mod h1:sAkXv1oPYgNTYBYsYs83HwpYp7R50mvgBGGcsOlJtOw= -github.com/Financial-Times/message-queue-gonsumer v0.0.0-20170622111749-6f96a5cb1e34 h1:c9jv/62SBTFAADwgx5S8uOZdPDkbPFTX031mIC0v6Kg= -github.com/Financial-Times/message-queue-gonsumer v0.0.0-20170622111749-6f96a5cb1e34/go.mod h1:A88i3psx3Zm80Ai2OYTrwzKkZGKj+x5KL02z+YrRd10= +github.com/Financial-Times/message-queue-gonsumer v1.0.0 h1:yR6DglDU/2kxb6zkZnCO2737PJmtHiywNB6UrOye1c8= +github.com/Financial-Times/message-queue-gonsumer v1.0.0/go.mod h1:mLj+qMs1gNSO4jgCdZ/B+sl3Whgb+9XhQJQpEgHEATs= github.com/Financial-Times/service-status-go v0.0.0-20160323111542-3f5199736a3d h1:USNBTIof6vWGM49SYrxvC5Y8NqyDL3YuuYmID81ORZQ= github.com/Financial-Times/service-status-go v0.0.0-20160323111542-3f5199736a3d/go.mod h1:7zULC9rrq6KxFkpB3Y5zNVaEwrf1g2m3dvXJBPDXyvM= github.com/Financial-Times/transactionid-utils-go v0.2.0 h1:YcET5Hd1fUGWWpQSVszYUlAc15ca8tmjRetUuQKRqEQ= @@ -16,6 +16,8 @@ github.com/aws/aws-sdk-go v1.12.34-0.20171122234130-9365c0c6f327 h1:Pzr+dv1XD7TV github.com/aws/aws-sdk-go v1.12.34-0.20171122234130-9365c0c6f327/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/davecgh/go-spew v0.0.0-20170829195320-a47672248388 h1:xOYbryI96Npr2YM3ar+j8HTeiuA+vzxhiwaw+kLCruk= github.com/davecgh/go-spew v0.0.0-20170829195320-a47672248388/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-ini/ini v1.51.1 h1:/QG3cj23k5V8mOl4JnNzUNhc1kr/jzMiNsNuWKcx8gM= @@ -45,8 +47,12 @@ github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1: github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc= github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M= github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 h1:gwcdIpH6NU2iF8CmcqD+CP6+1CkRBOhHaPR+iu6raBY= @@ -58,6 +64,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.1.5-0.20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.1.5-0.20171018052257-2aa2c176b9da h1:glZmY4mCDpnJuNJ4z+wbu5y2Qir8LgfkvYgv5as+LBY= github.com/stretchr/testify v1.1.5-0.20171018052257-2aa2c176b9da/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20170825220121-81e90905daef h1:R8ubLIilYRXIXpgjOg2l/ECVs3HzVKIjJEhxSsQ91u4= @@ -76,6 +83,9 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= @@ -91,3 +101,5 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/helm/generic-rw-s3/app-configs/generic-rw-s3_delivery.yaml b/helm/generic-rw-s3/app-configs/generic-rw-s3_delivery.yaml index 546b160..9d0099d 100644 --- a/helm/generic-rw-s3/app-configs/generic-rw-s3_delivery.yaml +++ b/helm/generic-rw-s3/app-configs/generic-rw-s3_delivery.yaml @@ -11,7 +11,7 @@ env: valueFrom: configMapKeyRef: name: global-config - key: kafka.proxy.url.with.protocol + key: kafka-msk.proxy.url.with.protocol - name: SRC_GROUP value: generic-rw-s3-ConceptSuggestions - name: SRC_TOPIC diff --git a/main.go b/main.go index 1a1467e..92855f5 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,7 @@ import ( "github.com/Financial-Times/generic-rw-s3/v2/service" "github.com/Financial-Times/go-logger" log "github.com/Financial-Times/go-logger/v2" - "github.com/Financial-Times/message-queue-gonsumer/consumer" + consumer "github.com/Financial-Times/message-queue-gonsumer" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -203,7 +203,7 @@ func runServer(appName string, uppLogger.Infof("listening on %v", port) if qConf.Topic != "" { - c := consumer.NewConsumer(qConf, qp.ProcessMsg, hc) + c := consumer.NewConsumer(qConf, qp.ProcessMsg, hc, uppLogger) go c.Start() defer c.Stop() } diff --git a/service/processor.go b/service/processor.go index be27aa5..b2f2f41 100644 --- a/service/processor.go +++ b/service/processor.go @@ -11,7 +11,7 @@ import ( "sync" "github.com/Financial-Times/go-logger" - "github.com/Financial-Times/message-queue-gonsumer/consumer" + consumer "github.com/Financial-Times/message-queue-gonsumer" transactionid "github.com/Financial-Times/transactionid-utils-go" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" diff --git a/service/processor_test.go b/service/processor_test.go index 93b5705..914fd15 100644 --- a/service/processor_test.go +++ b/service/processor_test.go @@ -14,7 +14,7 @@ import ( "testing" "github.com/Financial-Times/go-logger" - "github.com/Financial-Times/message-queue-gonsumer/consumer" + consumer "github.com/Financial-Times/message-queue-gonsumer" transactionid "github.com/Financial-Times/transactionid-utils-go" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" From 035214681b49f98c1024f98b7a9783f720822fd2 Mon Sep 17 00:00:00 2001 From: Ivan Nikolov Date: Mon, 13 Jan 2020 16:17:04 +0200 Subject: [PATCH 3/4] Change logging level of the message for skipping unchanged records --- service/processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/processor.go b/service/processor.go index b2f2f41..2415bbb 100644 --- a/service/processor.go +++ b/service/processor.go @@ -348,7 +348,7 @@ func (w *S3Writer) Write(uuid string, b *[]byte, ct string, tid string, ignoreHa if err != nil { return status, err } else if w.onlyUpdatesEnabled && !ignoreHash && status == UNCHANGED { - logger.WithTransactionID(tid).WithUUID(uuid).Debug("Concept has not been updated since last upload, record was skipped") + logger.WithTransactionID(tid).WithUUID(uuid).Info("Concept has not been updated since last upload, record was skipped") return status, nil } From 5ce3ccaeaf9b84b4ea338ce63741ad7ea1d3b21e Mon Sep 17 00:00:00 2001 From: Ivan Nikolov Date: Mon, 13 Jan 2020 18:14:09 +0200 Subject: [PATCH 4/4] Add log level setting in helm deploymet --- helm/generic-rw-s3/templates/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm/generic-rw-s3/templates/deployment.yaml b/helm/generic-rw-s3/templates/deployment.yaml index 611b649..c1fefff 100644 --- a/helm/generic-rw-s3/templates/deployment.yaml +++ b/helm/generic-rw-s3/templates/deployment.yaml @@ -59,6 +59,8 @@ spec: key: {{ .Values.bucket.key }} - name: ONLY_UPDATES_ENABLED value: "true" + - name: LOG_LEVEL + value: info {{ toYaml .Values.env | indent 8 }} ports: - containerPort: 8080