From 7e514c28c7f14b529922128bf8821df60620f8df Mon Sep 17 00:00:00 2001 From: Florian Bauer Date: Wed, 13 Dec 2023 05:13:06 +0000 Subject: [PATCH] squash: fix: upload memory optimization; refactor flag parsing; begin separation of packages Squashed commit of the following: * fix(kubernetes): Correct command line argument syntax * refactor(handler): Use Fprint instead of Fprintf for downloadLink * fix: upload memory optimization; refactor flag parsing; begin separation of packages Signed-off-by: Florian Bauer See merge request https://ref.ci/fsrvcorp/services/transfer/-/merge_requests/55 --- .dockerignore | 5 ++ .gitignore | 3 + .gitlab-ci.yml | 1 + .it/functions.sh | 2 +- .it/kubernetes/deployment.yaml | 2 +- Dockerfile | 2 +- example/docker-compose.yml | 7 +- go.mod | 24 ++++--- go.sum | 53 +++++++------- handler.go | 86 +++++++++++++--------- helper.go | 22 ------ internal/metrics/metrics.go | 58 +++++++++++++++ internal/metrics/middleware.go | 28 ++++++++ internal/metrics/roundtripper.go | 44 ++++++++++++ main.go | 118 +++++++++---------------------- worker.go | 8 ++- 16 files changed, 277 insertions(+), 186 deletions(-) create mode 100644 .dockerignore create mode 100644 internal/metrics/metrics.go create mode 100644 internal/metrics/middleware.go create mode 100644 internal/metrics/roundtripper.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a657a24 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +example +.it +.gitlab-ci.yml +.gitlab +.github diff --git a/.gitignore b/.gitignore index dc7e95b..bbaf487 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ /.idea /transfer /coverage +/example/data/transfer +!/example/data/transfer/.gitkeep +/example/data/.minio.sys diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1c7e9b..7012b06 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,6 +53,7 @@ Integration Tests: artifacts: true variables: TESTFMT_VERSION: 1.0.2 + DEFAULT_INGRESS_URL: "https://${DEFAULT_INGRESS_URL}" before_script: - (apt update && apt install -y curl wget) > /dev/null - wget -q "https://github.com/bonsai-oss/testfmt/releases/download/v${TESTFMT_VERSION}/testfmt_${TESTFMT_VERSION}_linux_amd64" -O testfmt && chmod +x testfmt diff --git a/.it/functions.sh b/.it/functions.sh index 605b17b..c922b32 100644 --- a/.it/functions.sh +++ b/.it/functions.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -declare -r URL="https://${DEFAULT_INGRESS_URL}" +declare URL="${DEFAULT_INGRESS_URL}" function generate_test_file() { local -r file_name="$1" diff --git a/.it/kubernetes/deployment.yaml b/.it/kubernetes/deployment.yaml index 1480e04..5acb23e 100644 --- a/.it/kubernetes/deployment.yaml +++ b/.it/kubernetes/deployment.yaml @@ -23,7 +23,7 @@ spec: command: - /app/transfer args: - - -link.prefix + - --link.prefix - https ports: - containerPort: 8080 diff --git a/Dockerfile b/Dockerfile index 068976f..4029a15 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,4 +21,4 @@ ENV AWS_ACCESS_KEY_ID="minio" ENV AWS_SECRET_ACCESS_KEY="minio123" COPY --from=builder /build/transfer /app/transfer COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -CMD ["/app/transfer", "-web.listen-address", ":8080", "-metrics.listen-address", ":8081"] +CMD ["/app/transfer", "--web.listen-address", ":8080", "--metrics.listen-address", ":8081"] diff --git a/example/docker-compose.yml b/example/docker-compose.yml index a4f027c..b56180a 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.7' services: minio: - image: quay.io/minio/minio:RELEASE.2021-11-09T03-21-45Z + image: quay.io/minio/minio:latest command: server --console-address ":9001" /data hostname: minio environment: @@ -17,7 +17,10 @@ services: retries: 3 transfer: - image: fsrv/transfer@sha256:3954c1425f289288b4b868ed339f9964e64b3454dd40781a73ff5f5ad51dddf3 + build: ../ hostname: transfer + environment: + S3_SECURE: 'false' ports: - "8080:8080" + - "8081:8081" diff --git a/go.mod b/go.mod index 1780ca2..47d004f 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module transfer go 1.21 require ( + github.com/alecthomas/kingpin/v2 v2.4.0 github.com/bonsai-oss/mux v1.8.1 github.com/fsrv-xyz/version v0.0.1 github.com/getsentry/sentry-go v0.25.0 @@ -12,27 +13,28 @@ require ( ) require ( + github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + github.com/xhit/go-str2duration/v2 v2.1.0 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 8d13668..1f6ba90 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= +github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bonsai-oss/mux v1.8.1 h1:+oCx4bLXEkn6O8Gyse39m2XH7AWWH/u9Rn9vSO3dwYU= @@ -15,10 +19,7 @@ github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -27,13 +28,13 @@ github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.65 h1:sOlB8T3nQK+TApTpuN3k4WD5KasvZIE3vVFzyyCa0go= @@ -53,40 +54,42 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handler.go b/handler.go index 23f9763..95733de 100644 --- a/handler.go +++ b/handler.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "crypto/sha512" "encoding/hex" "fmt" @@ -15,6 +14,8 @@ import ( "github.com/google/uuid" "github.com/minio/minio-go/v7" "github.com/prometheus/client_golang/prometheus" + + "transfer/internal/metrics" ) func (c *Config) HealthCheckHandler(w http.ResponseWriter, _ *http.Request) { @@ -53,7 +54,7 @@ func (c *Config) DownloadHandler(w http.ResponseWriter, r *http.Request) { statSpan.Status = sentry.SpanStatusOK filePath := fmt.Sprintf("%s/%s", id, filename) - object, err := c.minioClient.StatObject(r.Context(), p.S3BucketName, filePath, minio.StatObjectOptions{}) + object, err := c.minioClient.StatObject(statSpan.Context(), p.S3BucketName, filePath, minio.StatObjectOptions{}) if err != nil { switch minio.ToErrorResponse(err).StatusCode { case http.StatusNotFound: @@ -76,7 +77,7 @@ func (c *Config) DownloadHandler(w http.ResponseWriter, r *http.Request) { // only return checksum when called in sum mode if sumMode { - metricObjectAction.With(prometheus.Labels{"action": "sum"}).Inc() + metrics.ObjectAction.With(prometheus.Labels{metrics.LabelAction: "sum"}).Inc() _, httpResponseError := fmt.Fprintf(w, "%s %s\n", object.UserMetadata[ChecksumMetadataFieldName], filename) if httpResponseError != nil { sentry.CaptureMessage(fmt.Sprintf("%s: %s", err.Error(), r.URL.String())) @@ -90,7 +91,7 @@ func (c *Config) DownloadHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Disposition", "attachment; filename="+filename) objectGetSpan := handlerMainSpan.StartChild("object.get") - reader, err := c.minioClient.GetObject(r.Context(), p.S3BucketName, object.Key, minio.GetObjectOptions{}) + reader, err := c.minioClient.GetObject(objectGetSpan.Context(), p.S3BucketName, object.Key, minio.GetObjectOptions{}) if err != nil { objectGetSpan.Status = sentry.SpanStatusInternalError objectGetSpan.Finish() @@ -100,7 +101,7 @@ func (c *Config) DownloadHandler(w http.ResponseWriter, r *http.Request) { } objectGetSpan.Finish() - metricObjectAction.With(prometheus.Labels{"action": "download"}).Inc() + metrics.ObjectAction.With(prometheus.Labels{metrics.LabelAction: "download"}).Inc() objectCopySpan := handlerMainSpan.StartChild("object.copy") defer objectCopySpan.Finish() @@ -127,58 +128,73 @@ func (c *Config) UploadHandler(w http.ResponseWriter, r *http.Request) { } if !ok || filename == "" { - w.WriteHeader(http.StatusBadRequest) + http.Error(w, "filename not provided", http.StatusBadRequest) return } - if r.ContentLength > p.UploadLimitGB*GB { + if r.ContentLength > p.UploadLimitGB*metrics.GB { sentry.CaptureMessage("upload too large") - w.WriteHeader(http.StatusNotAcceptable) + http.Error(w, "upload too large", http.StatusRequestEntityTooLarge) return } metadata := make(map[string]string) sha512SumGenerator := sha512.New() - buf := &bytes.Buffer{} - tee := io.TeeReader(r.Body, buf) + pipeReader, pipeWriter := io.Pipe() + multiWriter := io.MultiWriter(sha512SumGenerator, pipeWriter) - copySpan := handlerMainSpan.StartChild("object.copy") + go func() { + copySpan := handlerMainSpan.StartChild("object.copy") + defer copySpan.Finish() + _, err := io.CopyN(multiWriter, r.Body, r.ContentLength) + pipeWriter.CloseWithError(err) + }() - written, err := io.CopyN(sha512SumGenerator, tee, r.ContentLength) - if written != r.ContentLength { - traceLog(c.logger, err) - sentry.CaptureException(err) - copySpan.Status = sentry.SpanStatusInternalError - copySpan.Finish() - w.WriteHeader(http.StatusInternalServerError) + objectForwardSpan := handlerMainSpan.StartChild("object.put") + + prefixId := uuid.NewString() + + uploadedObject, uploadError := c.minioClient.PutObject(objectForwardSpan.Context(), p.S3BucketName, prefixId+"/"+filename, pipeReader, r.ContentLength, minio.PutObjectOptions{ + ContentType: selectContentType(filename), + }) + + if uploadError != nil { + traceLog(c.logger, uploadError) + sentry.CaptureException(uploadError) + objectForwardSpan.Status = sentry.SpanStatusInternalError + w.WriteHeader(minio.ToErrorResponse(uploadError).StatusCode) return } - metadata[ChecksumMetadataFieldName] = hex.EncodeToString(sha512SumGenerator.Sum(nil)) - copySpan.Finish() - objectForwardSpan := handlerMainSpan.StartChild("object.put") - id := uuid.New() - _, err = c.minioClient.PutObject(r.Context(), p.S3BucketName, id.String()+"/"+filename, buf, r.ContentLength, minio.PutObjectOptions{ - ContentType: selectContentType(filename), - UserMetadata: metadata, + metadata[ChecksumMetadataFieldName] = hex.EncodeToString(sha512SumGenerator.Sum(nil)) + objectMetadataSpan := handlerMainSpan.StartChild("object.put.metadata") + _, copyError := c.minioClient.CopyObject(objectMetadataSpan.Context(), minio.CopyDestOptions{ + Bucket: p.S3BucketName, + Object: uploadedObject.Key, + UserMetadata: metadata, + ReplaceMetadata: true, + }, minio.CopySrcOptions{ + Bucket: uploadedObject.Bucket, + Object: uploadedObject.Key, }) - objectForwardSpan.Finish() - - if err != nil { - traceLog(c.logger, err) - sentry.CaptureException(err) + objectMetadataSpan.Finish() + if copyError != nil { + traceLog(c.logger, copyError) + sentry.CaptureException(copyError) objectForwardSpan.Status = sentry.SpanStatusInternalError - w.WriteHeader(minio.ToErrorResponse(err).StatusCode) + w.WriteHeader(minio.ToErrorResponse(copyError).StatusCode) return } - metricObjectSize.Observe(float64(r.ContentLength)) - metricObjectAction.With(prometheus.Labels{"action": "upload"}).Inc() + objectForwardSpan.Finish() + + metrics.ObjectSize.Observe(float64(r.ContentLength)) + metrics.ObjectAction.With(prometheus.Labels{metrics.LabelAction: "upload"}).Inc() - downloadLink := fmt.Sprintf("%s://%s/%s/%s\n", p.DownloadLinkPrefix, r.Host, id.String(), filename) + downloadLink := fmt.Sprintf("%s://%s/%s/%s\n", p.DownloadLinkPrefix, r.Host, prefixId, filename) // generate download link - _, downloadLinkResponseError := fmt.Fprintf(w, downloadLink) + _, downloadLinkResponseError := fmt.Fprint(w, downloadLink) handlerMainSpan.Data = map[string]interface{}{ "download_link": downloadLink, } diff --git a/helper.go b/helper.go index 288f398..7d317d2 100644 --- a/helper.go +++ b/helper.go @@ -7,30 +7,8 @@ import ( "path" "regexp" "runtime" - "time" - - "github.com/prometheus/client_golang/prometheus" ) -// apiMiddleware - logging and metrics for api endpoints -func apiMiddleware(handler http.HandlerFunc, logger *log.Logger, endpointName string) http.HandlerFunc { - if logger == nil { - logger = log.Default() - } - fn := func(w http.ResponseWriter, r *http.Request) { - metricEndpointRequests.With(prometheus.Labels{"endpoint": endpointName}).Inc() - start := time.Now() - - // serve http request - handler.ServeHTTP(w, r) - duration := time.Since(start) - metricOperationDuration.With(prometheus.Labels{"endpoint": endpointName}).Observe(duration.Seconds()) - - logger.Printf("%v %v %v", r.Method, r.RequestURI, duration) - } - return fn -} - // selectContentType - parse file extension and determine content type func selectContentType(filename string) string { extension := path.Ext(filename) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..67212b8 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,58 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + _ = iota + KB = 1 << (10 * iota) + MB + GB +) + +const ( + LabelMethod = "method" + LabelEndpoint = "endpoint" + LabelStatus = "status" + LabelAction = "action" +) + +const ( + namespace = "transfer" +) + +var ( + BackendRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Name: "backend_request_duration", + Help: "duration per backend request", + }, []string{LabelMethod, LabelEndpoint, LabelStatus}) + + ObjectSize = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Name: "object_size_bytes", + Help: "Uploaded objects by size", + Buckets: []float64{1 * KB, 10 * KB, 100 * KB, 1 * MB, 10 * MB, 100 * MB, 300 * MB, 600 * MB, 900 * MB}, + }) + + EndpointRequests = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "endpoint_requests", + Help: "HTTP endpoint requests", + }, []string{LabelEndpoint}) + + ObjectAction = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "object_action", + Help: "Actions applied to objects", + }, []string{LabelAction}) + + OperationDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Name: "operation_duration", + Help: "duration per endpoint", + Buckets: []float64{0.01, 0.05, 0.1, 0.2, 0.4, 1, 2, 4, 8, 10, 20}, + }, []string{LabelEndpoint}) +) diff --git a/internal/metrics/middleware.go b/internal/metrics/middleware.go new file mode 100644 index 0000000..e7351fa --- /dev/null +++ b/internal/metrics/middleware.go @@ -0,0 +1,28 @@ +package metrics + +import ( + "log" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// ApiMiddleware - logging and metrics for api endpoints +func ApiMiddleware(handler http.HandlerFunc, logger *log.Logger, endpointName string) http.HandlerFunc { + if logger == nil { + logger = log.Default() + } + fn := func(w http.ResponseWriter, r *http.Request) { + EndpointRequests.With(prometheus.Labels{LabelEndpoint: endpointName}).Inc() + start := time.Now() + + // serve http request + handler.ServeHTTP(w, r) + duration := time.Since(start) + OperationDuration.With(prometheus.Labels{LabelEndpoint: endpointName}).Observe(duration.Seconds()) + + logger.Printf("%v %v %v", r.Method, r.RequestURI, duration) + } + return fn +} diff --git a/internal/metrics/roundtripper.go b/internal/metrics/roundtripper.go new file mode 100644 index 0000000..31b7ae3 --- /dev/null +++ b/internal/metrics/roundtripper.go @@ -0,0 +1,44 @@ +package metrics + +import ( + "fmt" + "net/http" + "strconv" + "time" + + "github.com/getsentry/sentry-go" + "github.com/prometheus/client_golang/prometheus" +) + +type RoundTripper struct{} + +func (t RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + start := time.Now() + + span := sentry.SpanFromContext(req.Context()) + + if span != nil { + child := span.StartChild(fmt.Sprintf("%s %s", req.Method, req.URL.String())) + child.SetData("http.method", req.Method) + child.SetData("http.url", req.URL.String()) + child.SetData("http.content_length", strconv.FormatInt(req.ContentLength, 10)) + defer func() { + if child != nil { + child.Finish() + } + }() + } + + resp, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + return resp, err + } + + BackendRequestDuration.With(prometheus.Labels{ + LabelMethod: req.Method, + LabelEndpoint: req.URL.Host, + LabelStatus: strconv.Itoa(resp.StatusCode), + }).Observe(time.Since(start).Seconds()) + + return resp, err +} diff --git a/main.go b/main.go index 51151d6..09ffef4 100644 --- a/main.go +++ b/main.go @@ -2,37 +2,31 @@ package main import ( "context" - "flag" "fmt" "log" "net/http" "os" "os/signal" + "runtime" "strings" "sync" "time" + "github.com/alecthomas/kingpin/v2" "github.com/bonsai-oss/mux" "github.com/fsrv-xyz/version" "github.com/getsentry/sentry-go" sentryhttp "github.com/getsentry/sentry-go/http" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" + + "transfer/internal/metrics" ) // ChecksumMetadataFieldName - UserMetadata key for storing the checksum of the file const ChecksumMetadataFieldName = "Sha512sum" -const ( - _ = iota - KB = 1 << (10 * iota) - MB - GB -) - type State string const ( @@ -41,32 +35,6 @@ const ( ) var ( - metricObjectAction = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "transfer", - Name: "object_action", - Help: "Actions applied to objects", - }, []string{"action"}) - - metricEndpointRequests = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "transfer", - Name: "endpoint_requests", - Help: "HTTP endpoint requests", - }, []string{"endpoint"}) - - metricObjectSize = promauto.NewHistogram(prometheus.HistogramOpts{ - Namespace: "transfer", - Name: "object_size_bytes", - Help: "Uploaded objects by size", - Buckets: []float64{1 * KB, 10 * KB, 100 * KB, 1 * MB, 10 * MB, 100 * MB, 300 * MB, 600 * MB, 900 * MB}, - }) - - metricOperationDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "transfer", - Name: "operation_duration", - Help: "duration per endpoint", - Buckets: []float64{0.01, 0.05, 0.1, 0.2, 0.4, 1, 2, 4, 8, 10, 20}, - }, []string{"endpoint"}) - // declare initial state as unhealthy backendState = StateUnhealthy ) @@ -100,54 +68,32 @@ func init() { return } - flag.StringVar(&p.ListenAddress, "web.listen-address", ":8080", "web server listen address") - flag.StringVar(&p.MetricsListenAddress, "metrics.listen-address", "127.0.0.1:9042", "metrics endpoint listen address") - flag.Int64Var(&p.UploadLimitGB, "upload.limit", 2, "Upload limit in GiB") - flag.IntVar(&p.CleanupInterval, "cleanup.interval", 60, "interval in seconds for cleanup") - flag.IntVar(&p.HealthCheckInterval, "healthcheck.interval", 2, "interval in seconds for healthcheck") - flag.DurationVar(&p.HealthCheckReturnGap, "healthcheck.return.gap", 2*time.Second, "time in seconds for declaring the service as healthy after successful check") - flag.StringVar(&p.S3Endpoint, "s3.endpoint", "", "address to s3 endpoint") - flag.StringVar(&p.S3AccessKey, "s3.access", "", "s3 access key") - flag.StringVar(&p.S3SecretKey, "s3.secret", "", "s3 secret key") - flag.StringVar(&p.S3BucketName, "s3.bucket", "", "s3 storage bucket") - flag.BoolVar(&p.S3UseSecurity, "s3.secure", true, "use tls for connection") - flag.BoolVar(&p.DisableCleanupWorker, "cleanup.disable", false, "manage object deletion process") - flag.StringVar(&p.DownloadLinkPrefix, "link.prefix", "http", "prepending stuff for download link") - - showVersion := flag.Bool("version", false, "Show version") - - flag.Parse() - - if *showVersion { - fmt.Println(version.Print(os.Args[0])) - os.Exit(0) - } - - if os.Getenv("S3_ENDPOINT") != "" { - p.S3Endpoint = os.Getenv("S3_ENDPOINT") - } - if os.Getenv("AWS_ACCESS_KEY_ID") != "" { - p.S3AccessKey = os.Getenv("AWS_ACCESS_KEY_ID") - } - if os.Getenv("AWS_SECRET_ACCESS_KEY") != "" { - p.S3SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") - } - if os.Getenv("S3_BUCKET") != "" { - p.S3BucketName = os.Getenv("S3_BUCKET") - } - if os.Getenv("S3_SECURE") != "" { - switch os.Getenv("S3_SECURE") { - case "true": - p.S3UseSecurity = true - case "false": - p.S3UseSecurity = false - } - } + app := kingpin.New("transfer", "Daemon transferring files to s3 compatible storage") + app.Flag("web.listen-address", "web server listen address").Default(":8080").StringVar(&p.ListenAddress) + app.Flag("metrics.listen-address", "metrics endpoint listen address").Default("127.0.0.1:9042").StringVar(&p.MetricsListenAddress) + app.Flag("upload.limit", "Upload limit in GiB").Default("2").Int64Var(&p.UploadLimitGB) + app.Flag("cleanup.interval", "interval in seconds for cleanup").Default("60").IntVar(&p.CleanupInterval) + app.Flag("healthcheck.interval", "interval in seconds for healthcheck").Default("2").IntVar(&p.HealthCheckInterval) + app.Flag("healthcheck.return.gap", "time in seconds for declaring the service as healthy after successful check").Default("2s").DurationVar(&p.HealthCheckReturnGap) + app.Flag("s3.endpoint", "address to s3 endpoint").Envar("S3_ENDPOINT").StringVar(&p.S3Endpoint) + app.Flag("s3.access", "s3 access key").Envar("AWS_ACCESS_KEY_ID").StringVar(&p.S3AccessKey) + app.Flag("s3.secret", "s3 secret key").Envar("AWS_SECRET_ACCESS_KEY").StringVar(&p.S3SecretKey) + app.Flag("s3.bucket", "s3 storage bucket").Envar("S3_BUCKET").StringVar(&p.S3BucketName) + app.Flag("s3.secure", "use tls for connection").Envar("S3_SECURE").Default("true").BoolVar(&p.S3UseSecurity) + app.Flag("cleanup.disable", "manage object deletion process").Default("false").BoolVar(&p.DisableCleanupWorker) + app.Flag("link.prefix", "prepending stuff for download link").Default("http").StringVar(&p.DownloadLinkPrefix) + + app.HelpFlag.Short('h') + app.Version(version.Print(os.Args[0])) + kingpin.MustParse(app.Parse(os.Args[1:])) if p.S3AccessKey == "" || p.S3SecretKey == "" || p.S3Endpoint == "" || p.S3BucketName == "" { fmt.Println("no s3 details given") os.Exit(1) } + + // no thread limit + runtime.GOMAXPROCS(-1) } func webListener(server *http.Server, group *sync.WaitGroup) { @@ -165,8 +111,9 @@ func main() { c.logger = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile|log.Lmsgprefix) c.minioClient, err = minio.New(p.S3Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(p.S3AccessKey, p.S3SecretKey, ""), - Secure: p.S3UseSecurity, + Creds: credentials.NewStaticV4(p.S3AccessKey, p.S3SecretKey, ""), + Secure: p.S3UseSecurity, + Transport: metrics.RoundTripper{}, }) if err != nil { c.logger.Println(err) @@ -176,8 +123,9 @@ func main() { sentryInitError := sentry.Init(sentry.ClientOptions{ Release: version.Revision, TracesSampleRate: 1.0, - Debug: true, + Debug: false, EnableTracing: true, + AttachStacktrace: true, }) log.Println(sentryInitError) sentryHandler := sentryhttp.New(sentryhttp.Options{ @@ -186,9 +134,9 @@ func main() { applicationRouter := mux.NewRouter() applicationRouter.Use(sentryHandler.Handle) - applicationRouter.HandleFunc("/{filename}", apiMiddleware(c.UploadHandler, c.logger, "upload")).Methods(http.MethodPut) - applicationRouter.HandleFunc("/{id}/{filename}", apiMiddleware(c.DownloadHandler, c.logger, "download")).Methods(http.MethodGet) - applicationRouter.HandleFunc("/{id}/{filename}/{sum:sum}", apiMiddleware(c.DownloadHandler, c.logger, "sum")).Methods(http.MethodGet) + applicationRouter.HandleFunc("/{filename}", metrics.ApiMiddleware(c.UploadHandler, c.logger, "upload")).Methods(http.MethodPut) + applicationRouter.HandleFunc("/{id}/{filename}", metrics.ApiMiddleware(c.DownloadHandler, c.logger, "download")).Methods(http.MethodGet) + applicationRouter.HandleFunc("/{id}/{filename}/{sum:sum}", metrics.ApiMiddleware(c.DownloadHandler, c.logger, "sum")).Methods(http.MethodGet) metricsRouter := mux.NewRouter() metricsRouter.Handle("/metrics", promhttp.Handler()).Methods(http.MethodGet) diff --git a/worker.go b/worker.go index 3d7757a..9c887cd 100644 --- a/worker.go +++ b/worker.go @@ -8,6 +8,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/minio/minio-go/v7" "github.com/prometheus/client_golang/prometheus" + + "transfer/internal/metrics" ) // HealthCheckWorker - Worker for checking health of s3 backend @@ -68,13 +70,13 @@ func (c *Config) CleanupWorker(ctx context.Context, done chan<- interface{}) { sentryCleanupSpan := sentry.StartSpan( context.Background(), "object.cleanup", - sentry.TransactionName(fmt.Sprintf("cleanup %+q", object.Key)), + sentry.WithTransactionName(fmt.Sprintf("cleanup %+q", object.Key)), ) sentryCleanupSpan.SetTag("object.key", object.Key) traceLog(c.logger, "remove "+object.Key) - metricObjectAction.With(prometheus.Labels{"action": "delete"}).Inc() - if err := c.minioClient.RemoveObject(ctx, p.S3BucketName, object.Key, minio.RemoveObjectOptions{}); err != nil { + metrics.ObjectAction.With(prometheus.Labels{"action": "delete"}).Inc() + if err := c.minioClient.RemoveObject(sentryCleanupSpan.Context(), p.S3BucketName, object.Key, minio.RemoveObjectOptions{}); err != nil { sentryCleanupSpan.Status = sentry.SpanStatusInternalError sentry.CaptureException(err) traceLog(c.logger, err)