Skip to content
Permalink
Browse files

Back to net/http

Month of testing and profiling show that fasthttp doesn't gives us significatnt profit in memory and performance while being incompatible with many third-side packages and http/2
  • Loading branch information...
DarthSim committed May 8, 2019
1 parent c5263b1 commit 3d13e6afb29a0836079126f4f91e562d39b5af91
Showing with 221 additions and 23,968 deletions.
  1. +8 −0 config.go
  2. +4 −0 docs/memory_usage_tweaks.md
  3. +13 −59 etag.go
  4. +1 −4 go.mod
  5. +0 −12 go.sum
  6. +5 −6 log.go
  7. +2 −2 newrelic.go
  8. +12 −10 processing_options.go
  9. +15 −21 processing_options_test.go
  10. +86 −107 server.go
  11. +0 −27 vendor/github.com/klauspost/compress/LICENSE
  12. +0 −32 vendor/github.com/klauspost/compress/flate/copy.go
  13. +0 −42 vendor/github.com/klauspost/compress/flate/crc32_amd64.go
  14. +0 −214 vendor/github.com/klauspost/compress/flate/crc32_amd64.s
  15. +0 −35 vendor/github.com/klauspost/compress/flate/crc32_noasm.go
  16. +0 −1,353 vendor/github.com/klauspost/compress/flate/deflate.go
  17. +0 −184 vendor/github.com/klauspost/compress/flate/dict_decoder.go
  18. +0 −265 vendor/github.com/klauspost/compress/flate/gen.go
  19. +0 −701 vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go
  20. +0 −344 vendor/github.com/klauspost/compress/flate/huffman_code.go
  21. +0 −880 vendor/github.com/klauspost/compress/flate/inflate.go
  22. +0 −48 vendor/github.com/klauspost/compress/flate/reverse_bits.go
  23. +0 −900 vendor/github.com/klauspost/compress/flate/snappy.go
  24. +0 −115 vendor/github.com/klauspost/compress/flate/token.go
  25. +0 −344 vendor/github.com/klauspost/compress/gzip/gunzip.go
  26. +0 −251 vendor/github.com/klauspost/compress/gzip/gzip.go
  27. +0 −183 vendor/github.com/klauspost/compress/zlib/reader.go
  28. +0 −201 vendor/github.com/klauspost/compress/zlib/writer.go
  29. +0 −24 vendor/github.com/klauspost/cpuid/.gitignore
  30. +0 −23 vendor/github.com/klauspost/cpuid/.travis.yml
  31. +0 −35 vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt
  32. +0 −22 vendor/github.com/klauspost/cpuid/LICENSE
  33. +0 −145 vendor/github.com/klauspost/cpuid/README.md
  34. +0 −1,040 vendor/github.com/klauspost/cpuid/cpuid.go
  35. +0 −42 vendor/github.com/klauspost/cpuid/cpuid_386.s
  36. +0 −42 vendor/github.com/klauspost/cpuid/cpuid_amd64.s
  37. +0 −17 vendor/github.com/klauspost/cpuid/detect_intel.go
  38. +0 −23 vendor/github.com/klauspost/cpuid/detect_ref.go
  39. +0 −4 vendor/github.com/klauspost/cpuid/generate.go
  40. +0 −476 vendor/github.com/klauspost/cpuid/private-gen.go
  41. +0 −15 vendor/github.com/valyala/bytebufferpool/.travis.yml
  42. +0 −22 vendor/github.com/valyala/bytebufferpool/LICENSE
  43. +0 −21 vendor/github.com/valyala/bytebufferpool/README.md
  44. +0 −111 vendor/github.com/valyala/bytebufferpool/bytebuffer.go
  45. +0 −7 vendor/github.com/valyala/bytebufferpool/doc.go
  46. +0 −151 vendor/github.com/valyala/bytebufferpool/pool.go
  47. +0 −3 vendor/github.com/valyala/fasthttp/.gitignore
  48. +0 −36 vendor/github.com/valyala/fasthttp/.travis.yml
  49. +0 −25 vendor/github.com/valyala/fasthttp/LICENSE
  50. +0 −585 vendor/github.com/valyala/fasthttp/README.md
  51. +0 −4 vendor/github.com/valyala/fasthttp/TODO
  52. +0 −588 vendor/github.com/valyala/fasthttp/args.go
  53. +0 −437 vendor/github.com/valyala/fasthttp/bytesconv.go
  54. +0 −7 vendor/github.com/valyala/fasthttp/bytesconv_32.go
  55. +0 −7 vendor/github.com/valyala/fasthttp/bytesconv_64.go
  56. +0 −2,257 vendor/github.com/valyala/fasthttp/client.go
  57. +0 −13 vendor/github.com/valyala/fasthttp/coarseTime.go
  58. +0 −438 vendor/github.com/valyala/fasthttp/compress.go
  59. +0 −534 vendor/github.com/valyala/fasthttp/cookie.go
  60. +0 −37 vendor/github.com/valyala/fasthttp/doc.go
  61. +0 −2 vendor/github.com/valyala/fasthttp/fasthttputil/doc.go
  62. +0 −5 vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.key
  63. +0 −10 vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.pem
  64. +0 −94 vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go
  65. +0 −283 vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go
  66. +0 −28 vendor/github.com/valyala/fasthttp/fasthttputil/rsa.key
  67. +0 −17 vendor/github.com/valyala/fasthttp/fasthttputil/rsa.pem
  68. +0 −1,271 vendor/github.com/valyala/fasthttp/fs.go
  69. +0 −9 vendor/github.com/valyala/fasthttp/go.mod
  70. +0 −10 vendor/github.com/valyala/fasthttp/go.sum
  71. +0 −2,200 vendor/github.com/valyala/fasthttp/header.go
  72. +0 −1,766 vendor/github.com/valyala/fasthttp/http.go
  73. +0 −183 vendor/github.com/valyala/fasthttp/lbclient.go
  74. +0 −11 vendor/github.com/valyala/fasthttp/nocopy.go
  75. +0 −100 vendor/github.com/valyala/fasthttp/peripconn.go
  76. +0 −2,501 vendor/github.com/valyala/fasthttp/server.go
  77. +0 −28 vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.key
  78. +0 −17 vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.pem
  79. +0 −3 vendor/github.com/valyala/fasthttp/stackless/doc.go
  80. +0 −79 vendor/github.com/valyala/fasthttp/stackless/func.go
  81. +0 −139 vendor/github.com/valyala/fasthttp/stackless/writer.go
  82. +0 −176 vendor/github.com/valyala/fasthttp/status.go
  83. +0 −54 vendor/github.com/valyala/fasthttp/stream.go
  84. +0 −80 vendor/github.com/valyala/fasthttp/strings.go
  85. +0 −448 vendor/github.com/valyala/fasthttp/tcpdialer.go
  86. +0 −54 vendor/github.com/valyala/fasthttp/timer.go
  87. +0 −525 vendor/github.com/valyala/fasthttp/uri.go
  88. +0 −12 vendor/github.com/valyala/fasthttp/uri_unix.go
  89. +0 −12 vendor/github.com/valyala/fasthttp/uri_windows.go
  90. +0 −71 vendor/github.com/valyala/fasthttp/userdata.go
  91. +0 −237 vendor/github.com/valyala/fasthttp/workerpool.go
  92. +74 −0 vendor/golang.org/x/net/netutil/listen.go
  93. +1 −12 vendor/modules.txt
@@ -202,6 +202,7 @@ type config struct {

FreeMemoryInterval int
DownloadBufferSize int
GZipBufferSize int
BufferPoolCalibrationThreshold int
}

@@ -329,6 +330,7 @@ func init() {

intEnvConfig(&conf.FreeMemoryInterval, "IMGPROXY_FREE_MEMORY_INTERVAL")
intEnvConfig(&conf.DownloadBufferSize, "IMGPROXY_DOWNLOAD_BUFFER_SIZE")
intEnvConfig(&conf.GZipBufferSize, "IMGPROXY_GZIP_BUFFER_SIZE")
intEnvConfig(&conf.BufferPoolCalibrationThreshold, "IMGPROXY_BUFFER_POOL_CALIBRATION_THRESHOLD")

if len(conf.Keys) != len(conf.Salts) {
@@ -453,6 +455,12 @@ func init() {
logFatal("Download buffer size can't be greater than %d", ^uint32(0))
}

if conf.GZipBufferSize < 0 {
logFatal("GZip buffer size should be greater than or equal to 0")
} else if conf.GZipBufferSize > int(^uint32(0)) {
logFatal("GZip buffer size can't be greater than %d", ^uint32(0))
}

if conf.BufferPoolCalibrationThreshold < 64 {
logFatal("Buffer pool calibration threshold should be greater than or equal to 64")
}
@@ -8,6 +8,10 @@ There are some imgproxy options that can help you to optimize memory usage and d

imgproxy uses memory buffers to download source images. While these buffers are empty at the start by default, they can grow to a required size when imgproxy downloads an image. Allocating new memory to grow the buffers can cause memory fragmentation. Allocating required memory at the start can eliminate much of memory fragmentation since buffers won't grow. Setting `IMGPROXY_DOWNLOAD_BUFFER_SIZE` will tell imgproxy to initialize download buffers with _at least_ the specified size. It's recommended to use the estimated 95 percentile of your image sizes as the initial download buffers size.

### `IMGPROXY_GZIP_BUFFER_SIZE`

The same as `IMGPROXY_DOWNLOAD_BUFFER_SIZE` but for GZip buffers. If you use GZip compression of the resulting images, you can reduce memory fragmentation by using the estimated maximum size of the GZipped resulting image as the initial size of GZip buffers.

### `IMGPROXY_FREE_MEMORY_INTERVAL`

Working with a large amount of data can cause allocating some memory that is not used most of the time. That's why imgproxy enforces Go's garbage collector to free as much memory as possible and return it to the OS. The default interval of this action is 10 seconds, but you can change it by setting `IMGPROXY_FREE_MEMORY_INTERVAL`. Decreasing the interval can smooth the memory usage graph but it can also slow down imgproxy a little. Increasing has the opposite effect.
72 etag.go
@@ -9,70 +9,26 @@ import (
"sync"
)

type etagPool struct {
mutex sync.Mutex
top *etagPoolEntry
}

type etagPoolEntry struct {
type eTagCalc struct {
hash hash.Hash
enc *json.Encoder
next *etagPoolEntry
b []byte
}

func newEtagPool(n int) *etagPool {
pool := new(etagPool)

for i := 0; i < n; i++ {
pool.grow()
}

return pool
}

func (p *etagPool) grow() {
h := sha256.New()

enc := json.NewEncoder(h)
enc.SetEscapeHTML(false)
enc.SetIndent("", "")

p.top = &etagPoolEntry{
hash: h,
enc: enc,
b: make([]byte, 64),
next: p.top,
}
}

func (p *etagPool) Get() *etagPoolEntry {
p.mutex.Lock()
defer p.mutex.Unlock()

if p.top == nil {
p.grow()
}

entry := p.top
p.top = p.top.next
var eTagCalcPool = sync.Pool{
New: func() interface{} {
h := sha256.New()

return entry
}

func (p *etagPool) Put(e *etagPoolEntry) {
p.mutex.Lock()
defer p.mutex.Unlock()
enc := json.NewEncoder(h)
enc.SetEscapeHTML(false)
enc.SetIndent("", "")

e.next = p.top
p.top = e
return &eTagCalc{h, enc}
},
}

var eTagCalcPool *etagPool

func calcETag(ctx context.Context) ([]byte, context.CancelFunc) {
c := eTagCalcPool.Get()
cancel := func() { eTagCalcPool.Put(c) }
func calcETag(ctx context.Context) string {
c := eTagCalcPool.Get().(*eTagCalc)
defer eTagCalcPool.Put(c)

c.hash.Reset()
c.hash.Write(getImageData(ctx).Bytes())
@@ -84,7 +40,5 @@ func calcETag(ctx context.Context) ([]byte, context.CancelFunc) {
c.enc.Encode(conf)
c.enc.Encode(getProcessingOptions(ctx))

hex.Encode(c.b, c.hash.Sum(nil))

return c.b, cancel
return hex.EncodeToString(c.hash.Sum(nil))
}
5 go.mod
@@ -19,8 +19,6 @@ require (
github.com/googleapis/gax-go v0.0.0-20181219185031-c8a15bac9b9f // indirect
github.com/honeybadger-io/honeybadger-go v0.4.0
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
github.com/klauspost/compress v1.4.1 // indirect
github.com/klauspost/cpuid v1.2.0 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/mat/besticon v3.9.0+incompatible
github.com/matoous/go-nanoid v0.0.0-20181114085210-eab626deece6
@@ -33,9 +31,8 @@ require (
github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74 // indirect
github.com/shirou/gopsutil v2.18.12+incompatible // indirect
github.com/stretchr/testify v1.3.0
github.com/valyala/fasthttp v1.2.0
golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b
golang.org/x/net v0.0.0-20190110200230-915654e7eabc // indirect
golang.org/x/net v0.0.0-20190110200230-915654e7eabc
golang.org/x/oauth2 v0.0.0-20190110195249-fd3eaa146cbb // indirect
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb // indirect
12 go.sum
@@ -60,12 +60,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro=
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -115,11 +109,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.2.0 h1:dzZJf2IuMiclVjdw0kkT+f9u4YdrapbNyGAN47E/qnk=
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938=
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -130,7 +119,6 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
11 log.go
@@ -4,8 +4,7 @@ import (
"fmt"
"log"
"log/syslog"

"github.com/valyala/fasthttp"
"net/http"
)

const (
@@ -18,13 +17,13 @@ const (
logFatalSyslogFmt = "FATAL %s"
)

func logRequest(reqID string, rctx *fasthttp.RequestCtx) {
path := rctx.RequestURI()
func logRequest(reqID string, r *http.Request) {
path := r.URL.RequestURI()

log.Printf(logRequestFmt, reqID, rctx.Method(), path)
log.Printf(logRequestFmt, reqID, r.Method, path)

if syslogWriter != nil {
syslogWriter.Notice(fmt.Sprintf(logRequestSyslogFmt, reqID, rctx.Method(), path))
syslogWriter.Notice(fmt.Sprintf(logRequestSyslogFmt, reqID, r.Method, path))
}
}

@@ -38,8 +38,8 @@ func initNewrelic() {
newRelicEnabled = true
}

func startNewRelicTransaction(ctx context.Context, r *http.Request) (context.Context, context.CancelFunc) {
txn := newRelicApp.StartTransaction("request", nil, r)
func startNewRelicTransaction(ctx context.Context, rw http.ResponseWriter, r *http.Request) (context.Context, context.CancelFunc) {
txn := newRelicApp.StartTransaction("request", rw, r)
cancel := func() { txn.End() }
return context.WithValue(ctx, newRelicTransactionCtxKey, txn), cancel
}
@@ -11,12 +11,11 @@ import (
"encoding/base64"
"errors"
"fmt"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"

"github.com/valyala/fasthttp"
)

type urlOptions map[string][]string
@@ -234,7 +233,7 @@ func decodeBase64URL(parts []string) (string, string, error) {
return "", "", errInvalidURLEncoding
}

fullURL := fmt.Sprintf("%s%s", conf.BaseURL, imageURL)
fullURL := fmt.Sprintf("%s%s", conf.BaseURL, string(imageURL))

if _, err := url.ParseRequestURI(fullURL); err != nil {
return "", "", errInvalidImageURL
@@ -259,7 +258,7 @@ func decodePlainURL(parts []string) (string, string, error) {
if unescaped, err := url.PathUnescape(urlParts[0]); err == nil {
fullURL := fmt.Sprintf("%s%s", conf.BaseURL, unescaped)
if _, err := url.ParseRequestURI(fullURL); err == nil {
return fmt.Sprintf("%s%s", conf.BaseURL, unescaped), format, nil
return fullURL, format, nil
}
}

@@ -848,8 +847,11 @@ func parsePathBasic(parts []string, headers *processingHeaders) (string, *proces
return url, po, nil
}

func parsePath(ctx context.Context, rctx *fasthttp.RequestCtx) (context.Context, error) {
path := string(rctx.Request.URI().PathOriginal())
func parsePath(ctx context.Context, r *http.Request) (context.Context, error) {
path := r.URL.RawPath
if len(path) == 0 {
path = r.URL.Path
}
parts := strings.Split(strings.TrimPrefix(path, "/"), "/")

if len(parts) < 3 {
@@ -863,10 +865,10 @@ func parsePath(ctx context.Context, rctx *fasthttp.RequestCtx) (context.Context,
}

headers := &processingHeaders{
Accept: string(rctx.Request.Header.Peek("Accept")),
Width: string(rctx.Request.Header.Peek("Width")),
ViewportWidth: string(rctx.Request.Header.Peek("Viewport-Width")),
DPR: string(rctx.Request.Header.Peek("DPR")),
Accept: r.Header.Get("Accept"),
Width: r.Header.Get("Width"),
ViewportWidth: r.Header.Get("Viewport-Width"),
DPR: r.Header.Get("DPR"),
}

var imageURL string

0 comments on commit 3d13e6a

Please sign in to comment.
You can’t perform that action at this time.