From f58fa936cd4e34e5d0d458d37d9fb2d42b06d4db Mon Sep 17 00:00:00 2001 From: Aofei Sheng Date: Sat, 31 Aug 2019 01:04:32 +0800 Subject: [PATCH] refactor: simplify caching --- cfg/cfg.go | 3 + config.toml | 1 + go.mod | 1 + go.sum | 7 ++ handler/handler.go | 223 ++++++++++++++------------------------------- 5 files changed, 78 insertions(+), 157 deletions(-) diff --git a/cfg/cfg.go b/cfg/cfg.go index 6c4a868..6f83934 100644 --- a/cfg/cfg.go +++ b/cfg/cfg.go @@ -37,6 +37,9 @@ var ( // BucketName is the bucket name of the Qiniu Cloud Kodo. BucketName string `mapstructure:"bucket_name"` + + // BucketEndpoint is the bucket endpint of the Qiniu Cloud Kodo. + BucketEndpoint string `mapstructure:"bucket_endpoint"` } // Goproxy is the Goproxy configuration items. diff --git a/config.toml b/config.toml index c68e7fe..9324fbc 100644 --- a/config.toml +++ b/config.toml @@ -18,6 +18,7 @@ endpoint = "" access_key = "" secret_key = "" bucket_name = "" +bucket_endpoint = "" # Goproxy [goproxy] diff --git a/go.mod b/go.mod index 5de31fa..d162769 100644 --- a/go.mod +++ b/go.mod @@ -20,4 +20,5 @@ require ( golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect golang.org/x/sys v0.0.0-20190830142957-1e83adbbebd0 // indirect google.golang.org/appengine v1.6.2 // indirect + gopkg.in/ini.v1 v1.46.0 // indirect ) diff --git a/go.sum b/go.sum index 68cc459..63582d8 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,7 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/goproxy/goproxy v0.0.0-20190823142913-1b8111e773cd h1:pnZ9UAPOipKlBEUflG4JVlBx2M1Qrwvsq9FX7wcz0mY= github.com/goproxy/goproxy v0.0.0-20190823142913-1b8111e773cd/go.mod h1:9dijhHHhLCeanFRtcBcQz71gAAug0Idr86zcC8ru02A= @@ -58,6 +59,7 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= github.com/minio/cli v1.20.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= +github.com/minio/minio-go/v6 v6.0.34 h1:ESPDlIg8Pe2BRvsxPomd0xB72uLmsXrkDNoze36yb90= github.com/minio/minio-go/v6 v6.0.34/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS+b04Boc66Y= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -76,7 +78,9 @@ github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4 github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -148,7 +152,10 @@ google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.46.0 h1:VeDZbLYGaupuvIrsYCEOe/L/2Pcs5n7hdO1ZTjporag= +gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handler/handler.go b/handler/handler.go index c72b533..2867b64 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -2,10 +2,6 @@ package handler import ( "context" - "crypto/md5" - "encoding/hex" - "errors" - "fmt" "hash" "io" "io/ioutil" @@ -19,6 +15,7 @@ import ( "github.com/aofei/air" "github.com/goproxy/goproxy" "github.com/goproxy/goproxy.cn/cfg" + "github.com/goproxy/goproxy/cacher" "github.com/qiniu/api.v7/auth/qbox" "github.com/qiniu/api.v7/storage" "github.com/rs/zerolog/log" @@ -34,9 +31,6 @@ var ( // kodoMac is the credentials of the Qiniu Cloud Kodo. kodoMac *qbox.Mac - // kodoConfig is the configuration of the Qiniu Cloud Kodo. - kodoConfig *storage.Config - // kodoBucketManager is the manager of the Qiniu Cloud Kodo. kodoBucketManager *storage.BucketManager @@ -53,7 +47,15 @@ var ( func init() { g.GoBinName = cfg.Goproxy.GoBinName - g.Cacher = &cacher{} + g.Cacher = &kodoCacher{ + kodoCacher: &cacher.Kodo{ + Endpoint: cfg.Kodo.Endpoint, + AccessKey: cfg.Kodo.AccessKey, + SecretKey: cfg.Kodo.SecretKey, + BucketName: cfg.Kodo.BucketName, + }, + } + g.ErrorLogger = a.ErrorLogger kodoMac = qbox.NewMac(cfg.Kodo.AccessKey, cfg.Kodo.SecretKey) @@ -68,11 +70,9 @@ func init() { Msg("failed to get qiniu cloud kodo region") } - kodoConfig = &storage.Config{ + kodoBucketManager = storage.NewBucketManager(kodoMac, &storage.Config{ Region: kodoRegion, - } - - kodoBucketManager = storage.NewBucketManager(kodoMac, kodoConfig) + }) a.FILE("/robots.txt", "robots.txt") a.FILE("/favicon.ico", "favicon.ico", cachemanGas) @@ -99,7 +99,7 @@ func goproxyHandler(req *air.Request, res *air.Response) error { } else if fi.Fsize > 10<<20 { // File size > 10 MB fu := storage.MakePrivateURL( kodoMac, - cfg.Kodo.Endpoint, + cfg.Kodo.BucketEndpoint, fk, time.Now().Add(time.Hour).Unix(), ) @@ -112,66 +112,27 @@ func goproxyHandler(req *air.Request, res *air.Response) error { return nil } -// cacher implements the `goproxy.Cacher`. -type cacher struct{} +// kodoCacher implements the `goproxy.Cacher`. +type kodoCacher struct { + // kodoCacher is the underlying cacher. + kodoCacher *cacher.Kodo +} // NewHash implements the `goproxy.Cacher`. -func (*cacher) NewHash() hash.Hash { - return md5.New() +func (kc *kodoCacher) NewHash() hash.Hash { + return kc.kodoCacher.NewHash() } // Cache implements the `goproxy.Cacher`. -func (*cacher) Cache(ctx context.Context, name string) (goproxy.Cache, error) { - fileInfo, err := kodoBucketManager.Stat(cfg.Kodo.BucketName, name) - if err != nil { - if isKodoFileNotExist(err) { - return nil, goproxy.ErrCacheNotFound - } - - return nil, err - } - - deadline, ok := ctx.Deadline() - if !ok { - deadline = time.Now().Add(time.Hour) - } - - fileURL := storage.MakePrivateURL( - kodoMac, - cfg.Kodo.Endpoint, - name, - deadline.Unix(), - ) - - req, err := http.NewRequest(http.MethodHead, fileURL, nil) - if err != nil { - return nil, err - } - - res, err := http.DefaultClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, err - } - defer res.Body.Close() - - checksum, err := hex.DecodeString(res.Header.Get("x-qn-meta-checksum")) - if err != nil { - return nil, err - } - - return &cache{ - ctx: ctx, - url: fileURL, - name: name, - mimeType: fileInfo.MimeType, - size: fileInfo.Fsize, - modTime: time.Unix(fileInfo.PutTime*100/int64(time.Second), 0), - checksum: checksum, - }, nil +func (kc *kodoCacher) Cache( + ctx context.Context, + name string, +) (goproxy.Cache, error) { + return kc.kodoCacher.Cache(ctx, name) } // SetCache implements the `goproxy.Cacher`. -func (*cacher) SetCache(ctx context.Context, c goproxy.Cache) error { +func (kc *kodoCacher) SetCache(ctx context.Context, c goproxy.Cache) error { localCache, err := ioutil.TempFile(cfg.Goproxy.LocalCacheRoot, "") if err != nil { return err @@ -202,34 +163,30 @@ func (*cacher) SetCache(ctx context.Context, c goproxy.Cache) error { ) defer cancel() - scope := fmt.Sprintf("%s:%s", cfg.Kodo.BucketName, c.Name()) - checksumString := hex.EncodeToString(c.Checksum()) - storage.NewFormUploader(kodoConfig).PutFile( - ctx, - nil, - (&storage.PutPolicy{ - Scope: scope, - }).UploadToken(kodoMac), - c.Name(), - localCache.Name(), - &storage.PutExtra{ - Params: map[string]string{ - "x-qn-meta-checksum": checksumString, - }, - MimeType: c.MIMEType(), - }, - ) + lc, err := os.Open(localCache.Name()) + if err != nil { + return + } + + dc := &diskCache{ + file: lc, + name: c.Name(), + mimeType: c.MIMEType(), + size: c.Size(), + modTime: c.ModTime(), + checksum: c.Checksum(), + } + defer dc.Close() + + kc.kodoCacher.SetCache(ctx, dc) }() return nil } -// cache implements the `goproxy.Cache`. It is the cache unit of the `cacher`. -type cache struct { - ctx context.Context - url string - offset int64 - closed bool +// diskCache implements the `goproxy.Cache`. +type diskCache struct { + file *os.File name string mimeType string size int64 @@ -238,96 +195,43 @@ type cache struct { } // Read implements the `goproxy.Cache`. -func (c *cache) Read(b []byte) (int, error) { - if c.closed { - return 0, os.ErrClosed - } else if c.offset >= c.size { - return 0, io.EOF - } - - req, err := http.NewRequest(http.MethodGet, c.url, nil) - if err != nil { - return 0, err - } - - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", c.offset)) - - res, err := http.DefaultClient.Do(req.WithContext(c.ctx)) - if err != nil { - return 0, err - } - defer res.Body.Close() - - n, err := res.Body.Read(b) - c.offset += int64(n) - - return n, err +func (dc *diskCache) Read(b []byte) (int, error) { + return dc.file.Read(b) } // Seek implements the `goproxy.Cache`. -func (c *cache) Seek(offset int64, whence int) (int64, error) { - if c.closed { - return 0, os.ErrClosed - } - - switch whence { - case io.SeekStart: - case io.SeekCurrent: - offset += c.offset - case io.SeekEnd: - offset += c.size - default: - return 0, errors.New("invalid whence") - } - - if offset < 0 { - return 0, errors.New("negative position") - } - - c.offset = offset - - return c.offset, nil +func (dc *diskCache) Seek(offset int64, whence int) (int64, error) { + return dc.file.Seek(offset, whence) } // Close implements the `goproxy.Cache`. -func (c *cache) Close() error { - if c.closed { - return os.ErrClosed - } - - c.closed = true - - return nil +func (dc *diskCache) Close() error { + return dc.file.Close() } // Name implements the `goproxy.Cache`. -func (c *cache) Name() string { - return c.name +func (dc *diskCache) Name() string { + return dc.name } // MIMEType implements the `goproxy.Cache`. -func (c *cache) MIMEType() string { - return c.mimeType +func (dc *diskCache) MIMEType() string { + return dc.mimeType } // Size implements the `goproxy.Cache`. -func (c *cache) Size() int64 { - return c.size +func (dc *diskCache) Size() int64 { + return dc.size } // ModTime implements the `goproxy.Cache`. -func (c *cache) ModTime() time.Time { - return c.modTime +func (dc *diskCache) ModTime() time.Time { + return dc.modTime } // Checksum implements the `goproxy.Cache`. -func (c *cache) Checksum() []byte { - return c.checksum -} - -// isKodoFileNotExist reports whether the err means a Kodo file is not exist. -func isKodoFileNotExist(err error) bool { - return err != nil && err.Error() == "no such file or directory" +func (dc *diskCache) Checksum() []byte { + return dc.checksum } // splitPathQuery splits the p of the form "path?query" into path and query. @@ -342,3 +246,8 @@ func splitPathQuery(p string) (path, query string) { return p, "" } + +// isKodoFileNotExist reports whether the err means a Kodo file is not exist. +func isKodoFileNotExist(err error) bool { + return err != nil && err.Error() == "no such file or directory" +}