Skip to content

Commit

Permalink
refactor: simplify caching
Browse files Browse the repository at this point in the history
  • Loading branch information
@aofei
aofei committed Aug 30, 2019
1 parent 5d59cde commit f58fa93
Show file tree
Hide file tree
Showing 5 changed files with 78 additions and 157 deletions.
3 changes: 3 additions & 0 deletions cfg/cfg.go
View file
Expand Up @@ -37,6 +37,9 @@ var (

// BucketName is the bucket name of the Qiniu Cloud Kodo.
BucketName string `mapstructure:"bucket_name"`

// BucketEndpoint is the bucket endpint of the Qiniu Cloud Kodo.
BucketEndpoint string `mapstructure:"bucket_endpoint"`
}

// Goproxy is the Goproxy configuration items.
Expand Down
1 change: 1 addition & 0 deletions config.toml
View file
Expand Up @@ -18,6 +18,7 @@ endpoint = "<ENDPOINT>"
access_key = "<ACCESS_KEY>"
secret_key = "<SECRET_KEY>"
bucket_name = "<BUCKET_NAME>"
bucket_endpoint = "<BUCKET_ENDPOINT>"

# Goproxy
[goproxy]
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Expand Up @@ -20,4 +20,5 @@ require (
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect
golang.org/x/sys v0.0.0-20190830142957-1e83adbbebd0 // indirect
google.golang.org/appengine v1.6.2 // indirect
gopkg.in/ini.v1 v1.46.0 // indirect
)
7 changes: 7 additions & 0 deletions go.sum
View file
Expand Up @@ -41,6 +41,7 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/goproxy/goproxy v0.0.0-20190823142913-1b8111e773cd h1:pnZ9UAPOipKlBEUflG4JVlBx2M1Qrwvsq9FX7wcz0mY=
github.com/goproxy/goproxy v0.0.0-20190823142913-1b8111e773cd/go.mod h1:9dijhHHhLCeanFRtcBcQz71gAAug0Idr86zcC8ru02A=
Expand All @@ -58,6 +59,7 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
github.com/minio/cli v1.20.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
github.com/minio/minio-go/v6 v6.0.34 h1:ESPDlIg8Pe2BRvsxPomd0xB72uLmsXrkDNoze36yb90=
github.com/minio/minio-go/v6 v6.0.34/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS+b04Boc66Y=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
Expand All @@ -76,7 +78,9 @@ github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4
github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY=
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
Expand Down Expand Up @@ -148,7 +152,10 @@ google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.46.0 h1:VeDZbLYGaupuvIrsYCEOe/L/2Pcs5n7hdO1ZTjporag=
gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
223 changes: 66 additions & 157 deletions handler/handler.go
View file
Expand Up @@ -2,10 +2,6 @@ package handler

import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"io/ioutil"
Expand All @@ -19,6 +15,7 @@ import (
"github.com/aofei/air"
"github.com/goproxy/goproxy"
"github.com/goproxy/goproxy.cn/cfg"
"github.com/goproxy/goproxy/cacher"
"github.com/qiniu/api.v7/auth/qbox"
"github.com/qiniu/api.v7/storage"
"github.com/rs/zerolog/log"
Expand All @@ -34,9 +31,6 @@ var (
// kodoMac is the credentials of the Qiniu Cloud Kodo.
kodoMac *qbox.Mac

// kodoConfig is the configuration of the Qiniu Cloud Kodo.
kodoConfig *storage.Config

// kodoBucketManager is the manager of the Qiniu Cloud Kodo.
kodoBucketManager *storage.BucketManager

Expand All @@ -53,7 +47,15 @@ var (

func init() {
g.GoBinName = cfg.Goproxy.GoBinName
g.Cacher = &cacher{}
g.Cacher = &kodoCacher{
kodoCacher: &cacher.Kodo{
Endpoint: cfg.Kodo.Endpoint,
AccessKey: cfg.Kodo.AccessKey,
SecretKey: cfg.Kodo.SecretKey,
BucketName: cfg.Kodo.BucketName,
},
}

g.ErrorLogger = a.ErrorLogger

kodoMac = qbox.NewMac(cfg.Kodo.AccessKey, cfg.Kodo.SecretKey)
Expand All @@ -68,11 +70,9 @@ func init() {
Msg("failed to get qiniu cloud kodo region")
}

kodoConfig = &storage.Config{
kodoBucketManager = storage.NewBucketManager(kodoMac, &storage.Config{
Region: kodoRegion,
}

kodoBucketManager = storage.NewBucketManager(kodoMac, kodoConfig)
})

a.FILE("/robots.txt", "robots.txt")
a.FILE("/favicon.ico", "favicon.ico", cachemanGas)
Expand All @@ -99,7 +99,7 @@ func goproxyHandler(req *air.Request, res *air.Response) error {
} else if fi.Fsize > 10<<20 { // File size > 10 MB
fu := storage.MakePrivateURL(
kodoMac,
cfg.Kodo.Endpoint,
cfg.Kodo.BucketEndpoint,
fk,
time.Now().Add(time.Hour).Unix(),
)
Expand All @@ -112,66 +112,27 @@ func goproxyHandler(req *air.Request, res *air.Response) error {
return nil
}

// cacher implements the `goproxy.Cacher`.
type cacher struct{}
// kodoCacher implements the `goproxy.Cacher`.
type kodoCacher struct {
// kodoCacher is the underlying cacher.
kodoCacher *cacher.Kodo
}

// NewHash implements the `goproxy.Cacher`.
func (*cacher) NewHash() hash.Hash {
return md5.New()
func (kc *kodoCacher) NewHash() hash.Hash {
return kc.kodoCacher.NewHash()
}

// Cache implements the `goproxy.Cacher`.
func (*cacher) Cache(ctx context.Context, name string) (goproxy.Cache, error) {
fileInfo, err := kodoBucketManager.Stat(cfg.Kodo.BucketName, name)
if err != nil {
if isKodoFileNotExist(err) {
return nil, goproxy.ErrCacheNotFound
}

return nil, err
}

deadline, ok := ctx.Deadline()
if !ok {
deadline = time.Now().Add(time.Hour)
}

fileURL := storage.MakePrivateURL(
kodoMac,
cfg.Kodo.Endpoint,
name,
deadline.Unix(),
)

req, err := http.NewRequest(http.MethodHead, fileURL, nil)
if err != nil {
return nil, err
}

res, err := http.DefaultClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
}
defer res.Body.Close()

checksum, err := hex.DecodeString(res.Header.Get("x-qn-meta-checksum"))
if err != nil {
return nil, err
}

return &cache{
ctx: ctx,
url: fileURL,
name: name,
mimeType: fileInfo.MimeType,
size: fileInfo.Fsize,
modTime: time.Unix(fileInfo.PutTime*100/int64(time.Second), 0),
checksum: checksum,
}, nil
func (kc *kodoCacher) Cache(
ctx context.Context,
name string,
) (goproxy.Cache, error) {
return kc.kodoCacher.Cache(ctx, name)
}

// SetCache implements the `goproxy.Cacher`.
func (*cacher) SetCache(ctx context.Context, c goproxy.Cache) error {
func (kc *kodoCacher) SetCache(ctx context.Context, c goproxy.Cache) error {
localCache, err := ioutil.TempFile(cfg.Goproxy.LocalCacheRoot, "")
if err != nil {
return err
Expand Down Expand Up @@ -202,34 +163,30 @@ func (*cacher) SetCache(ctx context.Context, c goproxy.Cache) error {
)
defer cancel()

scope := fmt.Sprintf("%s:%s", cfg.Kodo.BucketName, c.Name())
checksumString := hex.EncodeToString(c.Checksum())
storage.NewFormUploader(kodoConfig).PutFile(
ctx,
nil,
(&storage.PutPolicy{
Scope: scope,
}).UploadToken(kodoMac),
c.Name(),
localCache.Name(),
&storage.PutExtra{
Params: map[string]string{
"x-qn-meta-checksum": checksumString,
},
MimeType: c.MIMEType(),
},
)
lc, err := os.Open(localCache.Name())
if err != nil {
return
}

dc := &diskCache{
file: lc,
name: c.Name(),
mimeType: c.MIMEType(),
size: c.Size(),
modTime: c.ModTime(),
checksum: c.Checksum(),
}
defer dc.Close()

kc.kodoCacher.SetCache(ctx, dc)
}()

return nil
}

// cache implements the `goproxy.Cache`. It is the cache unit of the `cacher`.
type cache struct {
ctx context.Context
url string
offset int64
closed bool
// diskCache implements the `goproxy.Cache`.
type diskCache struct {
file *os.File
name string
mimeType string
size int64
Expand All @@ -238,96 +195,43 @@ type cache struct {
}

// Read implements the `goproxy.Cache`.
func (c *cache) Read(b []byte) (int, error) {
if c.closed {
return 0, os.ErrClosed
} else if c.offset >= c.size {
return 0, io.EOF
}

req, err := http.NewRequest(http.MethodGet, c.url, nil)
if err != nil {
return 0, err
}

req.Header.Set("Range", fmt.Sprintf("bytes=%d-", c.offset))

res, err := http.DefaultClient.Do(req.WithContext(c.ctx))
if err != nil {
return 0, err
}
defer res.Body.Close()

n, err := res.Body.Read(b)
c.offset += int64(n)

return n, err
func (dc *diskCache) Read(b []byte) (int, error) {
return dc.file.Read(b)
}

// Seek implements the `goproxy.Cache`.
func (c *cache) Seek(offset int64, whence int) (int64, error) {
if c.closed {
return 0, os.ErrClosed
}

switch whence {
case io.SeekStart:
case io.SeekCurrent:
offset += c.offset
case io.SeekEnd:
offset += c.size
default:
return 0, errors.New("invalid whence")
}

if offset < 0 {
return 0, errors.New("negative position")
}

c.offset = offset

return c.offset, nil
func (dc *diskCache) Seek(offset int64, whence int) (int64, error) {
return dc.file.Seek(offset, whence)
}

// Close implements the `goproxy.Cache`.
func (c *cache) Close() error {
if c.closed {
return os.ErrClosed
}

c.closed = true

return nil
func (dc *diskCache) Close() error {
return dc.file.Close()
}

// Name implements the `goproxy.Cache`.
func (c *cache) Name() string {
return c.name
func (dc *diskCache) Name() string {
return dc.name
}

// MIMEType implements the `goproxy.Cache`.
func (c *cache) MIMEType() string {
return c.mimeType
func (dc *diskCache) MIMEType() string {
return dc.mimeType
}

// Size implements the `goproxy.Cache`.
func (c *cache) Size() int64 {
return c.size
func (dc *diskCache) Size() int64 {
return dc.size
}

// ModTime implements the `goproxy.Cache`.
func (c *cache) ModTime() time.Time {
return c.modTime
func (dc *diskCache) ModTime() time.Time {
return dc.modTime
}

// Checksum implements the `goproxy.Cache`.
func (c *cache) Checksum() []byte {
return c.checksum
}

// isKodoFileNotExist reports whether the err means a Kodo file is not exist.
func isKodoFileNotExist(err error) bool {
return err != nil && err.Error() == "no such file or directory"
func (dc *diskCache) Checksum() []byte {
return dc.checksum
}

// splitPathQuery splits the p of the form "path?query" into path and query.
Expand All @@ -342,3 +246,8 @@ func splitPathQuery(p string) (path, query string) {

return p, ""
}

// isKodoFileNotExist reports whether the err means a Kodo file is not exist.
func isKodoFileNotExist(err error) bool {
return err != nil && err.Error() == "no such file or directory"
}

0 comments on commit f58fa93

Please sign in to comment.