Switch to official mongo driver (#36)

* migration to the official mongo driver

* add injected timeout to gridfs store

* lint: minor warnings

* switch to go 1.13 and latest linter

.golangci.yml

@@ -69,4 +69,4 @@ issues:
   exclude-use-default: false
 
 service:
-  golangci-lint-version: 1.15.x
+  golangci-lint-version: 1.19.x

.travis.yml

@@ -4,21 +4,21 @@ services:
   - mongodb
 
 go:
-  - "1.12.x"
+  - "1.13.x"
 
 install: true
 
 before_install:
   - export TZ=America/Chicago
-  - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.17.1
+  - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.19.1
   - golangci-lint --version
   - go get github.com/mattn/goveralls
   - export MONGO_TEST=mongodb://127.0.0.1:27017
   - export PATH=$(pwd)/bin:$PATH
 
 script:
-  - GO111MODULE=on go get ./...
-  - GO111MODULE=on go mod vendor
-  - GO111MODULE=on go test -v -mod=vendor -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1;
+  - go get ./...
+  - go mod vendor
+  - go test -v -mod=vendor -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1;
   - golangci-lint run --tests=false  || travis_terminate 1;
   - $GOPATH/bin/goveralls -coverprofile=profile.cov -service=travis-ci

_example/go.sum

@@ -51,12 +51,14 @@ github.com/go-pkgz/rest v1.4.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0
 github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
 github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
 github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -117,13 +119,16 @@ github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7
 github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
 github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
 github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e h1:+NL1GDIUOKxVfbp2KoJQD9cTQ6dyP2co9q4yzmT9FZo=
 github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
 github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
 github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.0.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
@@ -132,6 +137,7 @@ github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FB
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

avatar/bolt.go

@@ -2,8 +2,6 @@ package avatar
 
 import (
 	"bytes"
-	"crypto/sha1"
-	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -58,7 +56,7 @@ func (b *BoltDB) Put(userID string, reader io.Reader) (avatar string, err error)
 			return errors.Wrapf(err, "can't put to bucket with %s", avatarID)
 		}
 		// store sha1 of the image
-		return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(b.sha1(buf.Bytes(), avatarID)))
+		return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(hash(buf.Bytes(), avatarID)))
 	})
 	return avatarID, err
 }
@@ -130,12 +128,3 @@ func (b *BoltDB) Close() error {
 func (b *BoltDB) String() string {
 	return fmt.Sprintf("boltdb, path=%s", b.fileName)
 }
-
-func (b *BoltDB) sha1(data []byte, avatarID string) (id string) {
-	h := sha1.New()
-	if _, err := h.Write(data); err != nil {
-		log.Printf("[DEBUG] can't apply sha1 for content of '%s', %s", avatarID, err)
-		return encodeID(avatarID)
-	}
-	return hex.EncodeToString(h.Sum(nil))
-}

avatar/gridfs.go

@@ -2,122 +2,153 @@ package avatar
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"time"
 
-	"github.com/globalsign/mgo"
-	"github.com/go-pkgz/mongo"
 	"github.com/pkg/errors"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/gridfs"
+	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
 // NewGridFS makes gridfs (mongo) avatar store
-func NewGridFS(conn *mongo.Connection) *GridFS {
-	return &GridFS{Connection: conn}
+func NewGridFS(client *mongo.Client, dbName, bucketName string, timeout time.Duration) *GridFS {
+	return &GridFS{client: client, db: client.Database(dbName), bucketName: bucketName, timeout: timeout}
 }
 
 // GridFS implements Store for GridFS
 type GridFS struct {
-	Connection *mongo.Connection
+	client     *mongo.Client
+	db         *mongo.Database
+	bucketName string
+	timeout    time.Duration
 }
 
 // Put avatar to gridfs object, try to resize
 func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error) {
 	id := encodeID(userID)
-	err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
-		fh, e := dbase.GridFS("fs").Create(id + imgSfx)
-		if e != nil {
-			return e
-		}
-		defer func() {
-			if err = fh.Close(); err != nil {
-				log.Printf("[WARN] can't close avatar file %v, %s", fh, err)
-			}
-		}()
-
-		_, e = io.Copy(fh, reader)
-		return e
-	})
+	bucket, err := gridfs.NewBucket(gf.db, &options.BucketOptions{Name: &gf.bucketName})
+	if err != nil {
+		return "", err
+	}
+
+	buf := &bytes.Buffer{}
+	if _, err = io.Copy(buf, reader); err != nil {
+		return "", errors.Wrapf(err, "can't read avatar for %s", userID)
+	}
+
+	avaHash := hash(buf.Bytes(), id)
+	_, err = bucket.UploadFromStream(id+imgSfx, buf, &options.UploadOptions{Metadata: bson.M{"hash": avaHash}})
 	return id + imgSfx, err
 }
 
 // Get avatar reader for avatar id.image
 func (gf *GridFS) Get(avatar string) (reader io.ReadCloser, size int, err error) {
+	bucket, err := gridfs.NewBucket(gf.db, &options.BucketOptions{Name: &gf.bucketName})
+	if err != nil {
+		return nil, 0, err
+	}
 	buf := &bytes.Buffer{}
-	err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
-		fh, e := dbase.GridFS("fs").Open(avatar)
-		if e != nil {
-			return errors.Wrapf(e, "can't load avatar %s", avatar)
-		}
-		if _, e = io.Copy(buf, fh); e != nil {
-			return errors.Wrapf(e, "can't copy avatar %s", avatar)
-		}
-		size = int(fh.Size())
-		return fh.Close()
-	})
-	return ioutil.NopCloser(buf), size, err
+	sz, e := bucket.DownloadToStreamByName(avatar, buf)
+	return ioutil.NopCloser(buf), int(sz), errors.Wrapf(e, "can't read avatar %s", avatar)
 }
 
+//
 // ID returns a fingerprint of the avatar content. Uses MD5 because gridfs provides it directly
 func (gf *GridFS) ID(avatar string) (id string) {
-	err := gf.Connection.WithDB(func(dbase *mgo.Database) error {
-		fh, e := dbase.GridFS("fs").Open(avatar)
-		if e != nil {
-			return errors.Wrapf(e, "can't open avatar %s", avatar)
-		}
-		id = fh.MD5()
-		return errors.Wrapf(fh.Close(), "can't close avatar")
-	})
+
+	finfo := struct {
+		ID       primitive.ObjectID `bson:"_id"`
+		Len      int                `bson:"length"`
+		FileName string             `bson:"filename"`
+		MetaData struct {
+			Hash string `bson:"hash"`
+		} `bson:"metadata"`
+	}{}
+
+	bucket, err := gridfs.NewBucket(gf.db, &options.BucketOptions{Name: &gf.bucketName})
+	if err != nil {
+		return encodeID(avatar)
+	}
+	cursor, err := bucket.Find(bson.M{"filename": avatar})
 	if err != nil {
-		log.Printf("[DEBUG] can't get file info '%s', %s", avatar, err)
 		return encodeID(avatar)
 	}
-	return id
+
+	ctx, cancel := context.WithTimeout(context.Background(), gf.timeout)
+	defer cancel()
+	if found := cursor.Next(ctx); found {
+		if err = cursor.Decode(&finfo); err != nil {
+			return encodeID(avatar)
+		}
+		return finfo.MetaData.Hash
+	}
+	return encodeID(avatar)
 }
 
 // Remove avatar from gridfs
 func (gf *GridFS) Remove(avatar string) error {
-	return gf.Connection.WithDB(func(dbase *mgo.Database) error {
-		fh, e := dbase.GridFS("fs").Open(avatar)
-		if e != nil {
-			return errors.Wrapf(e, "can't get avatar %s", avatar)
-		}
-		if e = fh.Close(); e != nil {
-			log.Printf("[WARN] can't close avatar %s, %s", avatar, e)
+	bucket, err := gridfs.NewBucket(gf.db, &options.BucketOptions{Name: &gf.bucketName})
+	if err != nil {
+		return err
+	}
+	cursor, err := bucket.Find(bson.M{"filename": avatar})
+	if err != nil {
+		return err
+	}
+
+	r := struct {
+		ID primitive.ObjectID `bson:"_id"`
+	}{}
+	ctx, cancel := context.WithTimeout(context.Background(), gf.timeout)
+	defer cancel()
+	if found := cursor.Next(ctx); found {
+		if err := cursor.Decode(&r); err != nil {
+			return err
 		}
-		return dbase.GridFS("fs").Remove(avatar)
-	})
+		return bucket.Delete(r.ID)
+	}
+	return errors.Errorf("avatar %s not found", avatar)
 }
 
 // List all avatars (ids) on gfs
 // note: id includes .image suffix
 func (gf *GridFS) List() (ids []string, err error) {
-
-	type gfsFile struct {
-		UploadDate time.Time `bson:"uploadDate"`
-		Length     int64     `bson:",minsize"`
-		MD5        string
-		Filename   string `bson:",omitempty"`
+	bucket, err := gridfs.NewBucket(gf.db, &options.BucketOptions{Name: &gf.bucketName})
+	if err != nil {
+		return nil, err
 	}
 
-	files := []gfsFile{}
-	err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
-		return dbase.GridFS("fs").Find(nil).All(&files)
-	})
-
-	for _, f := range files {
-		ids = append(ids, f.Filename)
+	gfsFile := struct {
+		Filename string `bson:"filename,omitempty"`
+	}{}
+	cursor, err := bucket.Find(bson.M{})
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), gf.timeout)
+	defer cancel()
+	for cursor.Next(ctx) {
+		if err := cursor.Decode(&gfsFile); err != nil {
+			return nil, err
+		}
+		ids = append(ids, gfsFile.Filename)
 	}
-	return ids, errors.Wrap(err, "can't list avatars")
+	return ids, nil
 }
 
 // Close gridfs does nothing but satisfies interface
 func (gf *GridFS) Close() error {
-	return nil
+	ctx, cancel := context.WithTimeout(context.Background(), gf.timeout)
+	defer cancel()
+	return gf.client.Disconnect(ctx)
 }
 
 func (gf *GridFS) String() string {
-	return fmt.Sprintf("mongo (grid fs), conn=%s", gf.Connection)
+	return fmt.Sprintf("mongo (grid fs), db=%s, bucket=%s", gf.db.Name(), gf.bucketName)
 }