diff --git a/README.md b/README.md index 13fdb2158..2dca5287d 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,13 @@ docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0. ``` +To link to local gosdk so that the changes are reflected on the blobber build please use the below command + +``` + +$ ./docker.local/bin/build.blobber.dev.sh + +``` For Apple M1 chip builds: ``` diff --git a/code/go/0chain.net/blobbercore/datastore/store.go b/code/go/0chain.net/blobbercore/datastore/store.go index 497ee6b55..529a3016a 100644 --- a/code/go/0chain.net/blobbercore/datastore/store.go +++ b/code/go/0chain.net/blobbercore/datastore/store.go @@ -5,9 +5,9 @@ import ( "fmt" "time" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/errors" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "gorm.io/driver/postgres" "gorm.io/gorm" @@ -39,12 +39,12 @@ func (store *Store) Open() error { config.Configuration.DBUserName, config.Configuration.DBName, config.Configuration.DBPassword)), &gorm.Config{}) if err != nil { - return errors.DBOpenError + return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) } sqldb, err := db.DB() if err != nil { - return errors.DBOpenError + return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) } sqldb.SetMaxIdleConns(100) diff --git a/code/go/0chain.net/blobbercore/errors/errors.go b/code/go/0chain.net/blobbercore/errors/errors.go deleted file mode 100644 index acd03356f..000000000 --- a/code/go/0chain.net/blobbercore/errors/errors.go +++ /dev/null @@ -1,10 +0,0 @@ -package errors - -import ( - "github.com/0chain/blobber/code/go/0chain.net/core/common" -) - -var ( - //DBOpenError - Error opening the db - DBOpenError = common.NewError("db_open_error", "Error opening the DB connection") -) diff --git a/code/go/0chain.net/blobbercore/filestore/fs_store.go b/code/go/0chain.net/blobbercore/filestore/fs_store.go index 64e43b492..728b703f2 100644 --- a/code/go/0chain.net/blobbercore/filestore/fs_store.go +++ b/code/go/0chain.net/blobbercore/filestore/fs_store.go @@ -45,9 +45,66 @@ type MinioConfiguration struct { var MinioConfig MinioConfiguration +type IFileBlockGetter interface { + GetFileBlock(fsStore *FileFSStore, allocationID string, fileData *FileInputData, blockNum int64, numBlocks int64) ([]byte, error) +} + +type FileBlockGetter struct { +} + +func (FileBlockGetter) GetFileBlock(fs *FileFSStore, allocationID string, fileData *FileInputData, blockNum int64, numBlocks int64) ([]byte, error) { + allocation, err := fs.SetupAllocation(allocationID, true) + if err != nil { + return nil, common.NewError("invalid_allocation", "Invalid allocation. "+err.Error()) + } + dirPath, destFile := GetFilePathFromHash(fileData.Hash) + fileObjectPath := filepath.Join(allocation.ObjectsPath, dirPath) + fileObjectPath = filepath.Join(fileObjectPath, destFile) + + file, err := os.Open(fileObjectPath) + if err != nil { + if os.IsNotExist(err) && fileData.OnCloud { + err = fs.DownloadFromCloud(fileData.Hash, fileObjectPath) + if err != nil { + return nil, common.NewError("minio_download_failed", "Unable to download from minio with err "+err.Error()) + } + file, err = os.Open(fileObjectPath) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + defer file.Close() + fileinfo, err := file.Stat() + if err != nil { + return nil, err + } + + filesize := int(fileinfo.Size()) + maxBlockNum := int64(filesize / CHUNK_SIZE) + // check for any left over bytes. Add one more go routine if required. + if remainder := filesize % CHUNK_SIZE; remainder != 0 { + maxBlockNum++ + } + + if blockNum > maxBlockNum || blockNum < 1 { + return nil, common.NewError("invalid_block_number", "Invalid block number") + } + buffer := make([]byte, CHUNK_SIZE*numBlocks) + n, err := file.ReadAt(buffer, ((blockNum - 1) * CHUNK_SIZE)) + if err != nil && err != io.EOF { + return nil, err + } + + return buffer[:n], nil +} + type FileFSStore struct { - RootDirectory string - Minio *minio.Client + RootDirectory string + Minio *minio.Client + fileBlockGetter IFileBlockGetter } type StoreAllocation struct { @@ -61,9 +118,14 @@ func SetupFSStore(rootDir string) (FileStore, error) { if err := createDirs(rootDir); err != nil { return nil, err } + return SetupFSStoreI(rootDir, FileBlockGetter{}) +} + +func SetupFSStoreI(rootDir string, fileBlockGetter IFileBlockGetter) (FileStore, error) { fsStore = &FileFSStore{ - RootDirectory: rootDir, - Minio: intializeMinio(), + RootDirectory: rootDir, + Minio: intializeMinio(), + fileBlockGetter: fileBlockGetter, } return fsStore, nil } @@ -280,52 +342,7 @@ func (fs *FileFSStore) GetFileBlockForChallenge(allocationID string, fileData *F } func (fs *FileFSStore) GetFileBlock(allocationID string, fileData *FileInputData, blockNum int64, numBlocks int64) ([]byte, error) { - allocation, err := fs.SetupAllocation(allocationID, true) - if err != nil { - return nil, common.NewError("invalid_allocation", "Invalid allocation. "+err.Error()) - } - dirPath, destFile := GetFilePathFromHash(fileData.Hash) - fileObjectPath := filepath.Join(allocation.ObjectsPath, dirPath) - fileObjectPath = filepath.Join(fileObjectPath, destFile) - - file, err := os.Open(fileObjectPath) - if err != nil { - if os.IsNotExist(err) && fileData.OnCloud { - err = fs.DownloadFromCloud(fileData.Hash, fileObjectPath) - if err != nil { - return nil, common.NewError("minio_download_failed", "Unable to download from minio with err "+err.Error()) - } - file, err = os.Open(fileObjectPath) - if err != nil { - return nil, err - } - } else { - return nil, err - } - } - defer file.Close() - fileinfo, err := file.Stat() - if err != nil { - return nil, err - } - - filesize := int(fileinfo.Size()) - maxBlockNum := int64(filesize / CHUNK_SIZE) - // check for any left over bytes. Add one more go routine if required. - if remainder := filesize % CHUNK_SIZE; remainder != 0 { - maxBlockNum++ - } - - if blockNum > maxBlockNum || blockNum < 1 { - return nil, common.NewError("invalid_block_number", "Invalid block number") - } - buffer := make([]byte, CHUNK_SIZE*numBlocks) - n, err := file.ReadAt(buffer, ((blockNum - 1) * CHUNK_SIZE)) - if err != nil && err != io.EOF { - return nil, err - } - - return buffer[:n], nil + return fs.fileBlockGetter.GetFileBlock(fs, allocationID, fileData, blockNum, numBlocks) } func (fs *FileFSStore) DeleteTempFile(allocationID string, fileData *FileInputData, connectionID string) error { diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 9e92f37e1..c2404b3d6 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -4,11 +4,18 @@ package handler import ( "context" + "encoding/json" + "errors" "net/http" "os" "runtime/pprof" "time" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/gosdk/zboxcore/fileref" + "gorm.io/gorm" + "go.uber.org/zap" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" @@ -59,6 +66,9 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/_statsJSON", common.UserRateLimit(common.ToJSONResponse(stats.StatsJSONHandler))) r.HandleFunc("/_cleanupdisk", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(CleanupDiskHandler)))) r.HandleFunc("/getstats", common.UserRateLimit(common.ToJSONResponse(stats.GetStatsHandler))) + + //marketplace related + r.HandleFunc("/v1/marketplace/shareinfo/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithConnection(MarketPlaceShareInfoHandler)))) } func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { @@ -329,3 +339,138 @@ func CleanupDiskHandler(ctx context.Context, r *http.Request) (interface{}, erro err := CleanupDiskFiles(ctx) return "cleanup", err } + +func RevokeShare(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) + + allocationID := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationObj, err := storageHandler.verifyAllocation(ctx, allocationID, true) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) + } + + sign := r.Header.Get(common.ClientSignatureHeader) + allocation, ok := mux.Vars(r)["allocation"] + if !ok { + return false, common.NewError("invalid_params", "Missing allocation tx") + } + valid, err := verifySignatureFromRequest(allocation, sign, allocationObj.OwnerPublicKey) + if !valid || err != nil { + return nil, common.NewError("invalid_signature", "Invalid signature") + } + + path := r.FormValue("path") + refereeClientID := r.FormValue("refereeClientID") + filePathHash := fileref.GetReferenceLookup(allocationID, path) + _, err = reference.GetReferenceFromLookupHash(ctx, allocationID, filePathHash) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) + } + clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + if clientID != allocationObj.OwnerID { + return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + } + err = reference.DeleteShareInfo(ctx, reference.ShareInfo{ + ClientID: refereeClientID, + FilePathHash: filePathHash, + }) + if errors.Is(err, gorm.ErrRecordNotFound) { + resp := map[string]interface{}{ + "status": http.StatusNotFound, + "message": "Path not found", + } + return resp, nil + } + if err != nil { + return nil, err + } + resp := map[string]interface{}{ + "status": http.StatusNoContent, + "message": "Path successfully removed from allocation", + } + return resp, nil +} + +func InsertShare(ctx context.Context, r *http.Request) (interface{}, error) { + ctx = setupHandlerContext(ctx, r) + + allocationID := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationObj, err := storageHandler.verifyAllocation(ctx, allocationID, true) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) + } + + sign := r.Header.Get(common.ClientSignatureHeader) + allocation, ok := mux.Vars(r)["allocation"] + if !ok { + return false, common.NewError("invalid_params", "Missing allocation tx") + } + valid, err := verifySignatureFromRequest(allocation, sign, allocationObj.OwnerPublicKey) + if !valid || err != nil { + return nil, common.NewError("invalid_signature", "Invalid signature") + } + + encryptionPublicKey := r.FormValue("encryption_public_key") + authTicketString := r.FormValue("auth_ticket") + authTicket := &readmarker.AuthTicket{} + + err = json.Unmarshal([]byte(authTicketString), &authTicket) + if err != nil { + return false, common.NewError("invalid_parameters", "Error parsing the auth ticket for download."+err.Error()) + } + + fileref, err := reference.GetReferenceFromLookupHash(ctx, allocationID, authTicket.FilePathHash) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) + } + + authTicketVerified, err := storageHandler.verifyAuthTicket(ctx, authTicketString, allocationObj, fileref, authTicket.ClientID) + if !authTicketVerified { + return nil, common.NewError("auth_ticket_verification_failed", "Could not verify the auth ticket. "+err.Error()) + } + + if err != nil { + return nil, err + } + + shareInfo := reference.ShareInfo{ + OwnerID: authTicket.OwnerID, + ClientID: authTicket.ClientID, + FilePathHash: authTicket.FilePathHash, + ReEncryptionKey: authTicket.ReEncryptionKey, + ClientEncryptionPublicKey: encryptionPublicKey, + ExpiryAt: common.ToTime(authTicket.Expiration), + } + + existingShare, err := reference.GetShareInfo(ctx, authTicket.ClientID, authTicket.FilePathHash) + if err != nil { + return nil, err + } + + if existingShare != nil { + err = reference.UpdateShareInfo(ctx, shareInfo) + } else { + err = reference.AddShareInfo(ctx, shareInfo) + } + if err != nil { + return nil, err + } + + resp := map[string]interface{}{ + "message": "Share info added successfully", + } + + return resp, nil +} + +func MarketPlaceShareInfoHandler(ctx context.Context, r *http.Request) (interface{}, error) { + if r.Method == "DELETE" { + return RevokeShare(ctx, r) + } + + if r.Method == "POST" { + return InsertShare(ctx, r) + } + + return nil, errors.New("invalid request method, only POST is allowed") +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index bb1ee5317..de830a258 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "fmt" "io" "mime/multipart" "net/http" @@ -13,6 +14,19 @@ import ( "testing" "time" + "github.com/0chain/gosdk/core/zcncrypto" + "github.com/0chain/gosdk/zboxcore/client" + zencryption "github.com/0chain/gosdk/zboxcore/encryption" + "github.com/0chain/gosdk/zboxcore/fileref" + "github.com/0chain/gosdk/zboxcore/marker" + "github.com/0chain/gosdk/zcncore" + "github.com/DATA-DOG/go-sqlmock" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "gorm.io/gorm" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" bconfig "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" @@ -23,23 +37,52 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/config" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/core/zcncrypto" - "github.com/0chain/gosdk/zcncore" - "github.com/DATA-DOG/go-sqlmock" - "github.com/gorilla/mux" - "github.com/stretchr/testify/assert" - "go.uber.org/zap" - "gorm.io/gorm" ) +type MockFileBlockGetter struct { + filestore.IFileBlockGetter +} + +var mockFileBlock []byte + +func (MockFileBlockGetter) GetFileBlock( + fsStore *filestore.FileFSStore, + allocationID string, + fileData *filestore.FileInputData, + blockNum int64, + numBlocks int64, +) ([]byte, error) { + return []byte(mockFileBlock), nil +} + +func setMockFileBlock(data []byte) { + mockFileBlock = data +} + +func resetMockFileBlock() { + mockFileBlock = []byte("mock") +} + +var encscheme zencryption.EncryptionScheme + +func setupEncryptionScheme() { + encscheme = zencryption.NewEncryptionScheme() + mnemonic := client.GetClient().Mnemonic + if err := encscheme.Initialize(mnemonic); err != nil { + panic("initialize encscheme") + } + encscheme.InitForEncryption("filetype:audio") +} + func init() { + resetMockFileBlock() common.ConfigRateLimits() chain.SetServerChain(&chain.Chain{}) config.Configuration.SignatureScheme = "bls0chain" logging.Logger = zap.NewNop() dir, _ := os.Getwd() - if _, err := filestore.SetupFSStore(dir + "/tmp"); err != nil { + if _, err := filestore.SetupFSStoreI(dir+"/tmp", MockFileBlockGetter{}); err != nil { panic(err) } bconfig.Configuration.MaxFileSize = int64(1 << 30) @@ -171,35 +214,92 @@ func setupHandlers() (*mux.Router, map[string]string) { ), ).Name(uName) + dPath := "/v1/file/download/{allocation}" + dName := "Download" + router.HandleFunc(dPath, common.UserRateLimit( + common.ToJSONResponse( + WithConnection(DownloadHandler), + ), + ), + ).Name(dName) + + sharePath := "/v1/marketplace/shareinfo/{allocation}" + shareName := "Share" + router.HandleFunc(sharePath, common.UserRateLimit( + common.ToJSONResponse( + WithReadOnlyConnection(MarketPlaceShareInfoHandler), + ), + ), + ).Name(shareName) + return router, map[string]string{ - opPath: opName, - rpPath: rpName, - sPath: sName, - otPath: otName, - collPath: collName, - rPath: rName, - cPath: cName, - aPath: aName, - uPath: uName, + opPath: opName, + rpPath: rpName, + sPath: sName, + otPath: otName, + collPath: collName, + rPath: rName, + cPath: cName, + aPath: aName, + uPath: uName, + sharePath: shareName, + dPath: dName, } } +func isEndpointRequireSignature(name string) bool { + switch name { + case "Download": + return false + default: + return true + } +} + func isEndpointAllowGetReq(name string) bool { switch name { - case "Stats", "Rename", "Copy", "Attributes", "Upload": + case "Stats", "Rename", "Copy", "Attributes", "Upload", "Share", "Download": return false default: return true } } +func GetAuthTicketForEncryptedFile(allocationID string, remotePath string, fileHash string, clientID string, encPublicKey string) (string, error) { + at := &marker.AuthTicket{} + at.AllocationID = allocationID + at.OwnerID = client.GetClientID() + at.ClientID = clientID + at.FileName = remotePath + at.FilePathHash = fileHash + at.ContentHash = "content_hash" + at.RefType = fileref.FILE + timestamp := int64(common.Now()) + at.Expiration = timestamp + 7776000 + at.Timestamp = timestamp + at.ReEncryptionKey = "regenkey" + err := at.Sign() + if err != nil { + return "", err + } + atBytes, err := json.Marshal(at) + if err != nil { + return "", err + } + return string(atBytes), nil +} + func TestHandlers_Requiring_Signature(t *testing.T) { setup(t) + clientJson := "{\"client_id\":\"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9\",\"client_key\":\"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083\",\"keys\":[{\"public_key\":\"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083\",\"private_key\":\"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907\"}],\"mnemonics\":\"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe\",\"version\":\"1.0\",\"date_created\":\"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805\"}" + require.NoError(t, client.PopulateClient(clientJson, "bls0chain")) + setupEncryptionScheme() router, handlers := setupHandlers() sch := zcncrypto.NewBLS0ChainScheme() + sch.Mnemonic = "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe" _, err := sch.GenerateKeys() if err != nil { t.Fatal(err) @@ -207,7 +307,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) { ts := time.Now().Add(time.Hour) alloc := makeTestAllocation(common.Timestamp(ts.Unix())) alloc.OwnerPublicKey = sch.GetPublicKey() - alloc.OwnerID = sch.GetPublicKey() + alloc.OwnerID = client.GetClientID() const ( path = "/path" @@ -225,12 +325,17 @@ func TestHandlers_Requiring_Signature(t *testing.T) { args args alloc *allocation.Allocation setupDbMock func(mock sqlmock.Sqlmock) + begin func() + end func() wantCode int wantBody string } ) negativeTests := make([]test, 0) for _, name := range handlers { + if !isEndpointRequireSignature(name) { + continue + } baseSetupDbMock := func(mock sqlmock.Sqlmock) { mock.ExpectBegin() @@ -1008,17 +1113,810 @@ func TestHandlers_Requiring_Signature(t *testing.T) { }, wantCode: http.StatusOK, }, - } - tests := append(positiveTests, negativeTests...) - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - mock := datastore.MockTheStore(t) - test.setupDbMock(mock) + { + name: "InsertShareInfo_OK_New_Share", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/marketplace/shareinfo/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } - router.ServeHTTP(test.args.w, test.args.r) + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + shareClientEncryptionPublicKey := "kkk" + shareClientID := "abcdefgh" + require.NoError(t, formWriter.WriteField("encryption_public_key", shareClientEncryptionPublicKey)) + remotePath := "/file.txt" + filePathHash := "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c" + authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, filePathHash, shareClientID, sch.GetPublicKey()) + if err != nil { + t.Fatal(err) + } + require.NoError(t, formWriter.WriteField("auth_ticket", authTicket)) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodPost, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + + return r + }(), + }, + alloc: alloc, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.Tx, "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). + WillReturnRows( + sqlmock.NewRows([]string{"path", "lookup_hash"}). + AddRow("/file.txt", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs("abcdefgh", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). + WillReturnRows(sqlmock.NewRows([]string{})) + aa := sqlmock.AnyArg() + + mock.ExpectExec(`INSERT INTO "marketplace_share_info"`). + WithArgs(client.GetClientID(), "abcdefgh", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c", "regenkey", aa, false, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + }, + wantCode: http.StatusOK, + wantBody: "{\"message\":\"Share info added successfully\"}\n", + }, + { + name: "UpdateShareInfo", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/marketplace/shareinfo/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + shareClientEncryptionPublicKey := "kkk" + shareClientID := "abcdefgh" + require.NoError(t, formWriter.WriteField("encryption_public_key", shareClientEncryptionPublicKey)) + remotePath := "/file.txt" + filePathHash := "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c" + authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, filePathHash, shareClientID, sch.GetPublicKey()) + if err != nil { + t.Fatal(err) + } + require.NoError(t, formWriter.WriteField("auth_ticket", authTicket)) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodPost, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + + return r + }(), + }, + alloc: alloc, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.Tx, "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). + WillReturnRows( + sqlmock.NewRows([]string{"path", "lookup_hash"}). + AddRow("/file.txt", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs("abcdefgh", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow("abcdefgh"), + ) + aa := sqlmock.AnyArg() + + mock.ExpectExec(`UPDATE "marketplace_share_info"`). + WithArgs("regenkey", "kkk", false, aa, "abcdefgh", "f15383a1130bd2fae1e52a7a15c432269eeb7def555f1f8b9b9a28bd9611362c"). + WillReturnResult(sqlmock.NewResult(0, 1)) + }, + wantCode: http.StatusOK, + wantBody: "{\"message\":\"Share info added successfully\"}\n", + }, + { + name: "RevokeShareInfo_OK_Existing_Share", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/marketplace/shareinfo/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + shareClientID := "abcdefgh" + remotePath := "/file.txt" + + require.NoError(t, formWriter.WriteField("refereeClientID", shareClientID)) + require.NoError(t, formWriter.WriteField("path", remotePath)) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodDelete, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + + return r + }(), + }, + alloc: alloc, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.Tx, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "lookup_hash"}). + AddRow("/file.txt", filePathHash), + ) + + mock.ExpectExec(regexp.QuoteMeta(`UPDATE "marketplace_share_info"`)). + WithArgs(true, "abcdefgh", filePathHash). + WillReturnResult(sqlmock.NewResult(0, 1)) + + }, + wantCode: http.StatusOK, + wantBody: "{\"message\":\"Path successfully removed from allocation\",\"status\":204}\n", + }, + { + name: "RevokeShareInfo_NotOK_For_Non_Existing_Share", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/marketplace/shareinfo/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + shareClientID := "abcdefgh" + remotePath := "/file.txt" + + require.NoError(t, formWriter.WriteField("refereeClientID", shareClientID)) + require.NoError(t, formWriter.WriteField("path", remotePath)) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodDelete, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + + return r + }(), + }, + alloc: alloc, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.Tx, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "lookup_hash"}). + AddRow("/file.txt", filePathHash), + ) + + mock.ExpectExec(regexp.QuoteMeta(`UPDATE "marketplace_share_info"`)). + WithArgs(true, "abcdefgh", filePathHash). + WillReturnResult(sqlmock.NewResult(0, 0)) + + }, + wantCode: http.StatusOK, + wantBody: "{\"message\":\"Path not found\",\"status\":404}\n", + }, + { + name: "DownloadFile_Record_Not_Found", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + remotePath := "/file.txt" + + require.NoError(t, formWriter.WriteField("path_hash", fileref.GetReferenceLookup(alloc.Tx, remotePath))) + require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) + rm := &marker.ReadMarker{} + rm.ClientID = client.GetClientID() + rm.ClientPublicKey = client.GetClientPublicKey() + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.OwnerID = client.GetClientID() + err = rm.Sign() + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodPost, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + + return r + }(), + }, + alloc: alloc, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash).WillReturnError(gorm.ErrRecordNotFound) + + }, + wantCode: http.StatusBadRequest, + wantBody: "{\"code\":\"download_file\",\"error\":\"download_file: invalid file path: record not found\"}\n\n", + }, + { + name: "DownloadFile_Unencrypted_return_file", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + remotePath := "/file.txt" + + require.NoError(t, formWriter.WriteField("path_hash", fileref.GetReferenceLookup(alloc.Tx, remotePath))) + require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) + rm := &marker.ReadMarker{} + rm.ClientID = client.GetClientID() + rm.ClientPublicKey = client.GetClientPublicKey() + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = client.GetClientID() + err = rm.Sign() + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodPost, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + + return r + }(), + }, + alloc: alloc, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "lookup_hash", "content_hash"}). + AddRow("/file.txt", "f", filePathHash, "abcd"), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(1) FROM "collaborators" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnError(gorm.ErrRecordNotFound) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow(client.GetClientID()), + ) + + aa := sqlmock.AnyArg() + + mock.ExpectExec(`UPDATE "read_markers"`). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + + mock.ExpectCommit() + }, + wantCode: http.StatusOK, + wantBody: "\"bW9jaw==\"\n", //base64encoded for mock string + }, + { + name: "DownloadFile_Encrypted_Permission_Denied_Unshared_File", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + remotePath := "/file.txt" + + pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) + require.NoError(t, formWriter.WriteField("path_hash", pathHash)) + require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) + authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, pathHash, client.GetClientID(), sch.GetPublicKey()) + if err != nil { + t.Fatal(err) + } + require.NoError(t, formWriter.WriteField("auth_token", authTicket)) + rm := &marker.ReadMarker{} + rm.ClientID = client.GetClientID() + rm.ClientPublicKey = client.GetClientPublicKey() + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = client.GetClientID() + err = rm.Sign() + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodPost, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + + return r + }(), + }, + alloc: alloc, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", "qCj3sXXeXUAByi1ERIbcfXzWN75dyocYzyRXnkStXio="), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(1) FROM "collaborators" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnError(gorm.ErrRecordNotFound) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow(client.GetClientID()), + ) + + aa := sqlmock.AnyArg() + + mock.ExpectExec(`UPDATE "read_markers"`). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(client.GetClientID(), filePathHash). + WillReturnError(gorm.ErrRecordNotFound) + + mock.ExpectCommit() + }, + wantCode: http.StatusBadRequest, + wantBody: "{\"error\":\"client does not have permission to download the file. share does not exist\"}\n\n", + }, + { + name: "DownloadFile_Encrypted_Permission_Allowed_shared_File", + args: args{ + w: httptest.NewRecorder(), + r: func() *http.Request { + handlerName := handlers["/v1/file/download/{allocation}"] + url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + if err != nil { + t.Fatal() + } + + body := bytes.NewBuffer(nil) + formWriter := multipart.NewWriter(body) + remotePath := "/file.txt" + + pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) + require.NoError(t, formWriter.WriteField("path_hash", pathHash)) + require.NoError(t, formWriter.WriteField("block_num", fmt.Sprintf("%d", 1))) + authTicket, err := GetAuthTicketForEncryptedFile(alloc.ID, remotePath, pathHash, client.GetClientID(), sch.GetPublicKey()) + if err != nil { + t.Fatal(err) + } + require.NoError(t, formWriter.WriteField("auth_token", authTicket)) + rm := &marker.ReadMarker{} + rm.ClientID = client.GetClientID() + rm.ClientPublicKey = client.GetClientPublicKey() + rm.BlobberID = "" + rm.AllocationID = alloc.ID + rm.ReadCounter = 1 + rm.OwnerID = client.GetClientID() + err = rm.Sign() + if err != nil { + t.Fatal(err) + } + rmData, err := json.Marshal(rm) + require.NoError(t, err) + require.NoError(t, formWriter.WriteField("read_marker", string(rmData))) + if err := formWriter.Close(); err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodPost, url.String(), body) + r.Header.Add("Content-Type", formWriter.FormDataContentType()) + if err != nil { + t.Fatal(err) + } + + hash := encryption.Hash(alloc.Tx) + sign, err := sch.Sign(hash) + if err != nil { + t.Fatal(err) + } + + r.Header.Set("Content-Type", formWriter.FormDataContentType()) + r.Header.Set(common.ClientSignatureHeader, sign) + r.Header.Set(common.ClientHeader, alloc.OwnerID) + r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + + return r + }(), + }, + alloc: alloc, + begin: func() { + dataToEncrypt := "data_to_encrypt" + encMsg, err := encscheme.Encrypt([]byte(dataToEncrypt)) + if err != nil { + t.Fatal(err) + } + header := make([]byte, 2*1024) + copy(header[:], encMsg.MessageChecksum+","+encMsg.OverallChecksum) + data := append(header, encMsg.EncryptedData...) + setMockFileBlock(data) + }, + end: func() { + resetMockFileBlock() + }, + setupDbMock: func(mock sqlmock.Sqlmock) { + mock.ExpectBegin() + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + WithArgs(alloc.Tx). + WillReturnRows( + sqlmock.NewRows( + []string{ + "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + }, + ). + AddRow( + alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + ), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + WithArgs(alloc.ID). + WillReturnRows( + sqlmock.NewRows([]string{"id", "allocation_id"}). + AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + ) + + filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + WithArgs(alloc.ID, filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"path", "type", "path_hash", "lookup_hash", "content_hash", "encrypted_key"}). + AddRow("/file.txt", "f", filePathHash, filePathHash, "content_hash", encscheme.GetEncryptedKey()), + ) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT count(1) FROM "collaborators" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnError(gorm.ErrRecordNotFound) + + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + WithArgs(client.GetClientID()). + WillReturnRows( + sqlmock.NewRows([]string{"client_id"}). + AddRow(client.GetClientID()), + ) + + aa := sqlmock.AnyArg() + + mock.ExpectExec(`UPDATE "read_markers"`). + WithArgs(client.GetClientPublicKey(), alloc.ID, alloc.OwnerID, aa, aa, aa, aa, aa, aa, aa). + WillReturnResult(sqlmock.NewResult(0, 0)) + + reEncryptionKey, _ := encscheme.GetReGenKey(encscheme.GetEncryptedKey(), "filetype:audio") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "marketplace_share_info" WHERE`)). + WithArgs(client.GetClientID(), filePathHash). + WillReturnRows( + sqlmock.NewRows([]string{"re_encryption_key", "client_encryption_public_key"}). + AddRow(reEncryptionKey, encscheme.GetEncryptedKey()), + ) + + mock.ExpectCommit() + }, + wantCode: http.StatusOK, + wantBody: "", + }, + } + tests := append(positiveTests, negativeTests...) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mock := datastore.MockTheStore(t) + test.setupDbMock(mock) + + if test.begin != nil { + test.begin() + } + router.ServeHTTP(test.args.w, test.args.r) + if test.end != nil { + test.end() + } assert.Equal(t, test.wantCode, test.args.w.Result().StatusCode) - if test.wantCode != http.StatusOK { + if test.wantCode != http.StatusOK || test.wantBody != "" { assert.Equal(t, test.wantBody, test.args.w.Body.String()) } }) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index aca28ebbb..29f0e499f 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -1,12 +1,18 @@ package handler import ( + "bytes" "context" "encoding/hex" "encoding/json" "errors" + "math" + "strings" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" + zencryption "github.com/0chain/gosdk/zboxcore/encryption" "net/http" "path/filepath" @@ -19,13 +25,12 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" - "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/node" + zfileref "github.com/0chain/gosdk/zboxcore/fileref" "gorm.io/datatypes" "gorm.io/gorm" @@ -173,9 +178,10 @@ func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, return } -func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) ( - resp interface{}, err error) { - +func (fsh *StorageHandler) DownloadFile( + ctx context.Context, + r *http.Request, +) (resp interface{}, err error) { if r.Method == "GET" { return nil, common.NewError("download_file", "invalid method used (GET), use POST instead") @@ -283,7 +289,9 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) ( isCollaborator = reference.IsACollaborator(ctx, fileref.ID, clientID) ) - if !isOwner && !isRepairer && !isCollaborator { + var authToken *readmarker.AuthTicket = nil + + if (!isOwner && !isRepairer && !isCollaborator) || len(r.FormValue("auth_token")) > 0 { var authTokenString = r.FormValue("auth_token") // check auth token @@ -294,6 +302,12 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) ( "cannot verify auth ticket: %v", err) } + authToken = &readmarker.AuthTicket{} + err = json.Unmarshal([]byte(authTokenString), &authToken) + if err != nil { + return nil, common.NewErrorf("download_file", + "error parsing the auth ticket for download: %v", err) + } // set payer: check for command line payer flag (--rx_pay) if r.FormValue("rx_pay") == "true" { payerID = clientID @@ -304,6 +318,24 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) ( "error parsing the auth ticket for download: %v", err) } + // we only check content hash if its authticket is referring to a file + if authToken.RefType == zfileref.FILE && authToken.ContentHash != fileref.ContentHash { + return nil, errors.New("content hash does not match the requested file content hash") + } + + if authToken.RefType == zfileref.DIRECTORY { + hashes := util.GetParentPathHashes(allocationTx, fileref.Path) + found := false + for _, hash := range hashes { + if hash == authToken.FilePathHash { + found = true + break + } + } + if !found { + return nil, errors.New("auth ticket is not authorized to download file specified") + } + } readMarker.AuthTicket = datatypes.JSON(authTokenString) // check for file payer flag @@ -395,12 +427,71 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) ( "couldn't save latest read marker: %v", err) } - var response = &blobberhttp.DownloadResponse{} - response.Success = true - response.LatestRM = readMarker - response.Data = respData - response.Path = fileref.Path - response.AllocationID = fileref.AllocationID + if len(fileref.EncryptedKey) > 0 { + if authToken == nil { + return nil, errors.New("auth ticket is required to download encrypted file") + } + // check if client is authorized to download + shareInfo, err := reference.GetShareInfo( + ctx, + readMarker.ClientID, + authToken.FilePathHash, + ) + if err != nil { + return nil, errors.New("error during share info lookup in database" + err.Error()) + } else if shareInfo == nil || shareInfo.Revoked { + return nil, errors.New("client does not have permission to download the file. share does not exist") + } + + buyerEncryptionPublicKey := shareInfo.ClientEncryptionPublicKey + encscheme := zencryption.NewEncryptionScheme() + // reEncrypt does not require pub / private key, + // we could probably make it a classless function + + if err := encscheme.Initialize(""); err != nil { + return nil, err + } + if err := encscheme.InitForDecryption("filetype:audio", fileref.EncryptedKey); err != nil { + return nil, err + } + if err != nil { + return nil, err + } + + totalSize := len(respData) + result := []byte{} + for i := 0; i < totalSize; i += reference.CHUNK_SIZE { + encMsg := &zencryption.EncryptedMessage{} + chunkData := respData[i:int64(math.Min(float64(i+reference.CHUNK_SIZE), float64(totalSize)))] + + encMsg.EncryptedData = chunkData[(2 * 1024):] + + headerBytes := chunkData[:(2 * 1024)] + headerBytes = bytes.Trim(headerBytes, "\x00") + headerString := string(headerBytes) + + headerChecksums := strings.Split(headerString, ",") + if len(headerChecksums) != 2 { + Logger.Error("Block has invalid header", zap.String("request Url", r.URL.String())) + return nil, errors.New("Block has invalid header for request " + r.URL.String()) + } + + encMsg.MessageChecksum, encMsg.OverallChecksum = headerChecksums[0], headerChecksums[1] + encMsg.EncryptedKey = encscheme.GetEncryptedKey() + + reEncMsg, err := encscheme.ReEncrypt(encMsg, shareInfo.ReEncryptionKey, buyerEncryptionPublicKey) + if err != nil { + return nil, err + } + + encData, err := reEncMsg.Marshal() + if err != nil { + return nil, err + } + result = append(result, encData...) + } + respData = result + } stats.FileBlockDownloaded(ctx, fileref.ID) return respData, nil diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 002f3ad3c..99f9d7200 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -146,10 +146,6 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) } - if fileref.Type != reference.FILE { - return nil, common.NewError("invalid_parameters", "Path is not a file.") - } - result := fileref.GetListingData(ctx) commitMetaTxns, err := reference.GetCommitMetaTxns(ctx, fileref.ID) @@ -757,3 +753,4 @@ func pathHashFromReq(r *http.Request, allocationID string) (string, error) { return pathHash, nil } + diff --git a/code/go/0chain.net/blobbercore/readmarker/entity.go b/code/go/0chain.net/blobbercore/readmarker/entity.go index bbfde663c..0e0dfbc46 100644 --- a/code/go/0chain.net/blobbercore/readmarker/entity.go +++ b/code/go/0chain.net/blobbercore/readmarker/entity.go @@ -21,16 +21,18 @@ type AuthTicket struct { OwnerID string `json:"owner_id"` AllocationID string `json:"allocation_id"` FilePathHash string `json:"file_path_hash"` + ContentHash string `json:"content_hash"` FileName string `json:"file_name"` RefType string `json:"reference_type"` Expiration common.Timestamp `json:"expiration"` Timestamp common.Timestamp `json:"timestamp"` ReEncryptionKey string `json:"re_encryption_key"` Signature string `json:"signature"` + Encrypted bool `json:"encrypted"` } func (rm *AuthTicket) GetHashData() string { - hashData := fmt.Sprintf("%v:%v:%v:%v:%v:%v:%v:%v:%v", rm.AllocationID, rm.ClientID, rm.OwnerID, rm.FilePathHash, rm.FileName, rm.RefType, rm.ReEncryptionKey, rm.Expiration, rm.Timestamp) + hashData := fmt.Sprintf("%v:%v:%v:%v:%v:%v:%v:%v:%v:%v:%v", rm.AllocationID, rm.ClientID, rm.OwnerID, rm.FilePathHash, rm.FileName, rm.RefType, rm.ReEncryptionKey, rm.Expiration, rm.Timestamp, rm.ContentHash, rm.Encrypted) return hashData } diff --git a/code/go/0chain.net/blobbercore/reference/ds_test.go b/code/go/0chain.net/blobbercore/reference/ds_test.go new file mode 100644 index 000000000..06bb793d8 --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/ds_test.go @@ -0,0 +1,31 @@ +package reference + +import ( + "testing" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/stretchr/testify/require" +) + +// this is just a dummy snippet to connect to local database +func TestMockDb(t *testing.T) { + t.Skip("Fails as the data store is not mocked, so Open returns a dial error") + config.Configuration.DBHost = "localhost" + config.Configuration.DBName = "blobber_meta" + config.Configuration.DBPort = "5431" + config.Configuration.DBUserName = "blobber_user" + config.Configuration.DBPassword = "" + + require.NoError(t, datastore.GetStore().Open()) + db := datastore.GetStore().GetDB() + if db == nil { + t.Log("err connecting to database") + return + } + ref := &Ref{} + err := db.Where(&Ref{AllocationID: "4f928c7857fabb5737347c42204eea919a4777f893f35724f563b932f64e2367", Path: "/hack.txt"}). + First(ref). + Error + require.NoError(t, err) +} diff --git a/code/go/0chain.net/blobbercore/reference/shareinfo.go b/code/go/0chain.net/blobbercore/reference/shareinfo.go new file mode 100644 index 000000000..f0ab62ec6 --- /dev/null +++ b/code/go/0chain.net/blobbercore/reference/shareinfo.go @@ -0,0 +1,84 @@ +package reference + +import ( + "context" + "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "gorm.io/gorm" +) + +type ShareInfo struct { + OwnerID string `gorm:"owner_id" json:"owner_id,omitempty"` + ClientID string `gorm:"client_id" json:"client_id"` + FilePathHash string `gorm:"file_path_hash" json:"file_path_hash,omitempty"` + ReEncryptionKey string `gorm:"re_encryption_key" json:"re_encryption_key,omitempty"` + ClientEncryptionPublicKey string `gorm:"client_encryption_public_key" json:"client_encryption_public_key,omitempty"` + Revoked bool `gorm:"revoked" json:"revoked"` + ExpiryAt time.Time `gorm:"expiry_at" json:"expiry_at,omitempty"` +} + +func TableName() string { + return "marketplace_share_info" +} + +// add share if it already doesnot exist +func AddShareInfo(ctx context.Context, shareInfo ShareInfo) error { + db := datastore.GetStore().GetTransaction(ctx) + return db.Table(TableName()).Create(shareInfo).Error +} + +func DeleteShareInfo(ctx context.Context, shareInfo ShareInfo) error { + db := datastore.GetStore().GetTransaction(ctx) + + result := db.Table(TableName()). + Where(&ShareInfo{ + ClientID: shareInfo.ClientID, + FilePathHash: shareInfo.FilePathHash, + Revoked: false, + }). + Updates(ShareInfo{ + Revoked: true, + }) + + if result.Error != nil { + return result.Error + } + + if result.RowsAffected == 0 { + return gorm.ErrRecordNotFound + } + return nil +} + +func UpdateShareInfo(ctx context.Context, shareInfo ShareInfo) error { + db := datastore.GetStore().GetTransaction(ctx) + + return db.Table(TableName()). + Where(&ShareInfo{ + ClientID: shareInfo.ClientID, + FilePathHash: shareInfo.FilePathHash, + }). + Select("Revoked", "ReEncryptionKey", "ExpiryAt", "ClientEncryptionPublicKey"). + Updates(shareInfo). + Error +} + +func GetShareInfo(ctx context.Context, clientID string, filePathHash string) (*ShareInfo, error) { + db := datastore.GetStore().GetTransaction(ctx) + shareInfo := &ShareInfo{} + err := db.Table(TableName()). + Where(&ShareInfo{ + ClientID: clientID, + FilePathHash: filePathHash, + }). + First(shareInfo).Error + + if err == gorm.ErrRecordNotFound { + return nil, nil + } + if err != nil { + return nil, err + } + return shareInfo, nil +} diff --git a/code/go/0chain.net/blobbercore/util/json.go b/code/go/0chain.net/blobbercore/util/json.go index d35570024..672eddfca 100644 --- a/code/go/0chain.net/blobbercore/util/json.go +++ b/code/go/0chain.net/blobbercore/util/json.go @@ -2,6 +2,7 @@ package util import ( "fmt" + "github.com/0chain/gosdk/zboxcore/fileref" "reflect" "strings" ) @@ -25,3 +26,18 @@ func UnmarshalValidation(v interface{}) error { return nil } + +func GetParentPathHashes(allocationTx string, filePath string) []string { + splitted := strings.Split(filePath, "/") + pathHashes := []string{} + + for i := 0; i < len(splitted); i++ { + path := strings.Join(splitted[:len(splitted) - i], "/") + if path == "" { + path = "/" + } + pathHash := fileref.GetReferenceLookup(allocationTx, path) + pathHashes = append(pathHashes, pathHash) + } + return pathHashes +} diff --git a/code/go/0chain.net/core/common/time.go b/code/go/0chain.net/core/common/time.go index d5ede55ba..47da3f17c 100644 --- a/code/go/0chain.net/core/common/time.go +++ b/code/go/0chain.net/core/common/time.go @@ -17,3 +17,9 @@ func Within(ts int64, seconds int64) bool { now := time.Now().Unix() return now > ts-seconds && now < ts+seconds } + +//ToTime - converts the common.Timestamp to time.Time +func ToTime(ts Timestamp) time.Time { + return time.Unix(int64(ts), 0) +} + diff --git a/code/go/0chain.net/core/encryption/keys_test.go b/code/go/0chain.net/core/encryption/keys_test.go index f53ed6207..4eb0d206e 100644 --- a/code/go/0chain.net/core/encryption/keys_test.go +++ b/code/go/0chain.net/core/encryption/keys_test.go @@ -1,14 +1,31 @@ package encryption import ( - "testing" + "encoding/hex" + "github.com/0chain/gosdk/zboxcore/client" "github.com/herumi/bls-go-binary/bls" "github.com/stretchr/testify/require" + "testing" "fmt" - "encoding/hex" ) +func TestSignatureVerify(t *testing.T) { + allocationId := "4f928c7857fabb5737347c42204eea919a4777f893f35724f563b932f64e2367" + walletConfig := "{\"client_id\":\"9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85\",\"client_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"keys\":[{\"public_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"private_key\":\"a3a88aad5d89cec28c6e37c2925560ce160ac14d2cdcf4a4654b2bb358fe7514\"}],\"mnemonics\":\"inside february piece turkey offer merry select combine tissue wave wet shift room afraid december gown mean brick speak grant gain become toy clown\",\"version\":\"1.0\",\"date_created\":\"2021-05-21 17:32:29.484657 +0545 +0545 m=+0.072791323\"}" + require.NoError(t, client.PopulateClient(walletConfig, "bls0chain")) + sig, serr := client.Sign(allocationId) + require.Nil(t, serr) + require.NotNil(t, sig) + + res, err := client.VerifySignature( + "fb0eb9351978091da350348211888b06ed1ce84ae40d08de3cc826cd85197188", + allocationId, + ) + require.Nil(t, err) + require.Equal(t, res, true) +} + func TestMiraclToHerumiPK(t *testing.T) { miraclpk1 := `0418a02c6bd223ae0dfda1d2f9a3c81726ab436ce5e9d17c531ff0a385a13a0b491bdfed3a85690775ee35c61678957aaba7b1a1899438829f1dc94248d87ed36817f6dfafec19bfa87bf791a4d694f43fec227ae6f5a867490e30328cac05eaff039ac7dfc3364e851ebd2631ea6f1685609fc66d50223cc696cb59ff2fee47ac` pk1 := MiraclToHerumiPK(miraclpk1) @@ -60,5 +77,4 @@ func TestDebugOnly(t *testing.T) { h = Hash(clientKeyBytes2) fmt.Println("hash2 ", h) - } diff --git a/code/go/0chain.net/validatorcore/storage/models_test.go b/code/go/0chain.net/validatorcore/storage/models_test.go index 1fb88e24e..c684c0d9a 100644 --- a/code/go/0chain.net/validatorcore/storage/models_test.go +++ b/code/go/0chain.net/validatorcore/storage/models_test.go @@ -622,8 +622,7 @@ func TestObjectPath_VerifyPath(t *testing.T) { if !tt.wantErr { require.NoError(t, err) } else { - t.Log(err) - assert.Contains(t, err.Error(), tt.wantErrMsg) + require.Contains(t, err.Error(), tt.wantErrMsg) } }) } diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index c084f4821..70a5fbc51 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -42,7 +42,7 @@ max_file_size: 10485760 #10MB update_allocations_interval: 1m # delegate wallet (must be set) -delegate_wallet: '8b87739cd6c966c150a8a6e7b327435d4a581d9d9cc1d86a88c8a13ae1ad7a96' +delegate_wallet: '2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9' # min stake allowed, tokens min_stake: 1.0 # max stake allowed, tokens diff --git a/docker.local/Dockerfile.dev b/docker.local/Dockerfile.dev new file mode 100644 index 000000000..a2c38d790 --- /dev/null +++ b/docker.local/Dockerfile.dev @@ -0,0 +1,48 @@ +FROM golang:1.14.9-alpine3.12 as blobber_build + +RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep + +# Install Herumi's cryptography +RUN apk add gmp gmp-dev openssl-dev && \ + cd /tmp && \ + wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ + mv mcl* mcl && \ + make -C mcl -j $(nproc) lib/libmclbn256.so install && \ + cp mcl/lib/libmclbn256.so /usr/local/lib && \ + rm -R /tmp/mcl +#TODO: create shared image and remove code duplicates! +RUN git clone https://github.com/herumi/bls /tmp/bls && \ + cd /tmp/bls && \ + git submodule init && \ + git submodule update && \ + make -j $(nproc) install && \ + cd - && \ + rm -R /tmp/bls + +ENV SRC_DIR=/blobber +ENV GO111MODULE=on + +# Download the dependencies: +# Will be cached if we don't change mod/sum files +ADD ./gosdk /gosdk +COPY ./blobber/code/go/0chain.net/go.mod ./blobber/code/go/0chain.net/go.sum $SRC_DIR/go/0chain.net/ +RUN cd $SRC_DIR/go/0chain.net && go mod download + +#Add the source code +ADD ./blobber/code/go/0chain.net $SRC_DIR/go/0chain.net + +WORKDIR $SRC_DIR/go/0chain.net/blobber + +ARG GIT_COMMIT +ENV GIT_COMMIT=$GIT_COMMIT +RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" + +# Copy the build artifact into a minimal runtime image: +FROM golang:1.14.9-alpine3.12 +RUN apk add gmp gmp-dev openssl-dev +COPY --from=blobber_build /usr/local/lib/libmcl*.so \ + /usr/local/lib/libbls*.so \ + /usr/local/lib/ +ENV APP_DIR=/blobber +WORKDIR $APP_DIR +COPY --from=blobber_build $APP_DIR/go/0chain.net/blobber/blobber $APP_DIR/bin/blobber diff --git a/docker.local/ValidatorDockerfile.dev b/docker.local/ValidatorDockerfile.dev new file mode 100644 index 000000000..1841a035f --- /dev/null +++ b/docker.local/ValidatorDockerfile.dev @@ -0,0 +1,46 @@ +FROM golang:1.14.9-alpine3.12 as validator_build + +RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep + +# Install Herumi's cryptography +RUN apk add gmp gmp-dev openssl-dev && \ + cd /tmp && \ + wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ + mv mcl* mcl && \ + make -C mcl -j $(nproc) lib/libmclbn256.so install && \ + cp mcl/lib/libmclbn256.so /usr/local/lib && \ + rm -R /tmp/mcl + +RUN git clone https://github.com/herumi/bls /tmp/bls && \ + cd /tmp/bls && \ + git submodule init && \ + git submodule update && \ + make -j $(nproc) install && \ + cd - && \ + rm -R /tmp/bls + +ENV SRC_DIR=/blobber +ENV GO111MODULE=on + +# Download the dependencies: +# Will be cached if we don't change mod/sum files +ADD ./gosdk /gosdk +COPY ./blobber/code/go/0chain.net/go.mod ./blobber/code/go/0chain.net/go.sum $SRC_DIR/go/0chain.net/ +RUN cd $SRC_DIR/go/0chain.net && go mod download + +#Add the source code +ADD ./blobber/code/go/0chain.net $SRC_DIR/go/0chain.net + +WORKDIR $SRC_DIR/go/0chain.net/validator + +RUN go build -v -tags "bn256 development" -ldflags "-X 0chain.net/core/build.BuildTag=$GIT_COMMIT" + +# Copy the build artifact into a minimal runtime image: +FROM golang:1.11.4-alpine3.8 +RUN apk add gmp gmp-dev openssl-dev +COPY --from=validator_build /usr/local/lib/libmcl*.so \ + /usr/local/lib/libbls*.so \ + /usr/local/lib/ +ENV APP_DIR=/blobber +WORKDIR $APP_DIR +COPY --from=validator_build $APP_DIR/go/0chain.net/validator/validator $APP_DIR/bin/validator diff --git a/docker.local/bin/blobber.stop_bls.sh b/docker.local/bin/blobber.stop_bls.sh new file mode 100755 index 000000000..a0b60b3e5 --- /dev/null +++ b/docker.local/bin/blobber.stop_bls.sh @@ -0,0 +1,11 @@ +#!/bin/sh +PWD=`pwd` +BLOBBER_DIR=`basename $PWD` +BLOBBER_ID=`echo my directory $BLOBBER_DIR | sed -e 's/.*\(.\)$/\1/'` + + +echo Stopping blobber$BLOBBER_ID ... + +# echo blobber$i + +BLOBBER=$BLOBBER_ID docker-compose -p blobber$BLOBBER_ID -f ../b0docker-compose.yml down diff --git a/docker.local/bin/build.blobber.dev.sh b/docker.local/bin/build.blobber.dev.sh new file mode 100755 index 000000000..2496338e0 --- /dev/null +++ b/docker.local/bin/build.blobber.dev.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -e + +GIT_COMMIT=$(git rev-list -1 HEAD) +echo $GIT_COMMIT + +docker build --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/ValidatorDockerfile.dev .. -t validator +docker build --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/Dockerfile.dev .. -t blobber + +for i in $(seq 1 6); +do + BLOBBER=$i docker-compose -p blobber$i -f docker.local/docker-compose.yml build --force-rm +done + +docker.local/bin/sync_clock.sh diff --git a/docker.local/bin/stop_all.sh b/docker.local/bin/stop_all.sh new file mode 100755 index 000000000..b44c473a4 --- /dev/null +++ b/docker.local/bin/stop_all.sh @@ -0,0 +1,12 @@ +#!/bin/bash +cd docker.local/blobber1 +../bin/blobber.stop_bls.sh +cd - +cd docker.local/blobber2 +../bin/blobber.stop_bls.sh +cd - +cd docker.local/blobber3 +../bin/blobber.stop_bls.sh +cd - +cd docker.local/blobber4 +../bin/blobber.stop_bls.sh diff --git a/docs/src/proxy_encryption_download_flow.plantuml b/docs/src/proxy_encryption_download_flow.plantuml new file mode 100644 index 000000000..74a28886c --- /dev/null +++ b/docs/src/proxy_encryption_download_flow.plantuml @@ -0,0 +1,16 @@ +@startuml + +actor Client + + +loop till all of requested file chunks are completely downloaded +Client -> Blobber : Request the file in range of blocks x..y +Blobber --> Database : Get reencryption key from postgres db table \n\ +for the client +Blobber --> Blobber : Iterate through all chunks and \n\ +encrypt every chunk using re-encryption key +Blobber --> Client : Return Re-encrypted message +Client --> Client : Re-decrypt the received message using private key +end + +@enduml diff --git a/docs/src/share.plantuml b/docs/src/share.plantuml new file mode 100644 index 000000000..08623795e --- /dev/null +++ b/docs/src/share.plantuml @@ -0,0 +1,17 @@ +@startuml + +title Buyer shares encrypted file +actor Seller +actor Buyer + +Buyer -> Seller: Client Id, encryption public key + +Seller -> Seller : Generate auth ticket for file / folder using \n\ +buyer client id as well as encryption public key +Seller --> Blobber : Upload auth ticket +Blobber --> Blobber : Save auth ticket to **marketplace_share_info** \n\ +table +Seller --> Seller : Check upload consensus status +Seller --> Seller : Return auth ticket + +@enduml diff --git a/docs/src/share_revoke.plantuml b/docs/src/share_revoke.plantuml new file mode 100644 index 000000000..739347a37 --- /dev/null +++ b/docs/src/share_revoke.plantuml @@ -0,0 +1,13 @@ +@startuml + +title Buyer revokes share for client +actor Seller + +Seller -> Seller : Revoke share using **client id**,\n\ +**allocation id** and **remote path** +Seller --> Blobber : Revoke share using provided parameters +Blobber --> Blobber : Find path share in **marketplace_share_info** \n\ +table and set **revoked** flag to true +Seller --> Seller : return consensus status, \n\ +we need 100% consensus for success +@enduml diff --git a/go.mod b/go.mod index a7cba7425..e32596678 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/0chain/blobber require ( - github.com/0chain/gosdk v1.1.6 + github.com/0chain/gosdk v1.2.68-0.20210701180605-719eb403820c github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/didip/tollbooth v4.0.2+incompatible diff --git a/go.sum b/go.sum index bc6a67c1a..519280b70 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/0chain/gosdk v1.1.6 h1:urbc9aTp57HAUVgJWP6TsSgRKX61hLBwtSyMV/ngCBo= github.com/0chain/gosdk v1.1.6/go.mod h1:edV5GwogiT6nK2+s4QcFCz3saUhkuFK6EIqNJfOt8xc= +github.com/0chain/gosdk v1.2.68-0.20210701180605-719eb403820c h1:cTxcdg5VcfHLoiT1C33gskjN2RJQqkOXpIuuUXBUAig= +github.com/0chain/gosdk v1.2.68-0.20210701180605-719eb403820c/go.mod h1:EntD+g2xA5SLUAm6JABMhkYAlTOmp0XhSnf/5xfwnV4= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= diff --git a/sql/16-add-marketplace-table.sql b/sql/16-add-marketplace-table.sql new file mode 100644 index 000000000..bbfd4b043 --- /dev/null +++ b/sql/16-add-marketplace-table.sql @@ -0,0 +1,23 @@ +\connect blobber_meta; + + +CREATE TABLE marketplace_share_info ( + id BIGSERIAL PRIMARY KEY, + owner_id VARCHAR(64) NOT NULL, + client_id VARCHAR(64) NOT NULL, + file_path_hash TEXT NOT NULL, + re_encryption_key TEXT NOT NULL, + client_encryption_public_key TEXT NOT NULL, + expiry_at TIMESTAMP NULL, + revoked BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_marketplace_share_info_for_owner ON marketplace_share_info(owner_id, file_path_hash); +CREATE INDEX idx_marketplace_share_info_for_client ON marketplace_share_info(client_id, file_path_hash); + +CREATE TRIGGER share_info_modtime BEFORE UPDATE ON marketplace_share_info FOR EACH ROW EXECUTE PROCEDURE update_modified_column(); + +GRANT ALL PRIVILEGES ON TABLE marketplace_share_info TO blobber_user; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO blobber_user; diff --git a/sql/17-add-indexes-to-reference-objects.sql b/sql/17-add-indexes-to-reference-objects.sql new file mode 100644 index 000000000..dc8146e84 --- /dev/null +++ b/sql/17-add-indexes-to-reference-objects.sql @@ -0,0 +1,4 @@ +\connect blobber_meta; + +CREATE INDEX idx_reference_objects_for_lookup_hash ON reference_objects(allocation_id, lookup_hash); +CREATE INDEX idx_reference_objects_for_path ON reference_objects(allocation_id, path);