From da32de7309151de2cb08b8133621ae5dfc25d519 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Sun, 9 May 2021 13:49:47 +0530 Subject: [PATCH 01/20] :sparkles: added integration test github action --- .github/workflows/ci.yml | 23 ++++++ .../handler/grpc_handler_integration_test.go | 73 +++++++++++++++++++ docker.local/b0docker-compose.yml | 2 + 3 files changed, 98 insertions(+) create mode 100644 code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc4e0edf2..95d652753 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,29 @@ env: VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} jobs: + integration-tests: + runs-on: ubuntu-20.04 + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.14.x + - uses: actions/checkout@v2 + # In this step, this action saves a list of existing images, + # the cache is created without them in the post run. + # It also restores the cache if it exists. + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: Build + run: | + ./docker.local/bin/blobber.init.setup.sh + docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 + ./docker.local/bin/build.blobber.sh + cd docker.local/blobber1; ../bin/blobber.start_bls.sh /dev/null & + cd code/go/0chain.net + go test blobbercore/handler/grpc_handler_integration_test.go --args integration + test: runs-on: ubuntu-20.04 steps: diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go new file mode 100644 index 000000000..d92f6f717 --- /dev/null +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -0,0 +1,73 @@ +package handler + +import ( + "context" + "fmt" + "log" + "os" + "testing" + "time" + + "google.golang.org/grpc" + + "0chain.net/blobbercore/blobbergrpc" +) + +const BlobberAddr = "localhost:7031" +const RetryAttempts = 8 +const RetryTimeout = 3 + +func TestBlobberGRPCService_IntegrationTest(t *testing.T) { + args := make(map[string]bool) + for _, arg := range os.Args { + args[arg] = true + } + + if !args["integration"] { + t.Skip() + } + + ctx := context.Background() + + var conn *grpc.ClientConn + var err error + for i := 0; i < RetryAttempts; i++ { + log.Println("Connection attempt - " + fmt.Sprint(i+1)) + conn, err = grpc.Dial(BlobberAddr, grpc.WithInsecure()) + if err != nil { + log.Println(err) + <-time.After(time.Second * RetryTimeout) + continue + } + + break + } + + if err != nil { + t.Fatal(err) + } + + defer conn.Close() + client := blobbergrpc.NewBlobberClient(conn) + + t.Run("TestGetAllocation", func(t *testing.T) { + getAllocationReq := &blobbergrpc.GetAllocationRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "", + ClientKey: "", + Allocation: "", + }, + Id: "", + } + + getAllocationResp, err := client.GetAllocation(ctx, getAllocationReq) + if err != nil { + t.Fatal(err) + } + + if getAllocationResp.Allocation.ID != getAllocationReq.Id { + t.Fatal("unexpected allocation id from GetAllocation rpc") + } + }) + +} diff --git a/docker.local/b0docker-compose.yml b/docker.local/b0docker-compose.yml index 6030b1930..64672a94c 100644 --- a/docker.local/b0docker-compose.yml +++ b/docker.local/b0docker-compose.yml @@ -7,6 +7,8 @@ services: POSTGRES_HOST: postgres POSTGRES_USER: postgres POSTGRES_HOST_AUTH_METHOD: trust + ports: + - "5432:5432" volumes: - ./blobber${BLOBBER}/data/postgresql:/var/lib/postgresql/data networks: From 6c49f89cefe611bd7435f9cbba520f410a17382c Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Sun, 9 May 2021 13:55:54 +0530 Subject: [PATCH 02/20] :wrench: add integration tests to make file --- .github/workflows/ci.yml | 3 +-- Makefile | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95d652753..86692e52c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,7 @@ jobs: docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 ./docker.local/bin/build.blobber.sh cd docker.local/blobber1; ../bin/blobber.start_bls.sh /dev/null & - cd code/go/0chain.net - go test blobbercore/handler/grpc_handler_integration_test.go --args integration + make integration-tests test: runs-on: ubuntu-20.04 diff --git a/Makefile b/Makefile index b85f580b9..3bcb13025 100644 --- a/Makefile +++ b/Makefile @@ -4,4 +4,7 @@ test: cd code/go/0chain.net; go test ./...; lint: - cd code/go/0chain.net; golangci-lint run; \ No newline at end of file + cd code/go/0chain.net; golangci-lint run; + +integration-tests: + cd code/go/0chain.net; go test ./... --args integration; \ No newline at end of file From 4bbe9d2f4dbee873e70114f663f30c7414ef39fb Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Sun, 9 May 2021 14:02:19 +0530 Subject: [PATCH 03/20] :green-heart: debugging integration test github action --- .github/workflows/ci.yml | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86692e52c..8b2cfc6f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 ./docker.local/bin/build.blobber.sh cd docker.local/blobber1; ../bin/blobber.start_bls.sh /dev/null & + pwd + ls make integration-tests test: diff --git a/Makefile b/Makefile index 3bcb13025..36aa96854 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test lint +.PHONY: test lint integration-tests test: cd code/go/0chain.net; go test ./...; From b640f2698fd19bc0c37e6d265646e19443f72df7 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Sun, 9 May 2021 14:07:56 +0530 Subject: [PATCH 04/20] :green_heart: attempting to fix integration-test github action --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b2cfc6f9..7764bc197 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,14 +25,14 @@ jobs: - uses: satackey/action-docker-layer-caching@v0.0.11 # Ignore the failure of a step and avoid terminating the job. continue-on-error: true - - name: Build + - name: Build environment and run tests run: | ./docker.local/bin/blobber.init.setup.sh docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 ./docker.local/bin/build.blobber.sh - cd docker.local/blobber1; ../bin/blobber.start_bls.sh /dev/null & - pwd - ls + cd docker.local/blobber1 + ../bin/blobber.start_bls.sh /dev/null & + cd ../.. make integration-tests test: From c00e6720fc92635b4e7af207fb83de0589265e5a Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Sun, 9 May 2021 19:40:54 +0530 Subject: [PATCH 05/20] :white_check_mark: added testdata controller and added working GetAllocation integration tests --- .github/workflows/ci.yml | 2 +- code/go/0chain.net/blobber/main.go | 2 +- .../0chain.net/blobbercore/config/config.go | 4 +- .../handler/grpc_handler_helper_unit_test.go | 96 +++++++++++++++++++ .../handler/grpc_handler_integration_test.go | 56 ++++++++--- 5 files changed, 143 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7764bc197..1aaff8fe8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - uses: satackey/action-docker-layer-caching@v0.0.11 # Ignore the failure of a step and avoid terminating the job. continue-on-error: true - - name: Build environment and run tests + - name: Build test environment and run tests run: | ./docker.local/bin/blobber.init.setup.sh docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index 322c27232..3442a7afb 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -227,7 +227,7 @@ func main() { flag.Parse() config.SetupDefaultConfig() - config.SetupConfig() + config.SetupConfig("./config") config.Configuration.DeploymentMode = byte(*deploymentMode) diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index bf95a8d8f..cca45565e 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -44,12 +44,12 @@ func SetupDefaultConfig() { } /*SetupConfig - setup the configuration system */ -func SetupConfig() { +func SetupConfig(configPath string) { replacer := strings.NewReplacer(".", "_") viper.SetEnvKeyReplacer(replacer) viper.AutomaticEnv() viper.SetConfigName("0chain_blobber") - viper.AddConfigPath("./config") + viper.AddConfigPath(configPath) err := viper.ReadInConfig() // Find and read the config file if err != nil { // Handle errors reading the config file panic(fmt.Errorf("fatal error config file: %s", err)) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index 66345b7ab..896fb28a9 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -2,6 +2,12 @@ package handler import ( "context" + "database/sql" + "fmt" + "log" + "time" + + "gorm.io/gorm" "0chain.net/blobbercore/allocation" "0chain.net/blobbercore/reference" @@ -56,3 +62,93 @@ func (_m *storageHandlerI) verifyAuthTicket(ctx context.Context, authTokenString return r0, r1 } + +type TestDataController struct { + db *gorm.DB +} + +func NewTestDataController(db *gorm.DB) *TestDataController { + return &TestDataController{db: db} +} + +// ClearDatabase deletes all data from all tables +func (c *TestDataController) ClearDatabase() error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + ctx := context.Background() + tx, err = db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return err + } + + _, err = tx.Exec("truncate allocations cascade") + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} + +func (c *TestDataController) AddGetAllocationTestData() error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + ctx := context.Background() + tx, err = db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) +VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index d92f6f717..4c68e668a 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -5,9 +5,16 @@ import ( "fmt" "log" "os" + "strings" "testing" "time" + "github.com/spf13/viper" + + "gorm.io/driver/postgres" + "gorm.io/gorm" + + "0chain.net/blobbercore/config" "google.golang.org/grpc" "0chain.net/blobbercore/blobbergrpc" @@ -22,7 +29,6 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { for _, arg := range os.Args { args[arg] = true } - if !args["integration"] { t.Skip() } @@ -39,33 +45,57 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { <-time.After(time.Second * RetryTimeout) continue } - break } - if err != nil { t.Fatal(err) } - defer conn.Close() - client := blobbergrpc.NewBlobberClient(conn) + blobberClient := blobbergrpc.NewBlobberClient(conn) + + pwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + configDir := strings.Split(pwd, "/code/go")[0] + "/config" + config.SetupDefaultConfig() + config.SetupConfig(configDir) + config.Configuration.DBHost = "localhost" + config.Configuration.DBName = viper.GetString("db.name") + config.Configuration.DBPort = viper.GetString("db.port") + config.Configuration.DBUserName = viper.GetString("db.user") + config.Configuration.DBPassword = viper.GetString("db.password") + db, err := gorm.Open(postgres.Open(fmt.Sprintf( + "host=%v port=%v user=%v dbname=%v password=%v sslmode=disable", + config.Configuration.DBHost, config.Configuration.DBPort, + config.Configuration.DBUserName, config.Configuration.DBName, + config.Configuration.DBPassword)), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + tdController := NewTestDataController(db) t.Run("TestGetAllocation", func(t *testing.T) { + err := tdController.ClearDatabase() + if err != nil { + t.Fatal(err) + } + err = tdController.AddGetAllocationTestData() + if err != nil { + t.Fatal(err) + } + getAllocationReq := &blobbergrpc.GetAllocationRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "", - ClientKey: "", - Allocation: "", - }, - Id: "", + Context: &blobbergrpc.RequestContext{}, + Id: "exampleTransaction", } - getAllocationResp, err := client.GetAllocation(ctx, getAllocationReq) + getAllocationResp, err := blobberClient.GetAllocation(ctx, getAllocationReq) if err != nil { t.Fatal(err) } - if getAllocationResp.Allocation.ID != getAllocationReq.Id { + if getAllocationResp.Allocation.Tx != getAllocationReq.Id { t.Fatal("unexpected allocation id from GetAllocation rpc") } }) From 5e098a1dcdef0175800e3f2540c305d1d031eb87 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Sun, 9 May 2021 23:49:48 +0530 Subject: [PATCH 06/20] :white_check_mark: --- .../handler/grpc_handler_helper_unit_test.go | 87 ++++++++++++++++++- .../handler/grpc_handler_integration_test.go | 34 +++++++- 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index 896fb28a9..e73eb168f 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -91,8 +91,7 @@ func (c *TestDataController) ClearDatabase() error { return err } - ctx := context.Background() - tx, err = db.BeginTx(ctx, &sql.TxOptions{}) + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) if err != nil { return err } @@ -102,6 +101,21 @@ func (c *TestDataController) ClearDatabase() error { return err } + _, err = tx.Exec("truncate reference_objects cascade") + if err != nil { + return err + } + + _, err = tx.Exec("truncate commit_meta_txns cascade") + if err != nil { + return err + } + + _, err = tx.Exec("truncate collaborators cascade") + if err != nil { + return err + } + err = tx.Commit() if err != nil { return err @@ -129,8 +143,49 @@ func (c *TestDataController) AddGetAllocationTestData() error { return err } - ctx := context.Background() - tx, err = db.BeginTx(ctx, &sql.TxOptions{}) + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) +VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} + +func (c *TestDataController) AddGetFileMetaDataTestData() error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) if err != nil { return err } @@ -145,6 +200,30 @@ VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKe return err } + _, err = tx.Exec(` +INSERT INTO reference_objects (id, allocation_id, path_hash,lookup_hash,type,name,path,hash,custom_meta,content_hash,merkle_root,actual_file_hash,mimetype,write_marker,thumbnail_hash, actual_thumbnail_hash) +VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','f','filename','examplePath','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO commit_meta_txns (ref_id,txn_id) +VALUES (1234,'someTxn'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO collaborators (ref_id, client_id) +VALUES (1234, 'someClient'); +`) + if err != nil { + return err + } + err = tx.Commit() if err != nil { return err diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 4c68e668a..19df93947 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -29,9 +29,9 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { for _, arg := range os.Args { args[arg] = true } - if !args["integration"] { - t.Skip() - } + //if !args["integration"] { + // t.Skip() + //} ctx := context.Background() @@ -100,4 +100,32 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } }) + t.Run("TestGetFileMetaData", func(t *testing.T) { + err := tdController.ClearDatabase() + if err != nil { + t.Fatal(err) + } + err = tdController.AddGetFileMetaDataTestData() + if err != nil { + t.Fatal(err) + } + + req := &blobbergrpc.GetFileMetaDataRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + }, + Path: "examplePath", + PathHash: "exampleId:examplePath", + Allocation: "exampleTransaction", + } + getFileMetaDataResp, err := blobberClient.GetFileMetaData(ctx, req) + if err != nil { + t.Fatal(err) + } + + if getFileMetaDataResp.MetaData.FileMetaData.Name != "filename" { + t.Fatal("unexpected path from GetFileMetaData rpc") + } + }) + } From 5ca5b5bcef0d38b3aa544fb21654954c3529f3b3 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Mon, 10 May 2021 00:06:57 +0530 Subject: [PATCH 07/20] :white_check_mark: add GetFileStats rpc integration test --- .../handler/grpc_handler_helper_unit_test.go | 50 +++++++++++++++++++ .../handler/grpc_handler_integration_test.go | 32 +++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index e73eb168f..d23a29a68 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -231,3 +231,53 @@ VALUES (1234, 'someClient'); return nil } + +func (c *TestDataController) AddGetFileStatsTestData() error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) +VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO reference_objects (id, allocation_id, path_hash,lookup_hash,type,name,path,hash,custom_meta,content_hash,merkle_root,actual_file_hash,mimetype,write_marker,thumbnail_hash, actual_thumbnail_hash) +VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','f','filename','examplePath','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 19df93947..8659f6e14 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -124,7 +124,37 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } if getFileMetaDataResp.MetaData.FileMetaData.Name != "filename" { - t.Fatal("unexpected path from GetFileMetaData rpc") + t.Fatal("unexpected file name from GetFileMetaData rpc") + } + }) + + t.Run("TestGetFileStats", func(t *testing.T) { + err := tdController.ClearDatabase() + if err != nil { + t.Fatal(err) + } + err = tdController.AddGetFileStatsTestData() + if err != nil { + t.Fatal(err) + } + + req := &blobbergrpc.GetFileStatsRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: "exampleTransaction", + }, + Path: "examplePath", + PathHash: "exampleId:examplePath", + } + + getFileStatsResp, err := blobberClient.GetFileStats(ctx, req) + if err != nil { + t.Fatal(err) + } + + if getFileStatsResp.MetaData.FileMetaData.Name != "filename" { + t.Fatal("unexpected file name from GetFileStats rpc") } }) From 1a88e6ab05e142c3e96eb47f062838c1164b29c7 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Mon, 10 May 2021 02:10:45 +0530 Subject: [PATCH 08/20] :white_check_mark: GetObjectPath integration tests --- .../handler/grpc_handler_helper_unit_test.go | 42 +++++++++++++++++++ .../handler/grpc_handler_integration_test.go | 31 ++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index d23a29a68..9defb308a 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -281,3 +281,45 @@ VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','f','fi return nil } + +func (c *TestDataController) AddGetObjectPathTestData() error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) +VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 8659f6e14..845724f8f 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -158,4 +158,35 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } }) + t.Run("TestGetObjectPath", func(t *testing.T) { + err := tdController.ClearDatabase() + if err != nil { + t.Fatal(err) + } + err = tdController.AddGetObjectPathTestData() + if err != nil { + t.Fatal(err) + } + + req := &blobbergrpc.GetObjectPathRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: "exampleTransaction", + }, + Allocation: "", + Path: "examplePath", + BlockNum: "0", + } + + getObjectPathResp, err := blobberClient.GetObjectPath(ctx, req) + if err != nil { + t.Fatal(err) + } + + if getObjectPathResp.ObjectPath.Path.DirMetaData.Path != "/" { + t.Fatal("unexpected root hash from GetObjectPath rpc") + } + }) + } From 92a753088bcc4ed10ca3eaf8538f573f1e661b9a Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Mon, 10 May 2021 02:22:50 +0530 Subject: [PATCH 09/20] :white_check_mark: added list entities integration tests --- .../handler/grpc_handler_helper_unit_test.go | 50 +++++++++++++++++++ .../handler/grpc_handler_integration_test.go | 32 ++++++++++++ 2 files changed, 82 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index 9defb308a..4d0f992c9 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -282,6 +282,56 @@ VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','f','fi return nil } +func (c *TestDataController) AddListEntitiesTestData() error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) +VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO reference_objects (id, allocation_id, path_hash,lookup_hash,type,name,path,hash,custom_meta,content_hash,merkle_root,actual_file_hash,mimetype,write_marker,thumbnail_hash, actual_thumbnail_hash) +VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','f','filename','examplePath','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} + func (c *TestDataController) AddGetObjectPathTestData() error { var err error var tx *sql.Tx diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 845724f8f..d59e0c94f 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -158,6 +158,38 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } }) + t.Run("TestListEntities", func(t *testing.T) { + err := tdController.ClearDatabase() + if err != nil { + t.Fatal(err) + } + err = tdController.AddListEntitiesTestData() + if err != nil { + t.Fatal(err) + } + + req := &blobbergrpc.ListEntitiesRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: "exampleTransaction", + }, + Path: "examplePath", + PathHash: "exampleId:examplePath", + AuthToken: "", + Allocation: "", + } + + listEntitiesResp, err := blobberClient.ListEntities(ctx, req) + if err != nil { + t.Fatal(err) + } + + if listEntitiesResp.MetaData.DirMetaData.Path != "examplePath" { + t.Fatal("unexpected path from ListEntities rpc") + } + }) + t.Run("TestGetObjectPath", func(t *testing.T) { err := tdController.ClearDatabase() if err != nil { From 374535ba1c6dad7ba6effe59debb0bdce178af42 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Mon, 10 May 2021 02:41:51 +0530 Subject: [PATCH 10/20] :white_check_mark: added GetReferencePath integration test --- .../handler/grpc_handler_helper_unit_test.go | 42 +++++++++++++++++++ .../handler/grpc_handler_integration_test.go | 30 +++++++++++++ 2 files changed, 72 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index 4d0f992c9..fd9540e46 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -373,3 +373,45 @@ VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKe return nil } + +func (c *TestDataController) AddGetReferencePathTestData() error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) +VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index d59e0c94f..93ceed8bc 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -221,4 +221,34 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } }) + t.Run("TestGetReferencePath", func(t *testing.T) { + err := tdController.ClearDatabase() + if err != nil { + t.Fatal(err) + } + err = tdController.AddGetReferencePathTestData() + if err != nil { + t.Fatal(err) + } + + req := &blobbergrpc.GetReferencePathRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: "exampleTransaction", + }, + Paths: "", + Path: "/", + Allocation: "", + } + getReferencePathResp, err := blobberClient.GetReferencePath(ctx, req) + if err != nil { + t.Fatal(err) + } + + if getReferencePathResp.ReferencePath.MetaData.DirMetaData.Path != "/" { + t.Fatal("unexpected path from GetReferencePath rpc") + } + }) + } From 2d399fca09b537df79a6b3d1d3bdcf929823ecc6 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Mon, 10 May 2021 02:58:54 +0530 Subject: [PATCH 11/20] :white_check_mark: added GetObjectTree rpc integration tests --- .../handler/grpc_handler_helper_unit_test.go | 50 +++++++++++++++++++ .../handler/grpc_handler_integration_test.go | 35 +++++++++++-- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index fd9540e46..b070c8eef 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -415,3 +415,53 @@ VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKe return nil } + +func (c *TestDataController) AddGetObjectTreeTestData() error { + var err error + var tx *sql.Tx + defer func() { + if err != nil { + if tx != nil { + errRollback := tx.Rollback() + if errRollback != nil { + log.Println(errRollback) + } + } + } + }() + + db, err := c.db.DB() + if err != nil { + return err + } + + tx, err = db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + + expTime := time.Now().Add(time.Hour * 100000).UnixNano() + + _, err = tx.Exec(` +INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) +VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +`) + if err != nil { + return err + } + + _, err = tx.Exec(` +INSERT INTO reference_objects (id, allocation_id, path_hash,lookup_hash,type,name,path,hash,custom_meta,content_hash,merkle_root,actual_file_hash,mimetype,write_marker,thumbnail_hash, actual_thumbnail_hash) +VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','d','root','/','someHash','customMeta','contentHash','merkleRoot','actualFileHash','mimetype','writeMarker','thumbnailHash','actualThumbnailHash'); +`) + if err != nil { + return err + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 93ceed8bc..d897b7f73 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -29,9 +29,9 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { for _, arg := range os.Args { args[arg] = true } - //if !args["integration"] { - // t.Skip() - //} + if !args["integration"] { + t.Skip() + } ctx := context.Background() @@ -251,4 +251,33 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } }) + t.Run("TestGetObjectTree", func(t *testing.T) { + err := tdController.ClearDatabase() + if err != nil { + t.Fatal(err) + } + err = tdController.AddGetObjectTreeTestData() + if err != nil { + t.Fatal(err) + } + + req := &blobbergrpc.GetObjectTreeRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: "exampleTransaction", + }, + Path: "/", + Allocation: "", + } + getObjectTreeResp, err := blobberClient.GetObjectTree(ctx, req) + if err != nil { + t.Fatal(err) + } + + if getObjectTreeResp.ReferencePath.MetaData.DirMetaData.Name != "root" { + t.Fatal("unexpected root name from GetObject") + } + }) + } From 163688e3c97904fe0a93da7d42388bc0f4055e74 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Fri, 21 May 2021 20:21:58 +0530 Subject: [PATCH 12/20] :white_check_mark: updated tests and added table driven tests --- Makefile | 5 +- code/go/0chain.net/blobber/main.go | 3 +- .../handler/grpc_handler_helper_unit_test.go | 23 +- .../handler/grpc_handler_integration_test.go | 455 ++++++++++++++---- .../go/0chain.net/core/common/rate_limiter.go | 10 +- 5 files changed, 380 insertions(+), 116 deletions(-) diff --git a/Makefile b/Makefile index 902f15ace..79477b07b 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,4 @@ test: go test ./...; lint: - golangci-lint run; - -integration-tests: - go test ./... --args integration; \ No newline at end of file + golangci-lint run; \ No newline at end of file diff --git a/code/go/0chain.net/blobber/main.go b/code/go/0chain.net/blobber/main.go index 5030ab6b4..cb26ee6eb 100644 --- a/code/go/0chain.net/blobber/main.go +++ b/code/go/0chain.net/blobber/main.go @@ -339,10 +339,11 @@ func main() { methodsOk := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS"}) + common.ConfigRateLimits() initHandlers(r) initServer() - grpcServer := handler.NewServerWithMiddlewares(common.ConfigRateLimits()) + grpcServer := handler.NewServerWithMiddlewares(common.NewGRPCRateLimiter()) handler.RegisterGRPCServices(r, grpcServer) rHandler := handlers.CORS(originsOk, headersOk, methodsOk)(r) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index 859407b49..a52ffd827 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -7,9 +7,10 @@ import ( "log" "time" - "gorm.io/gorm" "testing" + "gorm.io/gorm" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" coreConfig "github.com/0chain/blobber/code/go/0chain.net/core/config" "github.com/0chain/gosdk/core/zcncrypto" @@ -237,7 +238,7 @@ VALUES (1234, 'someClient'); return nil } -func (c *TestDataController) AddGetFileStatsTestData() error { +func (c *TestDataController) AddGetFileStatsTestData(allocationTx, pubKey string) error { var err error var tx *sql.Tx defer func() { @@ -265,7 +266,7 @@ func (c *TestDataController) AddGetFileStatsTestData() error { _, err = tx.Exec(` INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) -VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +VALUES ('exampleId' ,'` + allocationTx + `','exampleOwnerId','` + pubKey + `',` + fmt.Sprint(expTime) + `,'examplePayerId'); `) if err != nil { return err @@ -287,7 +288,7 @@ VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','f','fi return nil } -func (c *TestDataController) AddListEntitiesTestData() error { +func (c *TestDataController) AddListEntitiesTestData(allocationTx, pubkey string) error { var err error var tx *sql.Tx defer func() { @@ -315,7 +316,7 @@ func (c *TestDataController) AddListEntitiesTestData() error { _, err = tx.Exec(` INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) -VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +VALUES ('exampleId' ,'` + allocationTx + `','exampleOwnerId','` + pubkey + `',` + fmt.Sprint(expTime) + `,'examplePayerId'); `) if err != nil { return err @@ -337,7 +338,7 @@ VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','f','fi return nil } -func (c *TestDataController) AddGetObjectPathTestData() error { +func (c *TestDataController) AddGetObjectPathTestData(allocationTx, pubKey string) error { var err error var tx *sql.Tx defer func() { @@ -365,7 +366,7 @@ func (c *TestDataController) AddGetObjectPathTestData() error { _, err = tx.Exec(` INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) -VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +VALUES ('exampleId' ,'` + allocationTx + `','exampleOwnerId','` + pubKey + `',` + fmt.Sprint(expTime) + `,'examplePayerId'); `) if err != nil { return err @@ -379,7 +380,7 @@ VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKe return nil } -func (c *TestDataController) AddGetReferencePathTestData() error { +func (c *TestDataController) AddGetReferencePathTestData(allocationTx, pubkey string) error { var err error var tx *sql.Tx defer func() { @@ -407,7 +408,7 @@ func (c *TestDataController) AddGetReferencePathTestData() error { _, err = tx.Exec(` INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) -VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +VALUES ('exampleId' ,'` + allocationTx + `','exampleOwnerId','` + pubkey + `',` + fmt.Sprint(expTime) + `,'examplePayerId'); `) if err != nil { return err @@ -421,7 +422,7 @@ VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKe return nil } -func (c *TestDataController) AddGetObjectTreeTestData() error { +func (c *TestDataController) AddGetObjectTreeTestData(allocationTx, pubkey string) error { var err error var tx *sql.Tx defer func() { @@ -449,7 +450,7 @@ func (c *TestDataController) AddGetObjectTreeTestData() error { _, err = tx.Exec(` INSERT INTO allocations (id, tx, owner_id, owner_public_key, expiration_date, payer_id) -VALUES ('exampleId' ,'exampleTransaction','exampleOwnerId','exampleOwnerPublicKey',` + fmt.Sprint(expTime) + `,'examplePayerId'); +VALUES ('exampleId' ,'` + allocationTx + `','exampleOwnerId','` + pubkey + `',` + fmt.Sprint(expTime) + `,'examplePayerId'); `) if err != nil { return err diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index d897b7f73..d5fac4bc0 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -9,15 +9,16 @@ import ( "testing" "time" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/spf13/viper" "gorm.io/driver/postgres" "gorm.io/gorm" - "0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "google.golang.org/grpc" - - "0chain.net/blobbercore/blobbergrpc" ) const BlobberAddr = "localhost:7031" @@ -30,7 +31,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { args[arg] = true } if !args["integration"] { - t.Skip() + //t.Skip() } ctx := context.Background() @@ -85,18 +86,48 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { t.Fatal(err) } - getAllocationReq := &blobbergrpc.GetAllocationRequest{ - Context: &blobbergrpc.RequestContext{}, - Id: "exampleTransaction", + testCases := []struct { + name string + input *blobbergrpc.GetAllocationRequest + expectedTx string + expectingError bool + }{ + { + name: "Success", + input: &blobbergrpc.GetAllocationRequest{ + Context: &blobbergrpc.RequestContext{}, + Id: "exampleTransaction", + }, + expectedTx: "exampleTransaction", + expectingError: false, + }, + { + name: "UnknownAllocation", + input: &blobbergrpc.GetAllocationRequest{ + Context: &blobbergrpc.RequestContext{}, + Id: "exampleTransaction1", + }, + expectedTx: "", + expectingError: true, + }, } - getAllocationResp, err := blobberClient.GetAllocation(ctx, getAllocationReq) - if err != nil { - t.Fatal(err) - } + for _, tc := range testCases { + getAllocationResp, err := blobberClient.GetAllocation(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + continue + } - if getAllocationResp.Allocation.Tx != getAllocationReq.Id { - t.Fatal("unexpected allocation id from GetAllocation rpc") + if tc.expectingError { + t.Fatal("expected error") + } + + if getAllocationResp.Allocation.Tx != tc.expectedTx { + t.Fatal("response with wrong allocation transaction") + } } }) @@ -110,174 +141,400 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { t.Fatal(err) } - req := &blobbergrpc.GetFileMetaDataRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", + testCases := []struct { + name string + input *blobbergrpc.GetFileMetaDataRequest + expectedFileName string + expectingError bool + }{ + { + name: "Success", + input: &blobbergrpc.GetFileMetaDataRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + Allocation: "exampleTransaction", + }, + Path: "examplePath", + PathHash: "exampleId:examplePath", + Allocation: "exampleTransaction", + }, + expectedFileName: "filename", + expectingError: false, + }, + { + name: "Unknown file path", + input: &blobbergrpc.GetFileMetaDataRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + Allocation: "exampleTransaction", + }, + Path: "examplePath", + PathHash: "exampleId:examplePath123", + Allocation: "exampleTransaction", + }, + expectedFileName: "", + expectingError: true, }, - Path: "examplePath", - PathHash: "exampleId:examplePath", - Allocation: "exampleTransaction", - } - getFileMetaDataResp, err := blobberClient.GetFileMetaData(ctx, req) - if err != nil { - t.Fatal(err) } - if getFileMetaDataResp.MetaData.FileMetaData.Name != "filename" { - t.Fatal("unexpected file name from GetFileMetaData rpc") + for _, tc := range testCases { + getFileMetaDataResp, err := blobberClient.GetFileMetaData(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + if getFileMetaDataResp.MetaData.FileMetaData.Name != tc.expectedFileName { + t.Fatal("unexpected file name from GetFileMetaData rpc") + } } }) t.Run("TestGetFileStats", func(t *testing.T) { + + allocationTx := randString(32) + + pubKey, _, signScheme := GeneratePubPrivateKey(t) + clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) + err := tdController.ClearDatabase() if err != nil { t.Fatal(err) } - err = tdController.AddGetFileStatsTestData() + err = tdController.AddGetFileStatsTestData(allocationTx, pubKey) if err != nil { t.Fatal(err) } - req := &blobbergrpc.GetFileStatsRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: "exampleTransaction", + testCases := []struct { + name string + input *blobbergrpc.GetFileStatsRequest + expectedFileName string + expectingError bool + }{ + { + name: "Success", + input: &blobbergrpc.GetFileStatsRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: allocationTx, + ClientSignature: clientSignature, + }, + Path: "examplePath", + PathHash: "exampleId:examplePath", + }, + expectedFileName: "filename", + expectingError: false, + }, + { + name: "Unknown Path", + input: &blobbergrpc.GetFileStatsRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: allocationTx, + ClientSignature: clientSignature, + }, + Path: "examplePath", + PathHash: "exampleId:examplePath123", + }, + expectedFileName: "", + expectingError: true, }, - Path: "examplePath", - PathHash: "exampleId:examplePath", } - getFileStatsResp, err := blobberClient.GetFileStats(ctx, req) - if err != nil { - t.Fatal(err) - } + for _, tc := range testCases { + getFileStatsResp, err := blobberClient.GetFileStats(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + continue + } - if getFileStatsResp.MetaData.FileMetaData.Name != "filename" { - t.Fatal("unexpected file name from GetFileStats rpc") + if tc.expectingError { + t.Fatal("expected error") + } + + if getFileStatsResp.MetaData.FileMetaData.Name != tc.expectedFileName { + t.Fatal("unexpected file name from GetFileStats rpc") + } } + }) t.Run("TestListEntities", func(t *testing.T) { + allocationTx := randString(32) + + pubKey, _, signScheme := GeneratePubPrivateKey(t) + clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) + err := tdController.ClearDatabase() if err != nil { t.Fatal(err) } - err = tdController.AddListEntitiesTestData() + err = tdController.AddListEntitiesTestData(allocationTx, pubKey) if err != nil { t.Fatal(err) } - req := &blobbergrpc.ListEntitiesRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: "exampleTransaction", + testCases := []struct { + name string + input *blobbergrpc.ListEntitiesRequest + expectedPath string + expectingError bool + }{ + { + name: "Success", + input: &blobbergrpc.ListEntitiesRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: allocationTx, + ClientSignature: clientSignature, + }, + Path: "examplePath", + PathHash: "exampleId:examplePath", + AuthToken: "", + Allocation: "", + }, + expectedPath: "examplePath", + expectingError: false, + }, + { + name: "bad path", + input: &blobbergrpc.ListEntitiesRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: allocationTx, + ClientSignature: clientSignature, + }, + Path: "examplePath", + PathHash: "exampleId:examplePath123", + AuthToken: "", + Allocation: "", + }, + expectedPath: "", + expectingError: true, }, - Path: "examplePath", - PathHash: "exampleId:examplePath", - AuthToken: "", - Allocation: "", } - listEntitiesResp, err := blobberClient.ListEntities(ctx, req) - if err != nil { - t.Fatal(err) - } + for _, tc := range testCases { + listEntitiesResp, err := blobberClient.ListEntities(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + continue + } - if listEntitiesResp.MetaData.DirMetaData.Path != "examplePath" { - t.Fatal("unexpected path from ListEntities rpc") + if tc.expectingError { + t.Fatal("expected error") + } + + if listEntitiesResp.MetaData.DirMetaData.Path != tc.expectedPath { + t.Fatal("unexpected path from ListEntities rpc") + } } + }) t.Run("TestGetObjectPath", func(t *testing.T) { + allocationTx := randString(32) + + pubKey, _, signScheme := GeneratePubPrivateKey(t) + clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) + err := tdController.ClearDatabase() if err != nil { t.Fatal(err) } - err = tdController.AddGetObjectPathTestData() + err = tdController.AddGetObjectPathTestData(allocationTx, pubKey) if err != nil { t.Fatal(err) } - req := &blobbergrpc.GetObjectPathRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: "exampleTransaction", + testCases := []struct { + name string + input *blobbergrpc.GetObjectPathRequest + expectedPath string + expectingError bool + }{ + { + name: "Success", + input: &blobbergrpc.GetObjectPathRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: allocationTx, + ClientSignature: clientSignature, + }, + Allocation: "", + Path: "examplePath", + BlockNum: "0", + }, + expectedPath: "/", + expectingError: false, }, - Allocation: "", - Path: "examplePath", - BlockNum: "0", } - getObjectPathResp, err := blobberClient.GetObjectPath(ctx, req) - if err != nil { - t.Fatal(err) - } + for _, tc := range testCases { + getObjectPathResp, err := blobberClient.GetObjectPath(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + continue + } - if getObjectPathResp.ObjectPath.Path.DirMetaData.Path != "/" { - t.Fatal("unexpected root hash from GetObjectPath rpc") + if tc.expectingError { + t.Fatal("expected error") + } + + if getObjectPathResp.ObjectPath.Path.DirMetaData.Path != tc.expectedPath { + t.Fatal("unexpected root hash from GetObjectPath rpc") + } } }) t.Run("TestGetReferencePath", func(t *testing.T) { + allocationTx := randString(32) + + pubKey, _, signScheme := GeneratePubPrivateKey(t) + clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) + err := tdController.ClearDatabase() if err != nil { t.Fatal(err) } - err = tdController.AddGetReferencePathTestData() + err = tdController.AddGetReferencePathTestData(allocationTx, pubKey) if err != nil { t.Fatal(err) } - req := &blobbergrpc.GetReferencePathRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: "exampleTransaction", + testCases := []struct { + name string + input *blobbergrpc.GetReferencePathRequest + expectedPath string + expectingError bool + }{ + { + name: "Success", + input: &blobbergrpc.GetReferencePathRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: allocationTx, + ClientSignature: clientSignature, + }, + Paths: "", + Path: "/", + Allocation: "", + }, + expectedPath: "/", + expectingError: false, }, - Paths: "", - Path: "/", - Allocation: "", - } - getReferencePathResp, err := blobberClient.GetReferencePath(ctx, req) - if err != nil { - t.Fatal(err) } - if getReferencePathResp.ReferencePath.MetaData.DirMetaData.Path != "/" { - t.Fatal("unexpected path from GetReferencePath rpc") + for _, tc := range testCases { + getReferencePathResp, err := blobberClient.GetReferencePath(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + if getReferencePathResp.ReferencePath.MetaData.DirMetaData.Path != tc.expectedPath { + t.Fatal("unexpected path from GetReferencePath rpc") + } } }) t.Run("TestGetObjectTree", func(t *testing.T) { + allocationTx := randString(32) + + pubKey, _, signScheme := GeneratePubPrivateKey(t) + clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) + err := tdController.ClearDatabase() if err != nil { t.Fatal(err) } - err = tdController.AddGetObjectTreeTestData() + err = tdController.AddGetObjectTreeTestData(allocationTx, pubKey) if err != nil { t.Fatal(err) } - req := &blobbergrpc.GetObjectTreeRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: "exampleTransaction", + testCases := []struct { + name string + input *blobbergrpc.GetObjectTreeRequest + expectedFileName string + expectingError bool + }{ + { + name: "Success", + input: &blobbergrpc.GetObjectTreeRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: allocationTx, + ClientSignature: clientSignature, + }, + Path: "/", + Allocation: "", + }, + expectedFileName: "root", + expectingError: false, + }, + { + name: "bad path", + input: &blobbergrpc.GetObjectTreeRequest{ + Context: &blobbergrpc.RequestContext{ + Client: "exampleOwnerId", + ClientKey: "", + Allocation: allocationTx, + ClientSignature: clientSignature, + }, + Path: "/2", + Allocation: "", + }, + expectedFileName: "root", + expectingError: true, }, - Path: "/", - Allocation: "", - } - getObjectTreeResp, err := blobberClient.GetObjectTree(ctx, req) - if err != nil { - t.Fatal(err) } - if getObjectTreeResp.ReferencePath.MetaData.DirMetaData.Name != "root" { - t.Fatal("unexpected root name from GetObject") + for _, tc := range testCases { + + getObjectTreeResp, err := blobberClient.GetObjectTree(ctx, tc.input) + if err != nil { + if !tc.expectingError { + t.Fatal(err) + } + continue + } + + if tc.expectingError { + t.Fatal("expected error") + } + + if getObjectTreeResp.ReferencePath.MetaData.DirMetaData.Name != tc.expectedFileName { + t.Fatal("unexpected root name from GetObject") + } } + }) } diff --git a/code/go/0chain.net/core/common/rate_limiter.go b/code/go/0chain.net/core/common/rate_limiter.go index a4cacdda0..6bb04168f 100644 --- a/code/go/0chain.net/core/common/rate_limiter.go +++ b/code/go/0chain.net/core/common/rate_limiter.go @@ -32,7 +32,7 @@ func (rl *ratelimit) init() { const DefaultRequestPerSecond = 100000 //ConfigRateLimits - configure the rate limits -func ConfigRateLimits() *GRPCRateLimiter { +func ConfigRateLimits() { userRl := viper.GetFloat64("handlers.rate_limit") if userRl == 0 { @@ -41,6 +41,14 @@ func ConfigRateLimits() *GRPCRateLimiter { userRateLimit = &ratelimit{RequestsPerSecond: userRl} userRateLimit.init() +} + +func NewGRPCRateLimiter() *GRPCRateLimiter { + userRl := viper.GetFloat64("handlers.rate_limit") + + if userRl == 0 { + userRl = DefaultRequestPerSecond + } return &GRPCRateLimiter{rl.New(int(userRl))} } From fc7383c3289899577b2fc55c9a2848d7d6670dfe Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Fri, 21 May 2021 20:25:05 +0530 Subject: [PATCH 13/20] :green_heart: fix tests --- .../blobbercore/handler/grpc_handler_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index d5fac4bc0..fba4da54b 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -31,7 +31,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { args[arg] = true } if !args["integration"] { - //t.Skip() + t.Skip() } ctx := context.Background() From f2d004ac32e2c4b76317df79ecf0a02216b4c4a0 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Fri, 21 May 2021 22:04:18 +0530 Subject: [PATCH 14/20] :white_check_mark: updated tests to handle metadata --- .../handler/grpc_handler_integration_test.go | 149 +++++++++--------- 1 file changed, 75 insertions(+), 74 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index fba4da54b..88ff120d7 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -9,6 +9,10 @@ import ( "testing" "time" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + + "google.golang.org/grpc/metadata" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/spf13/viper" @@ -31,11 +35,9 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { args[arg] = true } if !args["integration"] { - t.Skip() + //t.Skip() } - ctx := context.Background() - var conn *grpc.ClientConn var err error for i := 0; i < RetryAttempts; i++ { @@ -95,8 +97,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { { name: "Success", input: &blobbergrpc.GetAllocationRequest{ - Context: &blobbergrpc.RequestContext{}, - Id: "exampleTransaction", + Id: "exampleTransaction", }, expectedTx: "exampleTransaction", expectingError: false, @@ -104,8 +105,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { { name: "UnknownAllocation", input: &blobbergrpc.GetAllocationRequest{ - Context: &blobbergrpc.RequestContext{}, - Id: "exampleTransaction1", + Id: "exampleTransaction1", }, expectedTx: "", expectingError: true, @@ -113,7 +113,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } for _, tc := range testCases { - getAllocationResp, err := blobberClient.GetAllocation(ctx, tc.input) + getAllocationResp, err := blobberClient.GetAllocation(context.Background(), tc.input) if err != nil { if !tc.expectingError { t.Fatal(err) @@ -143,17 +143,17 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { testCases := []struct { name string + context metadata.MD input *blobbergrpc.GetFileMetaDataRequest expectedFileName string expectingError bool }{ { name: "Success", + context: metadata.New(map[string]string{ + common.ClientHeader: "exampleOwnerId", + }), input: &blobbergrpc.GetFileMetaDataRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - Allocation: "exampleTransaction", - }, Path: "examplePath", PathHash: "exampleId:examplePath", Allocation: "exampleTransaction", @@ -163,11 +163,10 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { }, { name: "Unknown file path", + context: metadata.New(map[string]string{ + common.ClientHeader: "exampleOwnerId", + }), input: &blobbergrpc.GetFileMetaDataRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - Allocation: "exampleTransaction", - }, Path: "examplePath", PathHash: "exampleId:examplePath123", Allocation: "exampleTransaction", @@ -178,6 +177,8 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } for _, tc := range testCases { + ctx := context.Background() + ctx = metadata.NewOutgoingContext(ctx, tc.context) getFileMetaDataResp, err := blobberClient.GetFileMetaData(ctx, tc.input) if err != nil { if !tc.expectingError { @@ -214,36 +215,35 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { testCases := []struct { name string + context metadata.MD input *blobbergrpc.GetFileStatsRequest expectedFileName string expectingError bool }{ { name: "Success", + context: metadata.New(map[string]string{ + common.ClientHeader: "exampleOwnerId", + common.ClientSignatureHeader: clientSignature, + }), input: &blobbergrpc.GetFileStatsRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: allocationTx, - ClientSignature: clientSignature, - }, - Path: "examplePath", - PathHash: "exampleId:examplePath", + Path: "examplePath", + PathHash: "exampleId:examplePath", + Allocation: allocationTx, }, expectedFileName: "filename", expectingError: false, }, { name: "Unknown Path", + context: metadata.New(map[string]string{ + common.ClientHeader: "exampleOwnerId", + common.ClientSignatureHeader: clientSignature, + }), input: &blobbergrpc.GetFileStatsRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: allocationTx, - ClientSignature: clientSignature, - }, - Path: "examplePath", - PathHash: "exampleId:examplePath123", + Path: "examplePath", + PathHash: "exampleId:examplePath123", + Allocation: allocationTx, }, expectedFileName: "", expectingError: true, @@ -251,6 +251,8 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } for _, tc := range testCases { + ctx := context.Background() + ctx = metadata.NewOutgoingContext(ctx, tc.context) getFileStatsResp, err := blobberClient.GetFileStats(ctx, tc.input) if err != nil { if !tc.expectingError { @@ -287,40 +289,37 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { testCases := []struct { name string + context metadata.MD input *blobbergrpc.ListEntitiesRequest expectedPath string expectingError bool }{ { name: "Success", + context: metadata.New(map[string]string{ + common.ClientHeader: "exampleOwnerId", + common.ClientSignatureHeader: clientSignature, + }), input: &blobbergrpc.ListEntitiesRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: allocationTx, - ClientSignature: clientSignature, - }, Path: "examplePath", PathHash: "exampleId:examplePath", AuthToken: "", - Allocation: "", + Allocation: allocationTx, }, expectedPath: "examplePath", expectingError: false, }, { name: "bad path", + context: metadata.New(map[string]string{ + common.ClientHeader: "exampleOwnerId", + common.ClientSignatureHeader: clientSignature, + }), input: &blobbergrpc.ListEntitiesRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: allocationTx, - ClientSignature: clientSignature, - }, Path: "examplePath", PathHash: "exampleId:examplePath123", AuthToken: "", - Allocation: "", + Allocation: allocationTx, }, expectedPath: "", expectingError: true, @@ -328,6 +327,8 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } for _, tc := range testCases { + ctx := context.Background() + ctx = metadata.NewOutgoingContext(ctx, tc.context) listEntitiesResp, err := blobberClient.ListEntities(ctx, tc.input) if err != nil { if !tc.expectingError { @@ -364,20 +365,19 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { testCases := []struct { name string + context metadata.MD input *blobbergrpc.GetObjectPathRequest expectedPath string expectingError bool }{ { name: "Success", + context: metadata.New(map[string]string{ + common.ClientHeader: "exampleOwnerId", + common.ClientSignatureHeader: clientSignature, + }), input: &blobbergrpc.GetObjectPathRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: allocationTx, - ClientSignature: clientSignature, - }, - Allocation: "", + Allocation: allocationTx, Path: "examplePath", BlockNum: "0", }, @@ -387,6 +387,8 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } for _, tc := range testCases { + ctx := context.Background() + ctx = metadata.NewOutgoingContext(ctx, tc.context) getObjectPathResp, err := blobberClient.GetObjectPath(ctx, tc.input) if err != nil { if !tc.expectingError { @@ -422,22 +424,21 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { testCases := []struct { name string + context metadata.MD input *blobbergrpc.GetReferencePathRequest expectedPath string expectingError bool }{ { name: "Success", + context: metadata.New(map[string]string{ + common.ClientHeader: "exampleOwnerId", + common.ClientSignatureHeader: clientSignature, + }), input: &blobbergrpc.GetReferencePathRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: allocationTx, - ClientSignature: clientSignature, - }, Paths: "", Path: "/", - Allocation: "", + Allocation: allocationTx, }, expectedPath: "/", expectingError: false, @@ -445,6 +446,8 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } for _, tc := range testCases { + ctx := context.Background() + ctx = metadata.NewOutgoingContext(ctx, tc.context) getReferencePathResp, err := blobberClient.GetReferencePath(ctx, tc.input) if err != nil { if !tc.expectingError { @@ -480,34 +483,31 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { testCases := []struct { name string + context metadata.MD input *blobbergrpc.GetObjectTreeRequest expectedFileName string expectingError bool }{ { name: "Success", + context: metadata.New(map[string]string{ + common.ClientHeader: "exampleOwnerId", + common.ClientSignatureHeader: clientSignature, + }), input: &blobbergrpc.GetObjectTreeRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: allocationTx, - ClientSignature: clientSignature, - }, Path: "/", - Allocation: "", + Allocation: allocationTx, }, expectedFileName: "root", expectingError: false, }, { name: "bad path", + context: metadata.New(map[string]string{ + common.ClientHeader: "exampleOwnerId", + common.ClientSignatureHeader: clientSignature, + }), input: &blobbergrpc.GetObjectTreeRequest{ - Context: &blobbergrpc.RequestContext{ - Client: "exampleOwnerId", - ClientKey: "", - Allocation: allocationTx, - ClientSignature: clientSignature, - }, Path: "/2", Allocation: "", }, @@ -517,7 +517,8 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { } for _, tc := range testCases { - + ctx := context.Background() + ctx = metadata.NewOutgoingContext(ctx, tc.context) getObjectTreeResp, err := blobberClient.GetObjectTree(ctx, tc.input) if err != nil { if !tc.expectingError { From 577a6a0c99faeffe6b37303e63f5064e549ba998 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Fri, 21 May 2021 22:07:26 +0530 Subject: [PATCH 15/20] :green_heart: fix test --- .../blobbercore/handler/grpc_handler_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 88ff120d7..107c1a5e5 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -35,7 +35,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { args[arg] = true } if !args["integration"] { - //t.Skip() + t.Skip() } var conn *grpc.ClientConn From a32fcf20286499ae88c5e657bff08aeb3e16256f Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Fri, 21 May 2021 22:33:20 +0530 Subject: [PATCH 16/20] :white_check_mark: updated tests --- .../blobbercore/handler/grpc_handler_helper_unit_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index a52ffd827..bc8e50864 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -11,8 +11,6 @@ import ( "gorm.io/gorm" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - coreConfig "github.com/0chain/blobber/code/go/0chain.net/core/config" "github.com/0chain/gosdk/core/zcncrypto" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" @@ -474,8 +472,7 @@ VALUES (1234,'exampleId','exampleId:examplePath','exampleId:examplePath','d','ro func GeneratePubPrivateKey(t *testing.T) (string, string, zcncrypto.SignatureScheme) { - config.Configuration.Config = &coreConfig.Config{SignatureScheme: "bls0chain"} - signScheme := zcncrypto.NewSignatureScheme(config.Configuration.SignatureScheme) + signScheme := zcncrypto.NewSignatureScheme("bls0chain") wallet, err := signScheme.GenerateKeys() if err != nil { t.Fatal(err) From 1f7d3c36885121946d8d7afa6eab5135bb838385 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Sat, 22 May 2021 21:47:52 +0530 Subject: [PATCH 17/20] added integration test to make file to fix github action --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 79477b07b..ef5d2f84d 100644 --- a/Makefile +++ b/Makefile @@ -4,4 +4,7 @@ test: go test ./...; lint: - golangci-lint run; \ No newline at end of file + golangci-lint run; + +integration-tests: + go test ./... -args integration; \ No newline at end of file From caf3733c0bea8a9d50d88e51fa96b7a76ae39dab Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Sun, 23 May 2021 11:51:25 +0530 Subject: [PATCH 18/20] :art: simplified test implementation --- .../handler/grpc_handler_helper_unit_test.go | 22 +++++++++++++++++++ .../handler/grpc_handler_integration_test.go | 20 +++-------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go index bc8e50864..0b286f5e1 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_helper_unit_test.go @@ -5,8 +5,13 @@ import ( "database/sql" "fmt" "log" + "os" + "strings" "time" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/spf13/viper" + "testing" "gorm.io/gorm" @@ -484,3 +489,20 @@ func GeneratePubPrivateKey(t *testing.T) (string, string, zcncrypto.SignatureSch return keyPair.PublicKey, keyPair.PrivateKey, signScheme } + +func setupIntegrationTestConfig(t *testing.T) { + + pwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + configDir := strings.Split(pwd, "/code/go")[0] + "/config" + config.SetupDefaultConfig() + config.SetupConfig(configDir) + + config.Configuration.DBHost = "localhost" + config.Configuration.DBName = viper.GetString("db.name") + config.Configuration.DBPort = viper.GetString("db.port") + config.Configuration.DBUserName = viper.GetString("db.user") + config.Configuration.DBPassword = viper.GetString("db.password") +} diff --git a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go index 107c1a5e5..bde386249 100644 --- a/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go +++ b/code/go/0chain.net/blobbercore/handler/grpc_handler_integration_test.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "os" - "strings" "testing" "time" @@ -15,8 +14,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/encryption" - "github.com/spf13/viper" - "gorm.io/driver/postgres" "gorm.io/gorm" @@ -25,7 +22,7 @@ import ( "google.golang.org/grpc" ) -const BlobberAddr = "localhost:7031" +const BlobberTestAddr = "localhost:7031" const RetryAttempts = 8 const RetryTimeout = 3 @@ -42,7 +39,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { var err error for i := 0; i < RetryAttempts; i++ { log.Println("Connection attempt - " + fmt.Sprint(i+1)) - conn, err = grpc.Dial(BlobberAddr, grpc.WithInsecure()) + conn, err = grpc.Dial(BlobberTestAddr, grpc.WithInsecure()) if err != nil { log.Println(err) <-time.After(time.Second * RetryTimeout) @@ -56,18 +53,7 @@ func TestBlobberGRPCService_IntegrationTest(t *testing.T) { defer conn.Close() blobberClient := blobbergrpc.NewBlobberClient(conn) - pwd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - configDir := strings.Split(pwd, "/code/go")[0] + "/config" - config.SetupDefaultConfig() - config.SetupConfig(configDir) - config.Configuration.DBHost = "localhost" - config.Configuration.DBName = viper.GetString("db.name") - config.Configuration.DBPort = viper.GetString("db.port") - config.Configuration.DBUserName = viper.GetString("db.user") - config.Configuration.DBPassword = viper.GetString("db.password") + setupIntegrationTestConfig(t) db, err := gorm.Open(postgres.Open(fmt.Sprintf( "host=%v port=%v user=%v dbname=%v password=%v sslmode=disable", config.Configuration.DBHost, config.Configuration.DBPort, From 02da8de06f6723dd6fcb92af493fbbd5262cb670 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Sun, 23 May 2021 11:53:56 +0530 Subject: [PATCH 19/20] :green_heart: remove uneeded github actions --- .github/workflows/ci.yml | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b23a7f6a..d67c9f1ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ env: VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} jobs: - integration-tests: + build-and-test: runs-on: ubuntu-20.04 steps: - name: Install Go @@ -34,37 +34,6 @@ jobs: ../bin/blobber.start_bls.sh /dev/null & cd ../.. make integration-tests - - build: - runs-on: ubuntu-20.04 - steps: - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: 1.14.x - - uses: actions/checkout@v2 - # In this step, this action saves a list of existing images, - # the cache is created without them in the post run. - # It also restores the cache if it exists. - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: Build - run: | - docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0 - ./docker.local/bin/build.blobber.sh - - test: - runs-on: ubuntu-20.04 - steps: - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: 1.14.x - - uses: actions/checkout@v2 - - name: Test - run: make test - lint: runs-on: ubuntu-20.04 steps: From 71e06a30c9647800c2e23a7476f9ab76fa54c009 Mon Sep 17 00:00:00 2001 From: Shravan Shetty Date: Sun, 23 May 2021 11:56:03 +0530 Subject: [PATCH 20/20] :green_heart: renamed github action to test --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d67c9f1ce..2e913f0ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ env: VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} jobs: - build-and-test: + test: runs-on: ubuntu-20.04 steps: - name: Install Go