Adding grpc endpoints

.github/workflows/ci.yml

@@ -5,13 +5,16 @@ on:
   release:
     types:
       - published
+  pull_request:
+    paths-ignore:
+      - '**.md'
 
 env:
   BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }}
   VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }}
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-20.04
     steps:
       - name: Install Go
@@ -25,21 +28,15 @@ jobs:
       - uses: satackey/action-docker-layer-caching@v0.0.11
         # Ignore the failure of a step and avoid terminating the job.
         continue-on-error: true
-      - name: Build
+      - name: Build test environment and run tests
         run: |
+          ./docker.local/bin/blobber.init.setup.sh
           docker network create --driver=bridge --subnet=198.18.0.0/15 --gateway=198.18.0.255 testnet0
           ./docker.local/bin/build.blobber.sh
-  test:
-    runs-on: ubuntu-20.04
-    steps:
-    - name: Install Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.14.x
-    - uses: actions/checkout@v2
-    - name: Test
-      run: make test
-
+          cd docker.local/blobber1
+          ../bin/blobber.start_bls.sh </dev/null &>/dev/null &
+          cd ../..
+          make integration-tests
   lint:
     runs-on: ubuntu-20.04
     steps:
@@ -76,10 +73,9 @@ jobs:
           git reset --hard
       - name: Generate Files
         run: |
-          cd code/go/0chain.net/blobbercore
           ./scripts/generate-grpc.sh
       - name: Fail if any file has changed
-        run: if output=$(git status --porcelain) && [ -z "$output" ]; then exit 0; else git status; exit 1; fi;
+        run: if output=$(git status --porcelain) && [ -z "$output" ]; then exit 0; else git status; git diff; exit 1; fi;
 
   dockerize_blobber:
     runs-on: ubuntu-20.04

Makefile

@@ -1,7 +1,10 @@
-.PHONY: test lint
+.PHONY: test lint integration-tests
 
 test:
-	cd code/go/0chain.net; go test ./...;
+	go test ./...;
 
 lint:
-	cd code/go/0chain.net; golangci-lint run;
+	golangci-lint run --timeout 2m0s;
+
+integration-tests:
+	sudo go test ./... -args integration;

code/go/0chain.net/blobber/integration_tests.go

@@ -3,9 +3,9 @@
 package main
 
 import (
-	"0chain.net/core/logging"
+	"github.com/0chain/blobber/code/go/0chain.net/core/logging"
 
-	crpc "0chain.net/conductor/conductrpc" // integration tests
+	crpc "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" // integration tests
 )
 
 // start lock, where the miner is ready to connect to blockchain (BC)

code/go/0chain.net/blobber/main.go

@@ -12,27 +12,30 @@ import (
 	"runtime"
 	"strconv"
 	"time"
-	"go.uber.org/zap"
+
+	"google.golang.org/grpc/reflection"
+
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/challenge"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker"
+	"github.com/0chain/blobber/code/go/0chain.net/core/build"
+	"github.com/0chain/blobber/code/go/0chain.net/core/chain"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
+	"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
+	"github.com/0chain/blobber/code/go/0chain.net/core/logging"
+	. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
+	"github.com/0chain/blobber/code/go/0chain.net/core/node"
+
+	"github.com/0chain/gosdk/zcncore"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
 	"github.com/spf13/viper"
-
-	"github.com/0chain/gosdk/zcncore"
-	"0chain.net/blobbercore/allocation"
-	"0chain.net/blobbercore/challenge"
-	"0chain.net/blobbercore/config"
-	"0chain.net/blobbercore/datastore"
-	"0chain.net/blobbercore/filestore"
-	"0chain.net/blobbercore/handler"
-	"0chain.net/blobbercore/readmarker"
-	"0chain.net/blobbercore/writemarker"
-	"0chain.net/core/build"
-	"0chain.net/core/chain"
-	"0chain.net/core/common"
-	"0chain.net/core/encryption"
-	"0chain.net/core/node"
-	"0chain.net/core/logging"
-	. "0chain.net/core/logging"
+	"go.uber.org/zap"
 )
 
 var startTime time.Time
@@ -41,7 +44,7 @@ var filesDir *string
 var metadataDB *string
 
 func initHandlers(r *mux.Router) {
-	r.HandleFunc("/", func (w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		mc := chain.GetServerChain()
 
 		fmt.Fprintf(w, "<div>Running since %v ...\n", startTime)
@@ -285,7 +288,7 @@ func healthCheckOnChainWorker() {
 
 func setup(logDir string) error {
 	// init blockchain related stuff
-	zcncore.SetLogFile(logDir + "/0chainBlobber.log", false)
+	zcncore.SetLogFile(logDir+"/0chainBlobber.log", false)
 	zcncore.SetLogLevel(3)
 	if err := zcncore.InitZCNSDK(serverChain.BlockWorker, config.Configuration.SignatureScheme); err != nil {
 		return err
@@ -335,7 +338,7 @@ func main() {
 	flag.Parse()
 
 	config.SetupDefaultConfig()
-	config.SetupConfig()
+	config.SetupConfig("./config")
 
 	config.Configuration.DeploymentMode = byte(*deploymentMode)
 
@@ -364,10 +367,6 @@ func main() {
 		panic("Please specify --port which is the port on which requests are accepted")
 	}
 
-	if *grpcPortString == "" {
-		panic("Please specify --grpc_port which is the grpc port on which requests are accepted")
-	}
-
 	reader, err := os.Open(*keysFile)
 	if err != nil {
 		panic(err)
@@ -452,11 +451,14 @@ func main() {
 	methodsOk := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT",
 		"DELETE", "OPTIONS"})
 
-	rl := common.ConfigRateLimits()
+	common.ConfigRateLimits()
 	initHandlers(r)
 
-	grpcServer := handler.NewServerWithMiddlewares(rl)
-	handler.RegisterGRPCServices(r, grpcServer)
+	grpcServer := handler.NewGRPCServerWithMiddlewares(common.NewGRPCRateLimiter(), r)
+
+	if config.Development() {
+		reflection.Register(grpcServer)
+	}
 
 	rHandler := handlers.CORS(originsOk, headersOk, methodsOk)(r)
 	if config.Development() {
@@ -482,12 +484,24 @@ func main() {
 
 	Logger.Info("Ready to listen to the requests")
 	startTime = time.Now().UTC()
-	go func(grpcPort string) {
+	go func(gp *string) {
+		var grpcPort string
+		if gp != nil {
+			grpcPort = *gp
+		}
+
+		if grpcPort == "" {
+			Logger.Error("Could not start grpc server since grpc port has not been specified." +
+				" Please specify the grpc port in the --grpc_port build arguement to start the grpc server")
+			return
+		}
+
+		Logger.Info("listening too grpc requests on port - " + grpcPort)
 		lis, err := net.Listen("tcp", fmt.Sprintf(":%s", grpcPort))
 		if err != nil {
 			log.Fatalf("failed to listen: %v", err)
 		}
 		log.Fatal(grpcServer.Serve(lis))
-	}(*grpcPortString)
+	}(grpcPortString)
 	log.Fatal(server.ListenAndServe())
 }

code/go/0chain.net/blobbercore/allocation/allocationchange.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"errors"
 
-	"0chain.net/blobbercore/datastore"
-	"0chain.net/blobbercore/reference"
-	"0chain.net/core/common"
-	"0chain.net/core/logging"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
+	"github.com/0chain/blobber/code/go/0chain.net/core/logging"
 
 	"go.uber.org/zap"
 	"gorm.io/gorm"

code/go/0chain.net/blobbercore/allocation/attributesfilechange.go

@@ -5,11 +5,11 @@ import (
 	"encoding/json"
 	"path/filepath"
 
-	"0chain.net/blobbercore/reference"
-	"0chain.net/blobbercore/stats"
-	"0chain.net/core/common"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
 
-	. "0chain.net/core/logging"
+	. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
 	"go.uber.org/zap"
 )
 

code/go/0chain.net/blobbercore/allocation/copyfilechange.go

@@ -5,8 +5,8 @@ import (
 	"encoding/json"
 	"path/filepath"
 
-	"0chain.net/blobbercore/reference"
-	"0chain.net/core/common"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
 
 	"gorm.io/datatypes"
 )

code/go/0chain.net/blobbercore/allocation/deletefilechange.go

@@ -5,11 +5,11 @@ import (
 	"encoding/json"
 	"path/filepath"
 
-	"0chain.net/blobbercore/datastore"
-	"0chain.net/blobbercore/filestore"
-	"0chain.net/blobbercore/reference"
-	"0chain.net/core/common"
-	. "0chain.net/core/logging"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
+	. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
 
 	"go.uber.org/zap"
 )

code/go/0chain.net/blobbercore/allocation/entity.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"time"
 
-	"0chain.net/core/common"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
 
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"

code/go/0chain.net/blobbercore/allocation/newfilechange.go

@@ -6,11 +6,11 @@ import (
 	"path/filepath"
 	"strings"
 
-	"0chain.net/blobbercore/filestore"
-	"0chain.net/blobbercore/reference"
-	"0chain.net/blobbercore/stats"
-	"0chain.net/blobbercore/util"
-	"0chain.net/core/common"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/util"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
 )
 
 type NewFileChange struct {

code/go/0chain.net/blobbercore/allocation/protocol.go

@@ -6,12 +6,12 @@ import (
 	"errors"
 	"fmt"
 
-	"0chain.net/blobbercore/datastore"
-	"0chain.net/core/chain"
-	"0chain.net/core/common"
-	. "0chain.net/core/logging"
-	"0chain.net/core/node"
-	"0chain.net/core/transaction"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
+	"github.com/0chain/blobber/code/go/0chain.net/core/chain"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
+	. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
+	"github.com/0chain/blobber/code/go/0chain.net/core/node"
+	"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
 
 	"gorm.io/gorm"
 )

code/go/0chain.net/blobbercore/allocation/renamefilechange.go

@@ -5,10 +5,10 @@ import (
 	"encoding/json"
 	"path/filepath"
 
-	"0chain.net/blobbercore/reference"
-	"0chain.net/blobbercore/stats"
-	"0chain.net/core/common"
-	."0chain.net/core/logging"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
+	. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
 
 	"go.uber.org/zap"
 )
@@ -76,7 +76,7 @@ func (rf *RenameFileChange) ProcessChange(ctx context.Context, change *Allocatio
 		}
 	}
 	if idx < 0 {
-		Logger.Error("error in file rename", zap.Any("change",rf))
+		Logger.Error("error in file rename", zap.Any("change", rf))
 		return nil, common.NewError("file_not_found", "File to rename not found in blobber")
 	}
 	//dirRef.Children[idx] = affectedRef

code/go/0chain.net/blobbercore/allocation/updatefilechange.go

@@ -5,12 +5,12 @@ import (
 	"encoding/json"
 	"path/filepath"
 
-	"0chain.net/blobbercore/filestore"
-	"0chain.net/blobbercore/stats"
-	"0chain.net/blobbercore/reference"
-	"0chain.net/blobbercore/util"
-	"0chain.net/core/common"
-	. "0chain.net/core/logging"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/util"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
+	. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
 
 	"go.uber.org/zap"
 )

code/go/0chain.net/blobbercore/allocation/workers.go

@@ -8,16 +8,16 @@ import (
 	"math/big"
 	"time"
 
-	"0chain.net/blobbercore/datastore"
-	"0chain.net/blobbercore/reference"
-	"0chain.net/core/chain"
-	"0chain.net/core/common"
-	"0chain.net/core/lock"
-	"0chain.net/core/transaction"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
+	"github.com/0chain/blobber/code/go/0chain.net/core/chain"
+	"github.com/0chain/blobber/code/go/0chain.net/core/common"
+	"github.com/0chain/blobber/code/go/0chain.net/core/lock"
+	"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
 
 	"gorm.io/gorm"
 
-	. "0chain.net/core/logging"
+	. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
 	"go.uber.org/zap"
 )