Merge pull request #614 from fairDataSociety/staging

chore: #613, update bee to v2

.golangci.yml

@@ -1,7 +1,12 @@
 run:
   timeout: 10m
+  concurrency: 4
+  tests: false
+
 linters:
   enable:
     - misspell
     - gofmt
     - unconvert
+issues:
+  exclude-dirs: ["wasm"]

Makefile

@@ -1,6 +1,6 @@
 GO ?= go
 GOLANGCI_LINT ?= $$($(GO) env GOPATH)/bin/golangci-lint
-GOLANGCI_LINT_VERSION ?= v1.55.2
+GOLANGCI_LINT_VERSION ?= v1.57.1
 GOGOPROTOBUF ?= protoc-gen-gogofaster
 GOGOPROTOBUF_VERSION ?= v1.3.1
 
@@ -24,7 +24,7 @@ dist:
 
 .PHONY: lint
 lint: linter
-	$(GOLANGCI_LINT) run --skip-dirs wasm
+	$(GOLANGCI_LINT) run
 
 .PHONY: linter
 linter:

README.md

@@ -200,4 +200,91 @@ network: "testnet"
 bee:
   bee-api-endpoint: http://localhost:1633 # bee running on mainnet  
   postage-batch-id: <BATCH>
-```
+```
+
+### Integrating git with dfs
+
+To integrate git with dfs, we need to set the `git` configuration in the config file. We only need to set the credential helper for local git repositories.
+
+We create a file `.dfs-credentials` with the following content at any given location
+```
+#!/bin/bash
+token_file="<ABSOLUTE PATH FOR STRING ACCESS TOKEN>" # this needs to be absolute path
+
+username="<USERNAME>"
+password="<PASSWORD>"
+
+dfs="<FAIROS-DFS SERVER URL>" # http://localhost:9090 for local running fairOS-dfs server
+
+# Function to get the access token using the username and password
+get_access_token() {
+    local response=$(curl -s -X POST "$dfs/v2/user/login" -H "Content-Type: application/json" -d "{\"userName\": \"$username\", \"password\": \"$password\"}")
+    access_token=$(echo "$response" | jq -r '.accessToken')
+    # check if response has access token
+    if [ "$access_token" == "null"  ]; then
+        exit 1
+    fi
+    echo "$access_token" > "$token_file"
+    echo "$access_token"
+}
+
+get_saved_access_token() {
+    if [[ -f "$token_file" ]]; then
+        local saved_token=$(sed -n '1p' "$token_file")
+        if [ "$saved_token" == "null" ] || [ "$saved_token" == "" ]; then
+            return 1
+        fi
+        local response=$(curl --write-out '%{http_code}' --silent --output /dev/null  -s -X POST "$dfs/v1/user/stat" -H "Content-Type: application/json" -H "Authorisation: Bearer $saved_token" )
+        # check if response had success http code
+        if [[ response -eq 200 ]]; then
+            echo "$saved_token"
+            return 0
+        else
+            rm "$token_file"
+            return 1
+        fi
+    fi
+    return 1
+}
+
+access_token=$(get_saved_access_token)
+if [[ $? -ne 0 ]]; then
+    access_token=$(get_access_token)
+fi
+echo "username=$username"
+echo "password=$access_token"
+
+exit 0
+```
+
+After creating this file, we need to set the `credential.helper` in the git configuration
+
+```
+git config --local credential.helper "<Absolute path to .dfs-credentials file>"
+```
+
+#### How to push to dfs
+
+Currently, we only support pushing once, so its sort of archive. We can push the git repository to dfs using the following command
+
+```
+git init # initialize the git repository, run this inside the directory that you want to push to dfs
+git add . # add all the files to the git repository
+git commit -m "Initial commit" # commit the changes
+git remote add origin <DFS SERVER>/v1/git/<USERNAME>/<PODNAME>.git # add the remote origin
+git config --local credential.helper "<Absolute path to .dfs-credentials file>" # set the credential helper
+git push -u origin master # push the changes to the remote origin
+```
+
+### How to clone from dfs
+
+```
+git config --local credential.helper "<Absolute path to .dfs-credentials file>"
+git clone <DFS SERVER>/v1/git/<USERNAME>/<PODNAME>.git
+```
+
+NOTE: Before pushing into a pod, we have to first create a pod if it does not exist. A pod can not be used as two different repos. 
+Dfs stores the files of the repo as a git pack file. So, we cannot access each file of the repo individually. although we can access other files from dfs api as expected.
+
+
+

cmd/dfs/cmd/config.go

@@ -12,6 +12,7 @@ var (
 	optionVerbosity          = "verbosity"
 	optionBeeApi             = "bee.bee-api-endpoint"
 	optionBeePostageBatchId  = "bee.postage-batch-id"
+	optionBeeRedundancyLevel = "bee.redundancy-level"
 	optionFeedCacheSize      = "feed.cache-size"
 	optionFeedCacheTTL       = "feed.cache-ttl"
 	optionCookieDomain       = "cookie-domain"

cmd/dfs/cmd/dev.go

@@ -8,8 +8,8 @@ import (
 	"syscall"
 	"testing"
 
-	mockpost "github.com/ethersphere/bee/pkg/postage/mock"
-	mockstorer "github.com/ethersphere/bee/pkg/storer/mock"
+	mockpost "github.com/ethersphere/bee/v2/pkg/postage/mock"
+	mockstorer "github.com/ethersphere/bee/v2/pkg/storer/mock"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/api"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock"
@@ -53,14 +53,14 @@ func startDevServer() {
 	})
 	fmt.Println("Bee running at: ", beeUrl)
 	logger := logging.New(os.Stdout, logrus.DebugLevel)
-	mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger)
+	mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, 0, logger)
 	ens := mock2.NewMockNamespaceManager()
 
 	users := user.NewUsers(mockClient, ens, -1, 0, logger)
 	dfsApi := dfs.NewMockDfsAPI(mockClient, users, logger)
 	handler = api.NewMockHandler(dfsApi, logger, []string{"http://localhost:3000"})
 	defer handler.Close()
-	httpPort = ":9090"
+	httpPort = ":9093"
 	pprofPort = ":9091"
 	srv := startHttpService(logger)
 	fmt.Printf("Server running at:http://127.0.0.1%s\n", httpPort)

cmd/dfs/cmd/root.go

@@ -162,6 +162,7 @@ func writeConfig() {
 	c.Set(optionVerbosity, defaultVerbosity)
 	c.Set(optionBeeApi, defaultBeeApi)
 	c.Set(optionBeePostageBatchId, "")
+	c.Set(optionBeeRedundancyLevel, 0)
 	c.Set(optionCookieDomain, defaultCookieDomain)
 
 	if err := c.WriteConfigAs(cfgFile); err != nil {

cmd/dfs/cmd/server.go

@@ -44,14 +44,15 @@ import (
 )
 
 var (
-	pprof          bool
-	swag           bool
-	httpPort       string
-	pprofPort      string
-	cookieDomain   string
-	postageBlockId string
-	corsOrigins    []string
-	handler        *api.Handler
+	pprof           bool
+	swag            bool
+	httpPort        string
+	pprofPort       string
+	cookieDomain    string
+	postageBlockId  string
+	redundancyLevel uint8
+	corsOrigins     []string
+	handler         *api.Handler
 
 	//go:embed .well-known
 	staticFiles embed.FS
@@ -101,6 +102,9 @@ can consume it.`,
 		if err := config.BindPFlag(optionRPC, cmd.Flags().Lookup("rpc")); err != nil {
 			return err
 		}
+		if err := config.BindPFlag(optionBeeRedundancyLevel, cmd.Flags().Lookup("redundancyLevel")); err != nil {
+			fmt.Println("setting redundancy level to 0")
+		}
 		return config.BindPFlag(optionBeePostageBatchId, cmd.Flags().Lookup("postageBlockId"))
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -113,9 +117,15 @@ can consume it.`,
 		pprofPort = config.GetString(optionDFSPprofPort)
 		cookieDomain = config.GetString(optionCookieDomain)
 		postageBlockId = config.GetString(optionBeePostageBatchId)
+		redundancyLevel = uint8(config.GetUint(optionBeeRedundancyLevel))
 		corsOrigins = config.GetStringSlice(optionCORSAllowedOrigins)
 		verbosity = config.GetString(optionVerbosity)
 
+		if redundancyLevel > 4 {
+			fmt.Println("\nredundancyLevel should be between 0 and 4")
+			return fmt.Errorf("redundancyLevel should be between 0 and 4")
+		}
+
 		if postageBlockId == "" {
 			_ = cmd.Help()
 			fmt.Println("\npostageBlockId is required to run server")
@@ -184,8 +194,7 @@ can consume it.`,
 		logger.Info("verbosity      : ", verbosity)
 		logger.Info("httpPort       : ", httpPort)
 		logger.Info("pprofPort      : ", pprofPort)
-		logger.Info("pprofPort      : ", pprofPort)
-		logger.Info("pprofPort      : ", pprofPort)
+		logger.Info("redundancyLevel: ", redundancyLevel)
 		logger.Info("cookieDomain   : ", cookieDomain)
 		logger.Info("feedCacheSize  : ", config.GetInt(optionFeedCacheSize))
 		logger.Info("feedCacheTTL   : ", config.GetString(optionFeedCacheTTL))
@@ -203,6 +212,7 @@ can consume it.`,
 			Logger:             logger,
 			FeedCacheSize:      config.GetInt(optionFeedCacheSize),
 			FeedCacheTTL:       config.GetString(optionFeedCacheTTL),
+			RedundancyLevel:    redundancyLevel,
 		}
 
 		hdlr, err := api.New(ctx, opts)
@@ -246,6 +256,7 @@ func init() {
 	serverCmd.Flags().String("feedCacheTTL", "0s", "How long to keep feed updates in lru cache. 0s to disable")
 	serverCmd.Flags().String("cookieDomain", defaultCookieDomain, "the domain to use in the cookie")
 	serverCmd.Flags().String("postageBlockId", "", "the postage block used to store the data in bee")
+	serverCmd.Flags().Uint8("redundancyLevel", 0, "redundancy level for swarm erasure coding")
 	serverCmd.Flags().StringSlice("cors-origins", defaultCORSAllowedOrigins, "allow CORS headers for the given origins")
 	serverCmd.Flags().String("ens-network", "testnet", "network to use for authentication [not related to swarm network] (mainnet/testnet/play)")
 	serverCmd.Flags().String("rpc", "", "rpc endpoint for ens network. xDai for mainnet | Sepolia for testnet | local fdp-play rpc endpoint for play")
@@ -436,6 +447,13 @@ func startHttpService(logger logging.Logger) *http.Server {
 	docRouter.HandleFunc("/entry/get", handler.DocEntryGetHandler).Methods("GET")
 	docRouter.HandleFunc("/entry/del", handler.DocEntryDelHandler).Methods("DELETE")
 
+	gitRouter := baseRouter.PathPrefix("/git/").Subrouter()
+	gitRouter.Use(handler.GitAuthMiddleware)
+	gitRouter.HandleFunc("/{user}/{repo}.git/HEAD", handler.GitInfoRef).Methods("GET")
+	gitRouter.HandleFunc("/{user}/{repo}.git/info/refs", handler.GitInfoRef).Methods("GET")
+	gitRouter.HandleFunc("/{user}/{repo}.git/git-upload-pack", handler.GitUploadPack).Methods("POST")
+	gitRouter.HandleFunc("/{user}/{repo}.git/git-receive-pack", handler.GitReceivePack).Methods("POST")
+
 	var origins []string
 	for _, c := range corsOrigins {
 		c = strings.TrimSpace(c)

cmd/dfs/cmd/server_test.go

@@ -20,8 +20,8 @@ import (
 	"github.com/fairdatasociety/fairOS-dfs/pkg/acl/acl"
 	"github.com/stretchr/testify/assert"
 
-	mockpost "github.com/ethersphere/bee/pkg/postage/mock"
-	mockstorer "github.com/ethersphere/bee/pkg/storer/mock"
+	mockpost "github.com/ethersphere/bee/v2/pkg/postage/mock"
+	mockstorer "github.com/ethersphere/bee/v2/pkg/storer/mock"
 	"github.com/fairdatasociety/fairOS-dfs/cmd/common"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/api"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee"
@@ -57,7 +57,7 @@ func TestApis(t *testing.T) {
 	})
 
 	logger := logging.New(io.Discard, logrus.DebugLevel)
-	mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger)
+	mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, 0, logger)
 	ens := mock2.NewMockNamespaceManager()
 
 	users := user.NewUsers(mockClient, ens, 500, 0, logger)