Implement psclient interface (#107)

* Implement psclient interface

* Add string method to pieceID type

* try to fix linter errors

* Whoops missed an error

* More linter errors

* Typo

* Lol double typo

*  Get everything working, begin adding tests for psclient rpc

* goimports

* Forgot to change the piecestore cli when changed the piecestore code

* Fix CLI

* remove ID length, added validator to pieceID

* Move grpc ranger to client
Change client PUT api to take a reader rather than return a writer

* GRPCRanger -> PieceRanger; Make PieceRanger a RangeCloser

* Forgot to remove offset

* Added message upon successful store

* Do that thing dennis and kaloyan wanted

* goimports

* Make closeConn a part of the interface for psclient

* Use interface

* Removed uneccessary new lines

* goimport

* Whoops

* Actually we don't want to use the interface in Piece Ranger

* Renamed piecestore in examples to piecestore-client; moved piecestore-cli to examples

* Make comments look nicer

cmd/piecestore-cli/main.go → examples/piecestore-cli/main.go

@@ -34,37 +34,45 @@ func run(ctx context.Context, _ *cobra.Command, _ []string) error {
 		{
 			Name:      "store",
 			Aliases:   []string{"s"},
-			Usage:     "Store data by hash",
-			ArgsUsage: "[hash] [dataPath] [storeDir]",
+			Usage:     "Store data by id",
+			ArgsUsage: "[id] [dataPath] [storeDir]",
 			Action: func(c *cli.Context) error {
 				if c.Args().Get(0) == "" {
-					return argError.New("No input file specified")
+					return argError.New("No id specified")
 				}
 
+				id := c.Args().Get(0)
+
 				if c.Args().Get(1) == "" {
+					return argError.New("No input file specified")
+				}
+
+				path := c.Args().Get(1)
+
+				if c.Args().Get(2) == "" {
 					return argError.New("No output directory specified")
 				}
 
-				file, err := os.Open(c.Args().Get(0))
+				outputDir := c.Args().Get(2)
+
+				file, err := os.Open(path)
 				if err != nil {
 					return err
 				}
 
 				// Close the file when we are done
 				defer file.Close()
 
-				fileInfo, err := os.Stat(c.Args().Get(0))
+				fileInfo, err := os.Stat(path)
 				if err != nil {
 					return err
 				}
 
 				if fileInfo.IsDir() {
-					return argError.New(fmt.Sprintf("Path (%s) is a directory, not a file", c.Args().Get(0)))
+					return argError.New(fmt.Sprintf("Path (%s) is a directory, not a file", path))
 				}
 
-				id := pstore.DetermineID()
-
-				dataFileChunk, err := pstore.StoreWriter(id, c.Args().Get(1))
+				dataFileChunk, err := pstore.StoreWriter(id, outputDir)
 				if err != nil {
 					return err
 				}
@@ -80,11 +88,11 @@ func run(ctx context.Context, _ *cobra.Command, _ []string) error {
 		{
 			Name:      "retrieve",
 			Aliases:   []string{"r"},
-			Usage:     "Retrieve data by hash and print to Stdout",
-			ArgsUsage: "[hash] [storeDir]",
+			Usage:     "Retrieve data by id and print to Stdout",
+			ArgsUsage: "[id] [storeDir]",
 			Action: func(c *cli.Context) error {
 				if c.Args().Get(0) == "" {
-					return argError.New("Missing data Hash")
+					return argError.New("Missing data id")
 				}
 				if c.Args().Get(1) == "" {
 					return argError.New("Missing file path")
@@ -114,11 +122,11 @@ func run(ctx context.Context, _ *cobra.Command, _ []string) error {
 		{
 			Name:      "delete",
 			Aliases:   []string{"d"},
-			Usage:     "Delete data by hash",
-			ArgsUsage: "[hash] [storeDir]",
+			Usage:     "Delete data by id",
+			ArgsUsage: "[id] [storeDir]",
 			Action: func(c *cli.Context) error {
 				if c.Args().Get(0) == "" {
-					return argError.New("Missing data Hash")
+					return argError.New("Missing data id")
 				}
 				if c.Args().Get(1) == "" {
 					return argError.New("No directory specified")

examples/piecestore/rpc/client/main.go → examples/piecestore-client/rpc/client/main.go

@@ -16,7 +16,6 @@ import (
 	"github.com/zeebo/errs"
 	"google.golang.org/grpc"
 
-	"storj.io/storj/pkg/piecestore"
 	"storj.io/storj/pkg/piecestore/rpc/client"
 )
 
@@ -33,14 +32,15 @@ func main() {
 		log.Fatalf("did not connect: %s", err)
 	}
 	defer conn.Close()
-	routeClient := client.New(conn)
+	psClient := client.NewPSClient(conn)
 
 	app.Commands = []cli.Command{
 		{
 			Name:    "upload",
 			Aliases: []string{"u"},
 			Usage:   "upload data",
 			Action: func(c *cli.Context) error {
+
 				if c.Args().Get(0) == "" {
 					return argError.New("No input file specified")
 				}
@@ -62,45 +62,42 @@ func main() {
 				}
 
 				var length = fileInfo.Size()
-				var ttl = time.Now().Unix() + 86400
+				var ttl = time.Now().Add(24 * time.Hour)
 
 				// Created a section reader so that we can concurrently retrieve the same file.
 				dataSection := io.NewSectionReader(file, 0, length)
 
-				id := pstore.DetermineID()
+				id := client.NewPieceID()
 
-				writer, err := routeClient.StorePieceRequest(context.Background(), id, ttl)
-				if err != nil {
-					fmt.Printf("Failed to send meta data to server to store file of id: %s\n", id)
+				if err := psClient.Put(context.Background(), id, dataSection, ttl); err != nil {
+					fmt.Printf("Failed to Store data of id: %s\n", id)
 					return err
 				}
 
-				_, err = io.Copy(writer, dataSection)
-				if err != nil {
-					fmt.Printf("Failed to store file of id: %s\n", id)
-				} else {
-					fmt.Printf("successfully stored file of id: %s\n", id)
-				}
+				fmt.Printf("Successfully stored file of id: %s\n", id)
 
-				return writer.Close()
+				return nil
 			},
 		},
 		{
 			Name:    "download",
 			Aliases: []string{"d"},
 			Usage:   "download data",
 			Action: func(c *cli.Context) error {
-				if c.Args().Get(0) == "" {
+				const (
+					id int = iota
+					outputDir
+				)
+
+				if c.Args().Get(id) == "" {
 					return argError.New("No id specified")
 				}
 
-				id := c.Args().Get(0)
-
-				if c.Args().Get(1) == "" {
+				if c.Args().Get(outputDir) == "" {
 					return argError.New("No output file specified")
 				}
 
-				_, err := os.Stat(c.Args().Get(1))
+				_, err := os.Stat(c.Args().Get(outputDir))
 				if err != nil && !os.IsNotExist(err) {
 					return err
 				}
@@ -109,38 +106,44 @@ func main() {
 					return argError.New("File already exists")
 				}
 
-				dataPath := c.Args().Get(1)
-
-				if err = os.MkdirAll(filepath.Dir(dataPath), 0700); err != nil {
+				if err = os.MkdirAll(filepath.Dir(c.Args().Get(outputDir)), 0700); err != nil {
 					return err
 				}
 
 				// Create File on file system
-				dataFile, err := os.OpenFile(dataPath, os.O_RDWR|os.O_CREATE, 0755)
+				dataFile, err := os.OpenFile(c.Args().Get(outputDir), os.O_RDWR|os.O_CREATE, 0755)
 				if err != nil {
 					return err
 				}
 				defer dataFile.Close()
 
-				pieceInfo, err := routeClient.PieceMetaRequest(context.Background(), id)
+				pieceInfo, err := psClient.Meta(context.Background(), client.PieceID(c.Args().Get(id)))
+				if err != nil {
+					os.Remove(c.Args().Get(outputDir))
+					return err
+				}
+
+				ctx := context.Background()
+				rr, err := psClient.Get(ctx, client.PieceID(c.Args().Get(id)), pieceInfo.Size)
 				if err != nil {
-					os.Remove(dataPath)
+					fmt.Printf("Failed to retrieve file of id: %s\n", c.Args().Get(id))
+					os.Remove(c.Args().Get(outputDir))
 					return err
 				}
 
-				reader, err := routeClient.RetrievePieceRequest(context.Background(), id, 0, pieceInfo.Size)
+				reader, err := rr.Range(ctx, 0, pieceInfo.Size)
 				if err != nil {
-					fmt.Printf("Failed to retrieve file of id: %s\n", id)
-					os.Remove(dataPath)
+					fmt.Printf("Failed to retrieve file of id: %s\n", c.Args().Get(id))
+					os.Remove(c.Args().Get(outputDir))
 					return err
 				}
 
 				_, err = io.Copy(dataFile, reader)
 				if err != nil {
-					fmt.Printf("Failed to retrieve file of id: %s\n", id)
-					os.Remove(dataPath)
+					fmt.Printf("Failed to retrieve file of id: %s\n", c.Args().Get(id))
+					os.Remove(c.Args().Get(outputDir))
 				} else {
-					fmt.Printf("Successfully retrieved file of id: %s\n", id)
+					fmt.Printf("Successfully retrieved file of id: %s\n", c.Args().Get(id))
 				}
 
 				return reader.Close()
@@ -154,7 +157,7 @@ func main() {
 				if c.Args().Get(0) == "" {
 					return argError.New("Missing data Id")
 				}
-				err = routeClient.DeletePieceRequest(context.Background(), c.Args().Get(0))
+				err = psClient.Delete(context.Background(), client.PieceID(c.Args().Get(0)))
 
 				return err
 			},

pkg/piecestore/pstore.go

@@ -5,13 +5,11 @@ package pstore
 
 import (
 	"context"
-	"crypto/rand"
 	"io"
 	"os"
 	"path"
 	"path/filepath"
 
-	"github.com/mr-tron/base58/base58"
 	"github.com/zeebo/errs"
 
 	"storj.io/storj/pkg/ranger"
@@ -42,18 +40,6 @@ func PathByID(id, dir string) (string, error) {
 	return path.Join(dir, folder1, folder2, fileName), nil
 }
 
-// DetermineID creates random id
-func DetermineID() string {
-	b := make([]byte, 32)
-
-	_, err := rand.Read(b)
-	if err != nil {
-		panic(err)
-	}
-
-	return base58.Encode(b)
-}
-
 // StoreWriter stores data into piece store in multiple writes
 // 	id is the id of the data to be stored
 // 	dir is the pstore directory containing all other data stored

pkg/piecestore/pstore_test.go

@@ -264,19 +264,6 @@ func TestDelete(t *testing.T) {
 	}
 }
 
-func TestDetermineID(t *testing.T) {
-	t.Run("should return an id string", func(t *testing.T) {
-		assert := assert.New(t)
-		id := DetermineID()
-		assert.Equal(len(id) >= IDLength, true)
-	})
-
-	t.Run("should return a different string on each call", func(t *testing.T) {
-		assert := assert.New(t)
-		assert.NotEqual(DetermineID(), DetermineID())
-	})
-}
-
 func TestMain(m *testing.M) {
 	m.Run()
 }