From b175c3299b5564f9c3a01f7e81c1e3f2dc23a110 Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Sat, 29 Jul 2023 17:14:45 +0300 Subject: [PATCH 01/12] feat(rpc): lets keep blob separately --- cmd/celestia/blob.go | 233 +++++++++++++++++++++++++++++++++++++++++++ cmd/celestia/rpc.go | 117 ++-------------------- nodebuilder/store.go | 34 +++++-- 3 files changed, 264 insertions(+), 120 deletions(-) create mode 100644 cmd/celestia/blob.go diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go new file mode 100644 index 0000000000..c1db841f75 --- /dev/null +++ b/cmd/celestia/blob.go @@ -0,0 +1,233 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + + "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + + "github.com/celestiaorg/celestia-node/api/rpc/client" + "github.com/celestiaorg/celestia-node/blob" + options "github.com/celestiaorg/celestia-node/cmd" + "github.com/celestiaorg/celestia-node/nodebuilder" + blob_api "github.com/celestiaorg/celestia-node/nodebuilder/blob" + "github.com/celestiaorg/celestia-node/share" +) + +type response struct { + Result interface{} `json:"result"` + Error string `json:"error,omitempty"` +} + +var blobCmd = &cobra.Command{ + Use: "blob [method] [params...]", + Short: "Allow to interact with the Blob Service via JSON-RPC", + Args: cobra.MinimumNArgs(2), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + blobApi := client.Modules["blob"] + methods := reflect.VisibleFields(reflect.TypeOf(blobApi).Elem()) + var methodNames []string + for _, m := range methods { + methodNames = append(methodNames, m.Name+"\t"+parseSignatureForHelpstring(m)) + } + return methodNames, cobra.ShellCompDirectiveNoFileComp + }, + Run: func(cmd *cobra.Command, args []string) { + client, err := getRPCClient(cmd.Context()) + if err != nil { + panic(err) + } + defer client.Close() + + var requestFn func([]string, *blob_api.API) (interface{}, error) + + // switch over method names + switch args[0] { + default: + panic("invalid method requested") + case "Submit": + requestFn = handleSubmit + case "Get": + requestFn = handleGet + case "GetAll": + requestFn = handleGetAll + case "GetProof": + requestFn = handleGetProof + } + + output := prepareOutput(requestFn(args[1:], &client.Blob)) + fmt.Println(string(output)) + }, +} + +func handleGet(params []string, client *blob_api.API) (interface{}, error) { + // 1. Height + num, err := strconv.ParseUint(params[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing height: uint64 could not be parsed") + } + + // 2. Namespace + namespace, err := parseV0Namespace(params[1]) + if err != nil { + return nil, fmt.Errorf("error parsing namespace: %v", err) + } + + // 3: Commitment + commitment, err := base64.StdEncoding.DecodeString(params[2]) + if err != nil { + return nil, errors.New("error decoding commitment: base64 string could not be decoded") + } + + blob, err := client.Get(context.Background(), num, namespace, commitment) + if err != nil { + return nil, err + } + + if data, err := tryDecode(blob.Data); err == nil { + blob.Data = data + } + return blob, nil +} + +func handleGetAll(params []string, client *blob_api.API) (interface{}, error) { + // 1. Height + num, err := strconv.ParseUint(params[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing height: uint64 could not be parsed") + } + + // 2. Namespace + namespace, err := parseV0Namespace(params[1]) + if err != nil { + return nil, fmt.Errorf("error parsing namespace: %v", err) + } + + blobs, err := client.GetAll(context.Background(), num, []share.Namespace{namespace}) + if err != nil { + return nil, fmt.Errorf("error getting blobs: %s", err) + } + + for _, b := range blobs { + if data, err := tryDecode(b.Data); err == nil { + b.Data = data + } + } + return blobs, err +} + +func handleSubmit(params []string, client *blob_api.API) (interface{}, error) { + // 1. Namespace + var err error + namespace, err := parseV0Namespace(params[0]) + if err != nil { + panic(fmt.Sprintf("Error parsing namespace: %v", err)) + } + + // 2. Blob data + var blobData []byte + switch { + case strings.HasPrefix(params[1], "0x"): + decoded, err := hex.DecodeString(params[1][2:]) + if err != nil { + return nil, errors.New("error decoding blob: hex string could not be decoded") + } + blobData = decoded + case strings.HasPrefix(params[1], "\""): + // user input an utf string that needs to be encoded to base64 + src := []byte(params[1]) + blobData = make([]byte, base64.StdEncoding.EncodedLen(len(src))) + base64.StdEncoding.Encode(blobData, []byte(params[1])) + default: + // otherwise, we assume the user has already encoded their input to base64 + blobData, err = base64.StdEncoding.DecodeString(params[1]) + if err != nil { + return nil, errors.New("error decoding blob data: base64 string could not be decoded") + } + } + + parsedBlob, err := blob.NewBlobV0(namespace, blobData) + if err != nil { + return nil, fmt.Errorf("error creating blob: %v", err) + } + return client.Submit(context.Background(), []*blob.Blob{parsedBlob}) +} + +func handleGetProof(params []string, client *blob_api.API) (interface{}, error) { + // 1. Height + num, err := strconv.ParseUint(params[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing height: uint64 could not be parsed") + } + + // 2. NamespaceID + namespace, err := parseV0Namespace(params[1]) + if err != nil { + return nil, fmt.Errorf("error parsing namespace: %v", err) + } + + // 3: Commitment + commitment, err := base64.StdEncoding.DecodeString(params[2]) + if err != nil { + return nil, errors.New("error decoding commitment: base64 string could not be decoded") + } + return client.GetProof(context.Background(), num, namespace, commitment) +} + +func prepareOutput(data interface{}, err error) []byte { + errStr := "" + if err != nil { + errStr = err.Error() + } + + bytes, err := json.MarshalIndent(response{ + Result: data, + Error: errStr, + }, "", " ") + if err != nil { + panic(err) + } + return bytes +} + +// Golang marshals `[]byte` as a base64-encoded string, so without additional encoding data will +// match expectations. We need this functionality in order to get the original data. +func tryDecode(data []byte) ([]byte, error) { + return base64.StdEncoding.DecodeString(string(data)) +} + +func getRPCClient(ctx context.Context) (*client.Client, error) { + expanded, err := homedir.Expand(filepath.Clean(options.StorePath(ctx))) + if err != nil { + return nil, err + } + + store, err := nodebuilder.OpenStoreReadOnly(expanded) + if err != nil { + return nil, err + } + defer store.Close() + + cfg, err := store.Config() + if err != nil { + return nil, err + } + addr := cfg.RPC.Address + port := cfg.RPC.Port + listenAddr := "http://" + addr + ":" + port + + if authTokenFlag == "" { + authTokenFlag = os.Getenv(authEnvKey) + } + return client.NewClient(ctx, listenAddr, authTokenFlag) +} diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 64b21aaf86..848b9718fc 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -17,13 +17,17 @@ import ( "github.com/spf13/cobra" "github.com/celestiaorg/celestia-node/api/rpc/client" - "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/state" ) const ( authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" + addrEnvKey = "CELESTIA_NODE_RPC_ADDRESS" + portEnvKey = "CELESTIA_NODE_RPC_PORT" + + defaultAddress = "127.0.0.1" + defaultPort = "26658" ) var requestURL string @@ -61,6 +65,7 @@ func init() { false, "Print JSON-RPC request along with the response", ) + rpcCmd.AddCommand(blobCmd) rootCmd.AddCommand(rpcCmd) } @@ -115,116 +120,6 @@ func parseParams(method string, params []string) []interface{} { panic(fmt.Sprintf("Error parsing namespace: %v", err)) } parsedParams[1] = namespace - case "Submit": - // 1. Namespace - var err error - namespace, err := parseV0Namespace(params[0]) - if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) - } - // 2. Blob data - var blobData []byte - switch { - case strings.HasPrefix(params[1], "0x"): - decoded, err := hex.DecodeString(params[1][2:]) - if err != nil { - panic("Error decoding blob: hex string could not be decoded.") - } - blobData = decoded - case strings.HasPrefix(params[1], "\""): - // user input an utf string that needs to be encoded to base64 - src := []byte(params[1]) - blobData = make([]byte, base64.StdEncoding.EncodedLen(len(src))) - base64.StdEncoding.Encode(blobData, []byte(params[1])) - default: - // otherwise, we assume the user has already encoded their input to base64 - blobData, err = base64.StdEncoding.DecodeString(params[1]) - if err != nil { - panic("Error decoding blob data: base64 string could not be decoded.") - } - } - parsedBlob, err := blob.NewBlobV0(namespace, blobData) - if err != nil { - panic(fmt.Sprintf("Error creating blob: %v", err)) - } - parsedParams[0] = []*blob.Blob{parsedBlob} - // param count doesn't match input length, so cut off nil values - return parsedParams[:1] - case "SubmitPayForBlob": - // 1. Fee (state.Int is a string) - parsedParams[0] = params[0] - // 2. GasLimit (uint64) - num, err := strconv.ParseUint(params[1], 10, 64) - if err != nil { - panic("Error parsing gas limit: uint64 could not be parsed.") - } - parsedParams[1] = num - // 3. Namespace - namespace, err := parseV0Namespace(params[2]) - if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) - } - // 4. Blob data - var blobData []byte - switch { - case strings.HasPrefix(params[3], "0x"): - decoded, err := hex.DecodeString(params[3][2:]) - if err != nil { - panic("Error decoding blob: hex string could not be decoded.") - } - blobData = decoded - case strings.HasPrefix(params[3], "\""): - // user input an utf string that needs to be encoded to base64 - src := []byte(params[1]) - blobData = make([]byte, base64.StdEncoding.EncodedLen(len(src))) - base64.StdEncoding.Encode(blobData, []byte(params[3])) - default: - // otherwise, we assume the user has already encoded their input to base64 - blobData, err = base64.StdEncoding.DecodeString(params[3]) - if err != nil { - panic("Error decoding blob: base64 string could not be decoded.") - } - } - parsedBlob, err := blob.NewBlobV0(namespace, blobData) - if err != nil { - panic(fmt.Sprintf("Error creating blob: %v", err)) - } - parsedParams[2] = []*blob.Blob{parsedBlob} - return parsedParams[:3] - case "Get": - // 1. Height - num, err := strconv.ParseUint(params[0], 10, 64) - if err != nil { - panic("Error parsing height: uint64 could not be parsed.") - } - parsedParams[0] = num - // 2. NamespaceID - namespace, err := parseV0Namespace(params[1]) - if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) - } - parsedParams[1] = namespace - // 3: Commitment - commitment, err := base64.StdEncoding.DecodeString(params[2]) - if err != nil { - panic("Error decoding commitment: base64 string could not be decoded.") - } - parsedParams[2] = commitment - return parsedParams - case "GetAll": // NOTE: Over the cli, you can only pass one namespace - // 1. Height - num, err := strconv.ParseUint(params[0], 10, 64) - if err != nil { - panic("Error parsing height: uint64 could not be parsed.") - } - parsedParams[0] = num - // 2. Namespace - namespace, err := parseV0Namespace(params[1]) - if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) - } - parsedParams[1] = []share.Namespace{namespace} - return parsedParams case "QueryDelegation", "QueryUnbonding", "BalanceForAddress": var err error parsedParams[0], err = parseAddressFromString(params[0]) diff --git a/nodebuilder/store.go b/nodebuilder/store.go index f0957a16b6..88e0801dd0 100644 --- a/nodebuilder/store.go +++ b/nodebuilder/store.go @@ -26,6 +26,16 @@ var ( // Store encapsulates storage for the Node. Basically, it is the Store of all Stores. // It provides access for the Node data stored in root directory e.g. '~/.celestia'. type Store interface { + ReadCloser + // PutConfig alters the stored Node config. + PutConfig(*Config) error + + // Close closes the Store freeing up acquired resources and locks. + Close() error +} + +// ReadCloser provides a read-only access to the store +type ReadCloser interface { // Path reports the FileSystem path of Store. Path() string @@ -38,9 +48,6 @@ type Store interface { // Config loads the stored Node config. Config() (*Config, error) - // PutConfig alters the stored Node config. - PutConfig(*Config) error - // Close closes the Store freeing up acquired resources and locks. Close() error } @@ -50,6 +57,21 @@ type Store interface { // OpenStore takes a file Lock on directory, hence only one Store can be opened at a time under the // given 'path', otherwise ErrOpened is thrown. func OpenStore(path string, ring keyring.Keyring) (Store, error) { + s, err := OpenStoreReadOnly(path) + if err != nil { + return nil, err + } + store := s.(*fsStore) + store.keys, err = keystore.NewFSKeystore(keysPath(path), ring) + if err != nil { + return nil, err + } + + return store, nil +} + +// OpenStoreReadOnly opens the store without an access to modify it. +func OpenStoreReadOnly(path string) (ReadCloser, error) { path, err := storePath(path) if err != nil { return nil, err @@ -69,15 +91,9 @@ func OpenStore(path string, ring keyring.Keyring) (Store, error) { return nil, ErrNotInited } - ks, err := keystore.NewFSKeystore(keysPath(path), ring) - if err != nil { - return nil, err - } - return &fsStore{ path: path, dirLock: flock, - keys: ks, }, nil } From 1d7b2669e2044782d2814c52e6a9b1185d70f56a Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Mon, 31 Jul 2023 12:24:21 +0300 Subject: [PATCH 02/12] refactoring --- cmd/celestia/blob.go | 306 ++++++++++++++++++++++--------------------- cmd/celestia/rpc.go | 1 + nodebuilder/store.go | 34 ++--- 3 files changed, 169 insertions(+), 172 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index c1db841f75..9cb6819ed0 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -8,18 +8,13 @@ import ( "errors" "fmt" "os" - "path/filepath" - "reflect" "strconv" "strings" - "github.com/mitchellh/go-homedir" "github.com/spf13/cobra" "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/blob" - options "github.com/celestiaorg/celestia-node/cmd" - "github.com/celestiaorg/celestia-node/nodebuilder" blob_api "github.com/celestiaorg/celestia-node/nodebuilder/blob" "github.com/celestiaorg/celestia-node/share" ) @@ -29,159 +24,182 @@ type response struct { Error string `json:"error,omitempty"` } +var rpcClient blob_api.API + var blobCmd = &cobra.Command{ - Use: "blob [method] [params...]", + Use: "blob [command]", Short: "Allow to interact with the Blob Service via JSON-RPC", - Args: cobra.MinimumNArgs(2), - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - blobApi := client.Modules["blob"] - methods := reflect.VisibleFields(reflect.TypeOf(blobApi).Elem()) - var methodNames []string - for _, m := range methods { - methodNames = append(methodNames, m.Name+"\t"+parseSignatureForHelpstring(m)) + Args: cobra.NoArgs, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + client, err := getRPCClient(cmd.Context()) + if err != nil { + return err } - return methodNames, cobra.ShellCompDirectiveNoFileComp + + rpcClient = client.Blob + return nil }, +} + +var getCmd = &cobra.Command{ + Use: "Get [params]", + Args: cobra.ExactArgs(3), + Short: "Returns blob for the given namespace by commitment at a particular height.", Run: func(cmd *cobra.Command, args []string) { - client, err := getRPCClient(cmd.Context()) + num, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - panic(err) + fmt.Fprintln(os.Stderr, + fmt.Errorf("error parsing height: uint64 could not be parsed"), + ) + os.Exit(1) } - defer client.Close() - var requestFn func([]string, *blob_api.API) (interface{}, error) + // 2. Namespace + namespace, err := parseV0Namespace(args[1]) + if err != nil { + fmt.Fprintln(os.Stderr, + fmt.Errorf("error parsing namespace: %v", err), + ) + os.Exit(1) + } - // switch over method names - switch args[0] { - default: - panic("invalid method requested") - case "Submit": - requestFn = handleSubmit - case "Get": - requestFn = handleGet - case "GetAll": - requestFn = handleGetAll - case "GetProof": - requestFn = handleGetProof + // 3: Commitment + commitment, err := base64.StdEncoding.DecodeString(args[2]) + if err != nil { + fmt.Fprintln(os.Stderr, + errors.New("error decoding commitment: base64 string could not be decoded"), + ) + os.Exit(1) + } + + blob, err := rpcClient.Get(cmd.Context(), num, namespace, commitment) + if err == nil { + if data, decodeErr := tryDecode(blob.Data); decodeErr == nil { + blob.Data = data + } } - output := prepareOutput(requestFn(args[1:], &client.Blob)) - fmt.Println(string(output)) + output := prepareOutput(blob, err) + fmt.Fprintln(os.Stdout, string(output)) }, } -func handleGet(params []string, client *blob_api.API) (interface{}, error) { - // 1. Height - num, err := strconv.ParseUint(params[0], 10, 64) - if err != nil { - return nil, fmt.Errorf("error parsing height: uint64 could not be parsed") - } - - // 2. Namespace - namespace, err := parseV0Namespace(params[1]) - if err != nil { - return nil, fmt.Errorf("error parsing namespace: %v", err) - } +var getAllCmd = &cobra.Command{ + Use: "GetAll [params]", + Args: cobra.ExactArgs(2), + Short: "Returns all blobs for the given namespace at a particular height.", + Run: func(cmd *cobra.Command, args []string) { + num, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + fmt.Fprintln(os.Stderr, + fmt.Errorf("error parsing height: uint64 could not be parsed"), + ) + os.Exit(1) + } - // 3: Commitment - commitment, err := base64.StdEncoding.DecodeString(params[2]) - if err != nil { - return nil, errors.New("error decoding commitment: base64 string could not be decoded") - } + // 2. Namespace + namespace, err := parseV0Namespace(args[1]) + if err != nil { + fmt.Fprintln(os.Stderr, + fmt.Errorf("error parsing namespace: %v", err), + ) + os.Exit(1) + } - blob, err := client.Get(context.Background(), num, namespace, commitment) - if err != nil { - return nil, err - } + blobs, err := rpcClient.GetAll(cmd.Context(), num, []share.Namespace{namespace}) + if err == nil { + for _, b := range blobs { + if data, err := tryDecode(b.Data); err == nil { + b.Data = data + } + } + } - if data, err := tryDecode(blob.Data); err == nil { - blob.Data = data - } - return blob, nil + output := prepareOutput(blobs, err) + fmt.Fprintln(os.Stdout, string(output)) + }, } -func handleGetAll(params []string, client *blob_api.API) (interface{}, error) { - // 1. Height - num, err := strconv.ParseUint(params[0], 10, 64) - if err != nil { - return nil, fmt.Errorf("error parsing height: uint64 could not be parsed") - } - - // 2. Namespace - namespace, err := parseV0Namespace(params[1]) - if err != nil { - return nil, fmt.Errorf("error parsing namespace: %v", err) - } +var submitCmd = &cobra.Command{ + Use: "Submit [params]", + Args: cobra.ExactArgs(2), + Short: "Submit a blob at the given namespace. Note: only one blob is allowed to submit through the RPC.", + Run: func(cmd *cobra.Command, args []string) { + // 1. Namespace + namespace, err := parseV0Namespace(args[0]) + if err != nil { + fmt.Fprintln(os.Stderr, + fmt.Errorf("error parsing namespace: %v", err), + ) + os.Exit(1) + } - blobs, err := client.GetAll(context.Background(), num, []share.Namespace{namespace}) - if err != nil { - return nil, fmt.Errorf("error getting blobs: %s", err) - } + // 2. Blob data + var blobData []byte + switch { + case strings.HasPrefix(args[1], "0x"): + decoded, err := hex.DecodeString(args[1][2:]) + if err != nil { + fmt.Fprintln(os.Stderr, errors.New("error decoding blob: hex string could not be decoded")) + os.Exit(1) + } + blobData = decoded + case strings.HasPrefix(args[1], "\""): + // user input an utf string that needs to be encoded to base64 + src := []byte(args[1]) + blobData = make([]byte, base64.StdEncoding.EncodedLen(len(src))) + base64.StdEncoding.Encode(blobData, []byte(args[1])) + default: + // otherwise, we assume the user has already encoded their input to base64 + blobData, err = base64.StdEncoding.DecodeString(args[1]) + if err != nil { + fmt.Fprintln(os.Stderr, errors.New("error decoding blob data: base64 string could not be decoded")) + os.Exit(1) + } + } - for _, b := range blobs { - if data, err := tryDecode(b.Data); err == nil { - b.Data = data + parsedBlob, err := blob.NewBlobV0(namespace, blobData) + if err != nil { + fmt.Fprintln(os.Stderr, fmt.Errorf("error creating blob: %v", err)) + os.Exit(1) } - } - return blobs, err -} -func handleSubmit(params []string, client *blob_api.API) (interface{}, error) { - // 1. Namespace - var err error - namespace, err := parseV0Namespace(params[0]) - if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) - } + height, err := rpcClient.Submit(cmd.Context(), []*blob.Blob{parsedBlob}) + output := prepareOutput(height, err) + + fmt.Fprintln(os.Stdout, string(output)) + }, +} - // 2. Blob data - var blobData []byte - switch { - case strings.HasPrefix(params[1], "0x"): - decoded, err := hex.DecodeString(params[1][2:]) +var getProofCmd = &cobra.Command{ + Use: "GetProof [params]", + Args: cobra.ExactArgs(3), + Short: "Retrieves a blob in the given namespaces at the given height by commitment and returns its Proof.", + Run: func(cmd *cobra.Command, args []string) { + // 1. Height + num, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - return nil, errors.New("error decoding blob: hex string could not be decoded") + fmt.Fprintln(os.Stderr, fmt.Errorf("error parsing height: uint64 could not be parsed")) + os.Exit(1) } - blobData = decoded - case strings.HasPrefix(params[1], "\""): - // user input an utf string that needs to be encoded to base64 - src := []byte(params[1]) - blobData = make([]byte, base64.StdEncoding.EncodedLen(len(src))) - base64.StdEncoding.Encode(blobData, []byte(params[1])) - default: - // otherwise, we assume the user has already encoded their input to base64 - blobData, err = base64.StdEncoding.DecodeString(params[1]) + + // 2. NamespaceID + namespace, err := parseV0Namespace(args[1]) if err != nil { - return nil, errors.New("error decoding blob data: base64 string could not be decoded") + fmt.Fprintln(os.Stderr, fmt.Errorf("error parsing namespace: %v", err)) + os.Exit(1) } - } - - parsedBlob, err := blob.NewBlobV0(namespace, blobData) - if err != nil { - return nil, fmt.Errorf("error creating blob: %v", err) - } - return client.Submit(context.Background(), []*blob.Blob{parsedBlob}) -} -func handleGetProof(params []string, client *blob_api.API) (interface{}, error) { - // 1. Height - num, err := strconv.ParseUint(params[0], 10, 64) - if err != nil { - return nil, fmt.Errorf("error parsing height: uint64 could not be parsed") - } - - // 2. NamespaceID - namespace, err := parseV0Namespace(params[1]) - if err != nil { - return nil, fmt.Errorf("error parsing namespace: %v", err) - } + // 3: Commitment + commitment, err := base64.StdEncoding.DecodeString(args[2]) + if err != nil { + fmt.Fprintln(os.Stderr, errors.New("error decoding commitment: base64 string could not be decoded")) + } - // 3: Commitment - commitment, err := base64.StdEncoding.DecodeString(params[2]) - if err != nil { - return nil, errors.New("error decoding commitment: base64 string could not be decoded") - } - return client.GetProof(context.Background(), num, namespace, commitment) + proof, err := rpcClient.GetProof(cmd.Context(), num, namespace, commitment) + output := prepareOutput(proof, err) + fmt.Fprintln(os.Stdout, string(output)) + }, } func prepareOutput(data interface{}, err error) []byte { @@ -195,7 +213,8 @@ func prepareOutput(data interface{}, err error) []byte { Error: errStr, }, "", " ") if err != nil { - panic(err) + fmt.Fprintln(os.Stderr, err) + os.Exit(1) } return bytes } @@ -207,27 +226,20 @@ func tryDecode(data []byte) ([]byte, error) { } func getRPCClient(ctx context.Context) (*client.Client, error) { - expanded, err := homedir.Expand(filepath.Clean(options.StorePath(ctx))) - if err != nil { - return nil, err - } + key := os.Getenv(authEnvKey) + addr := os.Getenv(addrEnvKey) + port := os.Getenv(portEnvKey) - store, err := nodebuilder.OpenStoreReadOnly(expanded) - if err != nil { - return nil, err + if authTokenFlag == "" { + authTokenFlag = key } - defer store.Close() - - cfg, err := store.Config() - if err != nil { - return nil, err + if addr == "" { + addr = defaultAddress } - addr := cfg.RPC.Address - port := cfg.RPC.Port - listenAddr := "http://" + addr + ":" + port - - if authTokenFlag == "" { - authTokenFlag = os.Getenv(authEnvKey) + if port == "" { + port = defaultPort } + + listenAddr := "http://" + addr + ":" + port return client.NewClient(ctx, listenAddr, authTokenFlag) } diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 848b9718fc..3ca75907a9 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -65,6 +65,7 @@ func init() { false, "Print JSON-RPC request along with the response", ) + blobCmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) rpcCmd.AddCommand(blobCmd) rootCmd.AddCommand(rpcCmd) } diff --git a/nodebuilder/store.go b/nodebuilder/store.go index 88e0801dd0..f0957a16b6 100644 --- a/nodebuilder/store.go +++ b/nodebuilder/store.go @@ -26,16 +26,6 @@ var ( // Store encapsulates storage for the Node. Basically, it is the Store of all Stores. // It provides access for the Node data stored in root directory e.g. '~/.celestia'. type Store interface { - ReadCloser - // PutConfig alters the stored Node config. - PutConfig(*Config) error - - // Close closes the Store freeing up acquired resources and locks. - Close() error -} - -// ReadCloser provides a read-only access to the store -type ReadCloser interface { // Path reports the FileSystem path of Store. Path() string @@ -48,6 +38,9 @@ type ReadCloser interface { // Config loads the stored Node config. Config() (*Config, error) + // PutConfig alters the stored Node config. + PutConfig(*Config) error + // Close closes the Store freeing up acquired resources and locks. Close() error } @@ -57,21 +50,6 @@ type ReadCloser interface { // OpenStore takes a file Lock on directory, hence only one Store can be opened at a time under the // given 'path', otherwise ErrOpened is thrown. func OpenStore(path string, ring keyring.Keyring) (Store, error) { - s, err := OpenStoreReadOnly(path) - if err != nil { - return nil, err - } - store := s.(*fsStore) - store.keys, err = keystore.NewFSKeystore(keysPath(path), ring) - if err != nil { - return nil, err - } - - return store, nil -} - -// OpenStoreReadOnly opens the store without an access to modify it. -func OpenStoreReadOnly(path string) (ReadCloser, error) { path, err := storePath(path) if err != nil { return nil, err @@ -91,9 +69,15 @@ func OpenStoreReadOnly(path string) (ReadCloser, error) { return nil, ErrNotInited } + ks, err := keystore.NewFSKeystore(keysPath(path), ring) + if err != nil { + return nil, err + } + return &fsStore{ path: path, dirLock: flock, + keys: ks, }, nil } From e60ed4e4341bca407a44473e7085435b19206161 Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Mon, 31 Jul 2023 12:49:08 +0300 Subject: [PATCH 03/12] fix comment --- cmd/celestia/blob.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 9cb6819ed0..4a149cf4eb 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -220,7 +220,7 @@ func prepareOutput(data interface{}, err error) []byte { } // Golang marshals `[]byte` as a base64-encoded string, so without additional encoding data will -// match expectations. We need this functionality in order to get the original data. +// not match the expectations. We need this functionality in order to get the original data. func tryDecode(data []byte) ([]byte, error) { return base64.StdEncoding.DecodeString(string(data)) } From 3779d760e7240ab453dfe53d50b452166c2b4102 Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Mon, 31 Jul 2023 12:59:56 +0300 Subject: [PATCH 04/12] remove env vars --- cmd/celestia/blob.go | 15 ++------------- cmd/celestia/rpc.go | 5 ----- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 4a149cf4eb..a0c46b4a7b 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -226,20 +226,9 @@ func tryDecode(data []byte) ([]byte, error) { } func getRPCClient(ctx context.Context) (*client.Client, error) { - key := os.Getenv(authEnvKey) - addr := os.Getenv(addrEnvKey) - port := os.Getenv(portEnvKey) - if authTokenFlag == "" { - authTokenFlag = key - } - if addr == "" { - addr = defaultAddress - } - if port == "" { - port = defaultPort + authTokenFlag = os.Getenv(authEnvKey) } - listenAddr := "http://" + addr + ":" + port - return client.NewClient(ctx, listenAddr, authTokenFlag) + return client.NewClient(ctx, requestURL, authTokenFlag) } diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 3ca75907a9..26bffd22c6 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -23,11 +23,6 @@ import ( const ( authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" - addrEnvKey = "CELESTIA_NODE_RPC_ADDRESS" - portEnvKey = "CELESTIA_NODE_RPC_PORT" - - defaultAddress = "127.0.0.1" - defaultPort = "26658" ) var requestURL string From 700fde121a8003a563c15a592958b9cf28cf003c Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Mon, 31 Jul 2023 14:20:10 +0300 Subject: [PATCH 05/12] gracefully close the connection --- cmd/celestia/blob.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index a0c46b4a7b..f18c4d4e84 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -15,7 +15,6 @@ import ( "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/blob" - blob_api "github.com/celestiaorg/celestia-node/nodebuilder/blob" "github.com/celestiaorg/celestia-node/share" ) @@ -24,11 +23,11 @@ type response struct { Error string `json:"error,omitempty"` } -var rpcClient blob_api.API +var rpcClient *client.Client var blobCmd = &cobra.Command{ Use: "blob [command]", - Short: "Allow to interact with the Blob Service via JSON-RPC", + Short: "Allows to interact with the Blob Service via JSON-RPC", Args: cobra.NoArgs, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { client, err := getRPCClient(cmd.Context()) @@ -36,15 +35,18 @@ var blobCmd = &cobra.Command{ return err } - rpcClient = client.Blob + rpcClient = client return nil }, + PersistentPostRun: func(_ *cobra.Command, _ []string) { + rpcClient.Close() + }, } var getCmd = &cobra.Command{ Use: "Get [params]", Args: cobra.ExactArgs(3), - Short: "Returns blob for the given namespace by commitment at a particular height.", + Short: "Returns the blob for the given namespace by commitment at a particular height.", Run: func(cmd *cobra.Command, args []string) { num, err := strconv.ParseUint(args[0], 10, 64) if err != nil { @@ -72,7 +74,7 @@ var getCmd = &cobra.Command{ os.Exit(1) } - blob, err := rpcClient.Get(cmd.Context(), num, namespace, commitment) + blob, err := rpcClient.Blob.Get(cmd.Context(), num, namespace, commitment) if err == nil { if data, decodeErr := tryDecode(blob.Data); decodeErr == nil { blob.Data = data @@ -106,7 +108,7 @@ var getAllCmd = &cobra.Command{ os.Exit(1) } - blobs, err := rpcClient.GetAll(cmd.Context(), num, []share.Namespace{namespace}) + blobs, err := rpcClient.Blob.GetAll(cmd.Context(), num, []share.Namespace{namespace}) if err == nil { for _, b := range blobs { if data, err := tryDecode(b.Data); err == nil { @@ -123,7 +125,7 @@ var getAllCmd = &cobra.Command{ var submitCmd = &cobra.Command{ Use: "Submit [params]", Args: cobra.ExactArgs(2), - Short: "Submit a blob at the given namespace. Note: only one blob is allowed to submit through the RPC.", + Short: "Submit the blob at the given namespace. Note: only one blob is allowed to submit through the RPC.", Run: func(cmd *cobra.Command, args []string) { // 1. Namespace namespace, err := parseV0Namespace(args[0]) @@ -164,7 +166,7 @@ var submitCmd = &cobra.Command{ os.Exit(1) } - height, err := rpcClient.Submit(cmd.Context(), []*blob.Blob{parsedBlob}) + height, err := rpcClient.Blob.Submit(cmd.Context(), []*blob.Blob{parsedBlob}) output := prepareOutput(height, err) fmt.Fprintln(os.Stdout, string(output)) @@ -174,7 +176,7 @@ var submitCmd = &cobra.Command{ var getProofCmd = &cobra.Command{ Use: "GetProof [params]", Args: cobra.ExactArgs(3), - Short: "Retrieves a blob in the given namespaces at the given height by commitment and returns its Proof.", + Short: "Retrieves the blob in the given namespaces at the given height by commitment and returns its Proof.", Run: func(cmd *cobra.Command, args []string) { // 1. Height num, err := strconv.ParseUint(args[0], 10, 64) @@ -196,7 +198,7 @@ var getProofCmd = &cobra.Command{ fmt.Fprintln(os.Stderr, errors.New("error decoding commitment: base64 string could not be decoded")) } - proof, err := rpcClient.GetProof(cmd.Context(), num, namespace, commitment) + proof, err := rpcClient.Blob.GetProof(cmd.Context(), num, namespace, commitment) output := prepareOutput(proof, err) fmt.Fprintln(os.Stdout, string(output)) }, From 145ea982918a8f712c9d4ed317b88f807288154b Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Tue, 1 Aug 2023 16:13:33 +0300 Subject: [PATCH 06/12] refactoring --- cmd/celestia/blob.go | 226 ++++++++++++++++++------------------------- cmd/celestia/rpc.go | 39 ++++++++ 2 files changed, 134 insertions(+), 131 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index f18c4d4e84..ad95ef773c 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -1,236 +1,200 @@ package main import ( - "context" "encoding/base64" - "encoding/hex" "encoding/json" - "errors" "fmt" "os" "strconv" - "strings" "github.com/spf13/cobra" - "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/share" ) type response struct { Result interface{} `json:"result"` - Error string `json:"error,omitempty"` } -var rpcClient *client.Client - var blobCmd = &cobra.Command{ Use: "blob [command]", Short: "Allows to interact with the Blob Service via JSON-RPC", Args: cobra.NoArgs, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - client, err := getRPCClient(cmd.Context()) - if err != nil { - return err - } - - rpcClient = client - return nil - }, - PersistentPostRun: func(_ *cobra.Command, _ []string) { - rpcClient.Close() - }, } var getCmd = &cobra.Command{ - Use: "Get [params]", + Use: "get [params]", Args: cobra.ExactArgs(3), Short: "Returns the blob for the given namespace by commitment at a particular height.", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + num, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - fmt.Fprintln(os.Stderr, - fmt.Errorf("error parsing height: uint64 could not be parsed"), - ) - os.Exit(1) + return fmt.Errorf("error parsing a height:%v", err) } - // 2. Namespace namespace, err := parseV0Namespace(args[1]) if err != nil { - fmt.Fprintln(os.Stderr, - fmt.Errorf("error parsing namespace: %v", err), - ) - os.Exit(1) + return fmt.Errorf("error parsing a namespace:%v", err) } - // 3: Commitment commitment, err := base64.StdEncoding.DecodeString(args[2]) if err != nil { - fmt.Fprintln(os.Stderr, - errors.New("error decoding commitment: base64 string could not be decoded"), - ) - os.Exit(1) + return fmt.Errorf("error parsing a commitment:%v", err) } - blob, err := rpcClient.Blob.Get(cmd.Context(), num, namespace, commitment) - if err == nil { - if data, decodeErr := tryDecode(blob.Data); decodeErr == nil { - blob.Data = data + var output []byte + blob, err := client.Blob.Get(cmd.Context(), num, namespace, commitment) + if err != nil { + output, err = prepareOutput(err) + if err != nil { + return err + } + } else { + output, err = prepareOutput(blob) + if err != nil { + return err } } - output := prepareOutput(blob, err) fmt.Fprintln(os.Stdout, string(output)) + return nil }, } var getAllCmd = &cobra.Command{ - Use: "GetAll [params]", + Use: "getAll [params]", Args: cobra.ExactArgs(2), Short: "Returns all blobs for the given namespace at a particular height.", - Run: func(cmd *cobra.Command, args []string) { - num, err := strconv.ParseUint(args[0], 10, 64) + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + height, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - fmt.Fprintln(os.Stderr, - fmt.Errorf("error parsing height: uint64 could not be parsed"), - ) - os.Exit(1) + return fmt.Errorf("error parsing a height:%v", err) } - // 2. Namespace namespace, err := parseV0Namespace(args[1]) if err != nil { - fmt.Fprintln(os.Stderr, - fmt.Errorf("error parsing namespace: %v", err), - ) - os.Exit(1) + return fmt.Errorf("error parsing a namespace:%v", err) } - blobs, err := rpcClient.Blob.GetAll(cmd.Context(), num, []share.Namespace{namespace}) - if err == nil { - for _, b := range blobs { - if data, err := tryDecode(b.Data); err == nil { - b.Data = data - } + var output []byte + blobs, err := client.Blob.GetAll(cmd.Context(), height, []share.Namespace{namespace}) + if err != nil { + output, err = prepareOutput(err) + if err != nil { + return err + } + } else { + output, err = prepareOutput(blobs) + if err != nil { + return err } } - output := prepareOutput(blobs, err) fmt.Fprintln(os.Stdout, string(output)) + return nil }, } var submitCmd = &cobra.Command{ - Use: "Submit [params]", + Use: "submit [params]", Args: cobra.ExactArgs(2), Short: "Submit the blob at the given namespace. Note: only one blob is allowed to submit through the RPC.", - Run: func(cmd *cobra.Command, args []string) { - // 1. Namespace + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + namespace, err := parseV0Namespace(args[0]) if err != nil { - fmt.Fprintln(os.Stderr, - fmt.Errorf("error parsing namespace: %v", err), - ) - os.Exit(1) + return fmt.Errorf("error parsing a namespace:%v", err) } - // 2. Blob data - var blobData []byte - switch { - case strings.HasPrefix(args[1], "0x"): - decoded, err := hex.DecodeString(args[1][2:]) + parsedBlob, err := blob.NewBlobV0(namespace, []byte(args[1])) + if err != nil { + return fmt.Errorf("error creating a blob:%v", err) + } + + var output []byte + height, err := client.Blob.Submit(cmd.Context(), []*blob.Blob{parsedBlob}) + if err != nil { + output, err = prepareOutput(err) if err != nil { - fmt.Fprintln(os.Stderr, errors.New("error decoding blob: hex string could not be decoded")) - os.Exit(1) + return err } - blobData = decoded - case strings.HasPrefix(args[1], "\""): - // user input an utf string that needs to be encoded to base64 - src := []byte(args[1]) - blobData = make([]byte, base64.StdEncoding.EncodedLen(len(src))) - base64.StdEncoding.Encode(blobData, []byte(args[1])) - default: - // otherwise, we assume the user has already encoded their input to base64 - blobData, err = base64.StdEncoding.DecodeString(args[1]) + } else { + output, err = prepareOutput(height) if err != nil { - fmt.Fprintln(os.Stderr, errors.New("error decoding blob data: base64 string could not be decoded")) - os.Exit(1) + return err } } - parsedBlob, err := blob.NewBlobV0(namespace, blobData) - if err != nil { - fmt.Fprintln(os.Stderr, fmt.Errorf("error creating blob: %v", err)) - os.Exit(1) - } - - height, err := rpcClient.Blob.Submit(cmd.Context(), []*blob.Blob{parsedBlob}) - output := prepareOutput(height, err) - fmt.Fprintln(os.Stdout, string(output)) + return nil }, } var getProofCmd = &cobra.Command{ - Use: "GetProof [params]", + Use: "getProof [params]", Args: cobra.ExactArgs(3), Short: "Retrieves the blob in the given namespaces at the given height by commitment and returns its Proof.", - Run: func(cmd *cobra.Command, args []string) { - // 1. Height - num, err := strconv.ParseUint(args[0], 10, 64) + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + height, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - fmt.Fprintln(os.Stderr, fmt.Errorf("error parsing height: uint64 could not be parsed")) - os.Exit(1) + return fmt.Errorf("error parsing a height:%v", err) } - // 2. NamespaceID namespace, err := parseV0Namespace(args[1]) if err != nil { - fmt.Fprintln(os.Stderr, fmt.Errorf("error parsing namespace: %v", err)) - os.Exit(1) + return fmt.Errorf("error parsing a namespace:%v", err) } - // 3: Commitment commitment, err := base64.StdEncoding.DecodeString(args[2]) if err != nil { - fmt.Fprintln(os.Stderr, errors.New("error decoding commitment: base64 string could not be decoded")) + return fmt.Errorf("error parsing a commitment:%v", err) + } + + var output []byte + proof, err := client.Blob.GetProof(cmd.Context(), height, namespace, commitment) + if err != nil { + output, err = prepareOutput(err) + if err != nil { + return err + } + } else { + output, err = prepareOutput(proof) + if err != nil { + return err + } } - proof, err := rpcClient.Blob.GetProof(cmd.Context(), num, namespace, commitment) - output := prepareOutput(proof, err) fmt.Fprintln(os.Stdout, string(output)) + return nil }, } -func prepareOutput(data interface{}, err error) []byte { - errStr := "" - if err != nil { - errStr = err.Error() - } - +func prepareOutput(data interface{}) ([]byte, error) { bytes, err := json.MarshalIndent(response{ Result: data, - Error: errStr, }, "", " ") if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + return nil, err } - return bytes -} - -// Golang marshals `[]byte` as a base64-encoded string, so without additional encoding data will -// not match the expectations. We need this functionality in order to get the original data. -func tryDecode(data []byte) ([]byte, error) { - return base64.StdEncoding.DecodeString(string(data)) -} - -func getRPCClient(ctx context.Context) (*client.Client, error) { - if authTokenFlag == "" { - authTokenFlag = os.Getenv(authEnvKey) - } - - return client.NewClient(ctx, requestURL, authTokenFlag) + return bytes, nil } diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 26bffd22c6..a1bdf9efaa 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -2,9 +2,11 @@ package main import ( "bytes" + "context" "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "log" @@ -23,6 +25,8 @@ import ( const ( authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" + + rpcClientKey = "rpcClient" ) var requestURL string @@ -69,6 +73,25 @@ var rpcCmd = &cobra.Command{ Use: "rpc [namespace] [method] [params...]", Short: "Send JSON-RPC request", Args: cobra.MinimumNArgs(2), + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + client, err := newRPCClient(cmd.Context()) + if err != nil { + return err + } + + ctx := context.WithValue(cmd.Context(), rpcClientKey, client) + cmd.SetContext(ctx) + return nil + }, + PersistentPostRunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + client.Close() + return nil + }, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { modules := client.Modules if len(args) == 0 { @@ -336,8 +359,24 @@ func decodeToBytes(param string) ([]byte, error) { } return decoded, nil } + func parseJSON(param string) (json.RawMessage, error) { var raw json.RawMessage err := json.Unmarshal([]byte(param), &raw) return raw, err } + +func newRPCClient(ctx context.Context) (*client.Client, error) { + if authTokenFlag == "" { + authTokenFlag = os.Getenv(authEnvKey) + } + return client.NewClient(ctx, requestURL, authTokenFlag) +} + +func rpcClient(ctx context.Context) (*client.Client, error) { + client, ok := ctx.Value(rpcClientKey).(*client.Client) + if !ok { + return nil, errors.New("rpc client was not set") + } + return client, nil +} From 84ede8e18c85847fbe73d7af6cbbeabd1dbb8de8 Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Thu, 3 Aug 2023 13:57:21 +0300 Subject: [PATCH 07/12] fixes --- cmd/celestia/blob.go | 25 ++++++++++++++++++------- cmd/celestia/rpc.go | 1 - 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index ad95ef773c..358dc63128 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -17,6 +17,10 @@ type response struct { Result interface{} `json:"result"` } +func init() { + blobCmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) +} + var blobCmd = &cobra.Command{ Use: "blob [command]", Short: "Allows to interact with the Blob Service via JSON-RPC", @@ -24,7 +28,7 @@ var blobCmd = &cobra.Command{ } var getCmd = &cobra.Command{ - Use: "get [params]", + Use: "get [height, namespace, commitment]", Args: cobra.ExactArgs(3), Short: "Returns the blob for the given namespace by commitment at a particular height.", RunE: func(cmd *cobra.Command, args []string) error { @@ -33,7 +37,7 @@ var getCmd = &cobra.Command{ return err } - num, err := strconv.ParseUint(args[0], 10, 64) + height, err := strconv.ParseUint(args[0], 10, 64) if err != nil { return fmt.Errorf("error parsing a height:%v", err) } @@ -49,7 +53,7 @@ var getCmd = &cobra.Command{ } var output []byte - blob, err := client.Blob.Get(cmd.Context(), num, namespace, commitment) + blob, err := client.Blob.Get(cmd.Context(), height, namespace, commitment) if err != nil { output, err = prepareOutput(err) if err != nil { @@ -68,7 +72,7 @@ var getCmd = &cobra.Command{ } var getAllCmd = &cobra.Command{ - Use: "getAll [params]", + Use: "get-all [height, namespace]", Args: cobra.ExactArgs(2), Short: "Returns all blobs for the given namespace at a particular height.", RunE: func(cmd *cobra.Command, args []string) error { @@ -107,7 +111,7 @@ var getAllCmd = &cobra.Command{ } var submitCmd = &cobra.Command{ - Use: "submit [params]", + Use: "submit [namespace, blobData]", Args: cobra.ExactArgs(2), Short: "Submit the blob at the given namespace. Note: only one blob is allowed to submit through the RPC.", RunE: func(cmd *cobra.Command, args []string) error { @@ -134,7 +138,14 @@ var submitCmd = &cobra.Command{ return err } } else { - output, err = prepareOutput(height) + response := struct { + Height uint64 `json:"uint64"` + Commitment blob.Commitment `json:"commitment"` + }{ + Height: height, + Commitment: parsedBlob.Commitment, + } + output, err = prepareOutput(response) if err != nil { return err } @@ -146,7 +157,7 @@ var submitCmd = &cobra.Command{ } var getProofCmd = &cobra.Command{ - Use: "getProof [params]", + Use: "get-proof [height, namespace, commitment]", Args: cobra.ExactArgs(3), Short: "Retrieves the blob in the given namespaces at the given height by commitment and returns its Proof.", RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index a1bdf9efaa..d493bbf5e4 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -64,7 +64,6 @@ func init() { false, "Print JSON-RPC request along with the response", ) - blobCmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) rpcCmd.AddCommand(blobCmd) rootCmd.AddCommand(rpcCmd) } From cbc178b56a5693235c4ec21a2fa37e981f36e74a Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Thu, 3 Aug 2023 14:10:32 +0300 Subject: [PATCH 08/12] rework prepare output --- cmd/celestia/blob.go | 72 +++++++++++++++----------------------------- 1 file changed, 25 insertions(+), 47 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 358dc63128..944a82b782 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -52,18 +52,11 @@ var getCmd = &cobra.Command{ return fmt.Errorf("error parsing a commitment:%v", err) } - var output []byte blob, err := client.Blob.Get(cmd.Context(), height, namespace, commitment) + + output, err := prepareOutput(blob, err) if err != nil { - output, err = prepareOutput(err) - if err != nil { - return err - } - } else { - output, err = prepareOutput(blob) - if err != nil { - return err - } + return err } fmt.Fprintln(os.Stdout, string(output)) @@ -93,16 +86,10 @@ var getAllCmd = &cobra.Command{ var output []byte blobs, err := client.Blob.GetAll(cmd.Context(), height, []share.Namespace{namespace}) + + output, err = prepareOutput(blobs, err) if err != nil { - output, err = prepareOutput(err) - if err != nil { - return err - } - } else { - output, err = prepareOutput(blobs) - if err != nil { - return err - } + return err } fmt.Fprintln(os.Stdout, string(output)) @@ -130,25 +117,19 @@ var submitCmd = &cobra.Command{ return fmt.Errorf("error creating a blob:%v", err) } - var output []byte height, err := client.Blob.Submit(cmd.Context(), []*blob.Blob{parsedBlob}) + + response := struct { + Height uint64 `json:"uint64"` + Commitment blob.Commitment `json:"commitment"` + }{ + Height: height, + Commitment: parsedBlob.Commitment, + } + + output, err := prepareOutput(response, err) if err != nil { - output, err = prepareOutput(err) - if err != nil { - return err - } - } else { - response := struct { - Height uint64 `json:"uint64"` - Commitment blob.Commitment `json:"commitment"` - }{ - Height: height, - Commitment: parsedBlob.Commitment, - } - output, err = prepareOutput(response) - if err != nil { - return err - } + return err } fmt.Fprintln(os.Stdout, string(output)) @@ -181,18 +162,11 @@ var getProofCmd = &cobra.Command{ return fmt.Errorf("error parsing a commitment:%v", err) } - var output []byte proof, err := client.Blob.GetProof(cmd.Context(), height, namespace, commitment) + + output, err := prepareOutput(proof, err) if err != nil { - output, err = prepareOutput(err) - if err != nil { - return err - } - } else { - output, err = prepareOutput(proof) - if err != nil { - return err - } + return err } fmt.Fprintln(os.Stdout, string(output)) @@ -200,7 +174,11 @@ var getProofCmd = &cobra.Command{ }, } -func prepareOutput(data interface{}) ([]byte, error) { +func prepareOutput(data interface{}, err error) ([]byte, error) { + if err != nil { + data = err + } + bytes, err := json.MarshalIndent(response{ Result: data, }, "", " ") From b69055fd0ea15ba8fe4dfc937924966a71abcfa4 Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Thu, 3 Aug 2023 16:53:55 +0300 Subject: [PATCH 09/12] add printOutput --- cmd/celestia/blob.go | 48 +++++++++++++------------------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 944a82b782..c600e1acca 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -13,10 +13,6 @@ import ( "github.com/celestiaorg/celestia-node/share" ) -type response struct { - Result interface{} `json:"result"` -} - func init() { blobCmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) } @@ -54,12 +50,7 @@ var getCmd = &cobra.Command{ blob, err := client.Blob.Get(cmd.Context(), height, namespace, commitment) - output, err := prepareOutput(blob, err) - if err != nil { - return err - } - - fmt.Fprintln(os.Stdout, string(output)) + printOutput(blob, err) return nil }, } @@ -84,15 +75,9 @@ var getAllCmd = &cobra.Command{ return fmt.Errorf("error parsing a namespace:%v", err) } - var output []byte blobs, err := client.Blob.GetAll(cmd.Context(), height, []share.Namespace{namespace}) - output, err = prepareOutput(blobs, err) - if err != nil { - return err - } - - fmt.Fprintln(os.Stdout, string(output)) + printOutput(blobs, err) return nil }, } @@ -127,12 +112,7 @@ var submitCmd = &cobra.Command{ Commitment: parsedBlob.Commitment, } - output, err := prepareOutput(response, err) - if err != nil { - return err - } - - fmt.Fprintln(os.Stdout, string(output)) + printOutput(response, err) return nil }, } @@ -164,26 +144,26 @@ var getProofCmd = &cobra.Command{ proof, err := client.Blob.GetProof(cmd.Context(), height, namespace, commitment) - output, err := prepareOutput(proof, err) - if err != nil { - return err - } - - fmt.Fprintln(os.Stdout, string(output)) + printOutput(proof, err) return nil }, } -func prepareOutput(data interface{}, err error) ([]byte, error) { +func printOutput(data interface{}, err error) { if err != nil { data = err } - bytes, err := json.MarshalIndent(response{ + resp := struct { + Result interface{} `json:"result"` + }{ Result: data, - }, "", " ") + } + + bytes, err := json.MarshalIndent(resp, "", " ") if err != nil { - return nil, err + fmt.Fprintln(os.Stderr, err) + os.Exit(1) } - return bytes, nil + fmt.Fprintln(os.Stdout, string(bytes)) } From dfede7eed6eb7733e6ea7c54c1451cab2dbedc01 Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Thu, 3 Aug 2023 17:13:45 +0300 Subject: [PATCH 10/12] feat: add flag that allows to print blobs data as a plain text --- cmd/celestia/blob.go | 60 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index c600e1acca..8a8dc392e5 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "reflect" "strconv" "github.com/spf13/cobra" @@ -13,8 +14,23 @@ import ( "github.com/celestiaorg/celestia-node/share" ) +var fomatedData bool + func init() { blobCmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) + + getCmd.PersistentFlags().BoolVar( + &fomatedData, + "data.format", + false, + "printed blob's data as a plain text", + ) + getAllCmd.PersistentFlags().BoolVar( + &fomatedData, + "data.format", + false, + "printed blob's data as a plain text", + ) } var blobCmd = &cobra.Command{ @@ -154,6 +170,10 @@ func printOutput(data interface{}, err error) { data = err } + if fomatedData && err == nil { + data = formatData(data) + } + resp := struct { Result interface{} `json:"result"` }{ @@ -167,3 +187,43 @@ func printOutput(data interface{}, err error) { } fmt.Fprintln(os.Stdout, string(bytes)) } + +func formatData(data interface{}) interface{} { + type tempBlob struct { + Namespace []byte `json:"namespace"` + Data string `json:"data"` + ShareVersion uint32 `json:"share_version"` + Commitment []byte `json:"commitment"` + } + + if reflect.TypeOf(data).Kind() == reflect.Slice { + blobs, ok := data.([]*blob.Blob) + if !ok { + fmt.Fprintln(os.Stderr, "could not cast to []blob.Blob") + os.Exit(1) + } + + result := make([]tempBlob, len(blobs)) + for i, b := range blobs { + result[i] = tempBlob{ + Namespace: b.Namespace(), + Data: string(b.Data), + ShareVersion: b.ShareVersion, + Commitment: b.Commitment, + } + } + return result + } + + b, ok := data.(*blob.Blob) + if !ok { + fmt.Fprintln(os.Stderr, "could not cast to blob.Blob") + os.Exit(1) + } + return tempBlob{ + Namespace: b.Namespace(), + Data: string(b.Data), + ShareVersion: b.ShareVersion, + Commitment: b.Commitment, + } +} From ec0849ebff98f64c6d8a7f7d0c060eb07d3d6bcd Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Thu, 3 Aug 2023 17:27:06 +0300 Subject: [PATCH 11/12] chore: rename flag --- cmd/celestia/blob.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 8a8dc392e5..9dcd72dcb7 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -14,20 +14,20 @@ import ( "github.com/celestiaorg/celestia-node/share" ) -var fomatedData bool +var plaintext bool func init() { blobCmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) getCmd.PersistentFlags().BoolVar( - &fomatedData, - "data.format", + &plaintext, + "plaintext", false, "printed blob's data as a plain text", ) getAllCmd.PersistentFlags().BoolVar( - &fomatedData, - "data.format", + &plaintext, + "plaintext", false, "printed blob's data as a plain text", ) @@ -170,7 +170,7 @@ func printOutput(data interface{}, err error) { data = err } - if fomatedData && err == nil { + if plaintext && err == nil { data = formatData(data) } From 12b261def8b9513594843cdb7093ea86549d9dbb Mon Sep 17 00:00:00 2001 From: Viacheslav Gonkivskyi Date: Fri, 4 Aug 2023 14:22:55 +0300 Subject: [PATCH 12/12] fix lint --- cmd/celestia/rpc.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index d493bbf5e4..05439ad7b4 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -23,11 +23,7 @@ import ( "github.com/celestiaorg/celestia-node/state" ) -const ( - authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" - - rpcClientKey = "rpcClient" -) +const authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" var requestURL string var authTokenFlag string @@ -73,12 +69,12 @@ var rpcCmd = &cobra.Command{ Short: "Send JSON-RPC request", Args: cobra.MinimumNArgs(2), PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - client, err := newRPCClient(cmd.Context()) + rpcClient, err := newRPCClient(cmd.Context()) if err != nil { return err } - ctx := context.WithValue(cmd.Context(), rpcClientKey, client) + ctx := context.WithValue(cmd.Context(), rpcClientKey{}, rpcClient) cmd.SetContext(ctx) return nil }, @@ -372,8 +368,10 @@ func newRPCClient(ctx context.Context) (*client.Client, error) { return client.NewClient(ctx, requestURL, authTokenFlag) } +type rpcClientKey struct{} + func rpcClient(ctx context.Context) (*client.Client, error) { - client, ok := ctx.Value(rpcClientKey).(*client.Client) + client, ok := ctx.Value(rpcClientKey{}).(*client.Client) if !ok { return nil, errors.New("rpc client was not set") }