diff --git a/README.md b/README.md index 1cfc971..ece5604 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ nmngd node extract-addrbook ~/.node_home_source/config/addrbook.json ~/.node_hom nmngd node prune-addrbook ~/.node_home/config/addrbook.json nmngd node prune-data ~/.node_home --binary xxxd [--backup-pvs ~/priv_validator_state.json.backup] nmngd node state-sync ~/.node_home --binary xxxd --rpc http://localhost:26657 [--address-book /home/x/.node/config/addrbook.json] [--peers nodeid@127.0.0.1:26656] [--seeds seed@1.1.1.1:26656] [--max-duration 12h] +nmngd node dump-snapshot ~/.node_home --binary xxxd [--max-duration 1h] [--no-service] [--service-name xxx] [--external-rpc https://rpc1.example.com:443 --external-rpc https://rpc2.example.com:443] [--fix-genesis] nmngd node zip-snapshot ~/.node_home ``` diff --git a/cmd/node/dump_snapshot/dump_snapshot.go b/cmd/node/dump_snapshot/dump_snapshot.go new file mode 100644 index 0000000..89dc52c --- /dev/null +++ b/cmd/node/dump_snapshot/dump_snapshot.go @@ -0,0 +1,422 @@ +package dump_snapshot + +import ( + "fmt" + "github.com/bcdevtools/node-management/types" + "github.com/bcdevtools/node-management/utils" + "github.com/spf13/cobra" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strings" + "time" +) + +const ( + flagNoService = "no-service" + flagServiceName = "service-name" + flagExternalRpc = "external-rpc" + flagBinary = "binary" + flagMaxDuration = "max-duration" + flagXCrisisSkipAssertInvariants = "x-crisis-skip-assert-invariants" + flagFixGenesis = "fix-genesis" +) + +func GetDumpSnapshotCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "dump-snapshot [node_home]", + Short: "Dump snapshot from node, using Cosmos-SDK snapshot commands", + Long: "Dump snapshot from node, using Cosmos-SDK snapshot commands.\n" + + "The node will be stopped, data will be exported, dumped and restore into another node home directory (eg ~/.gaia → ~/.gaia-dump).", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + utils.MustNotUserRoot() + + var exitWithError bool + defer func() { + if exitWithError { + os.Exit(1) + } + }() + + // Process: + // - Stop service + // - Export snapshot + // - Dump snapshot + // - Restore snapshot + // - Start service + + /** + Coding convention: + due to resource cleanup, the code is written in a way that ensure the cleanup methods are called, + so that the resources are released properly. + Usage of os.Exit() is avoided, and the exitWithError flag is used to indicate the error. + Usage of functions that call os.Exit() is prohibited. + */ + + nodeHomeDirectory := strings.TrimSpace(args[0]) + binary, _ := cmd.Flags().GetString(flagBinary) + maxDuration, _ := cmd.Flags().GetDuration(flagMaxDuration) + noService, _ := cmd.Flags().GetBool(flagNoService) + if !noService && !utils.IsLinux() { + noService = true + fmt.Printf("INF: --%s is forced on Non-Linux\n", flagNoService) + } + + if err := validateNodeHomeDirectory(nodeHomeDirectory); err != nil { + utils.PrintlnStdErr("ERR: invalid node home directory:", err) + exitWithError = true + return + } + nodeHomeDirectory, err := filepath.Abs(nodeHomeDirectory) + if err != nil { + utils.PrintlnStdErr("ERR: failed to get absolute path of node home directory:", err) + exitWithError = true + return + } + + if err := validateBinary(binary); err != nil { + utils.PrintlnStdErr("ERR: invalid binary path:", err) + utils.PrintfStdErr("ERR: correct flag --%s\n", flagBinary) + exitWithError = true + return + } + + serviceName, err := getServiceName(noService, binary, cmd) + if err != nil { + utils.PrintlnStdErr("ERR: failed to get service name") + utils.PrintlnStdErr("ERR:", err.Error()) + exitWithError = true + return + } + + const minOfMaxDuration = 30 * time.Minute + if maxDuration < minOfMaxDuration { + utils.PrintfStdErr("ERR: minimum accepted for --%s is %s\n", flagMaxDuration, minOfMaxDuration) + exitWithError = true + return + } + + var registeredCleanup []func() + execCleanup := func() { + for i, cleanup := range registeredCleanup { + func(i int, cleanup func()) { + defer func() { + r := recover() + if err != nil { + utils.PrintlnStdErr("ERR: panic in cleanup[", i, "]:", r) + } + }() + + cleanup() + }(i, cleanup) + } + } + + defer execCleanup() + + go func() { + time.Sleep(maxDuration) + utils.PrintlnStdErr("ERR: timeout") + execCleanup() + os.Exit(1) + }() + + parentHomeDir, homeDirName := path.Split(nodeHomeDirectory) + + dumpDirName := homeDirName + "-dump" + dumpHomeDir := path.Join(parentHomeDir, dumpDirName) + + if err := prepareDumpNodeHomeDirectory(dumpHomeDir, nodeHomeDirectory); err != nil { + utils.PrintlnStdErr("ERR: failed to prepare dump home directory:", err) + exitWithError = true + return + } + if cmd.Flags().Changed(flagFixGenesis) { + fmt.Println("INF: fixing genesis initial_height") + genesisFilePath := path.Join(dumpHomeDir, "config", "genesis.json") + _ = utils.LaunchApp("/bin/bash", []string{ + "-c", fmt.Sprintf(`jq '.initial_height = "1"' %s > %s.tmp && mv %s.tmp %s`, genesisFilePath, genesisFilePath, genesisFilePath, genesisFilePath), + }) + } + + fmt.Println("INF: force reset dump home directory") + ec := utils.LaunchApp( + binary, []string{ + "tendermint", "unsafe-reset-all", + "--home", dumpHomeDir, + "--keep-addr-book", + }, + ) + if ec != 0 { + utils.PrintlnStdErr("ERR: failed to unsafe-reset-all the dump home directory") + exitWithError = true + return + } + + appOriginalNodeMutex, appDumpNodeMutex, errAcqSi := acquireSingletonInstance(nodeHomeDirectory, dumpHomeDir) + defer func() { + if appOriginalNodeMutex != nil { + appOriginalNodeMutex.ReleaseLockWL() + } + if appDumpNodeMutex != nil { + appDumpNodeMutex.ReleaseLockWL() + } + }() + if errAcqSi != nil { + utils.PrintlnStdErr("ERR: failed to acquire singleton instance:", errAcqSi) + exitWithError = true + return + } + + if !noService { + fmt.Println("INF: stopping service") + ec := utils.LaunchApp("sudo", []string{"systemctl", "stop", serviceName}) + if ec != 0 { + utils.PrintlnStdErr("ERR: failed to stop service") + exitWithError = true + return + } + time.Sleep(15 * time.Second) // wait completely shutdown + } + + fmt.Println("INF: exporting snapshot") + _ = utils.LaunchApp( + binary, []string{ + "snapshots", "export", "--home", nodeHomeDirectory, + }, + ) + + fmt.Println("INF: checking snapshots") + snapshots, err := loadSnapshotList(binary, nodeHomeDirectory) + if err != nil { + utils.PrintlnStdErr("ERR: failed to get list after exported snapshot:", err) + exitWithError = true + return + } else if len(snapshots) == 0 { + utils.PrintlnStdErr("ERR: failed to get list after exported snapshot") + exitWithError = true + return + } + + snapshots.Sort() + mostRecentSnapshot := snapshots[0] + fmt.Println("INF: most recent snapshot:", mostRecentSnapshot.height, ", format", mostRecentSnapshot.format, ", chunks", mostRecentSnapshot.chunks) + + outputFileName := fmt.Sprintf("dump-snapshot.%d-%d.tar.gz", mostRecentSnapshot.height, mostRecentSnapshot.format) + fmt.Println("INF: dumping snapshot") + ec = utils.LaunchApp(binary, []string{ + "snapshots", "dump", mostRecentSnapshot.HeightStr(), mostRecentSnapshot.FormatStr(), + "--home", nodeHomeDirectory, + "--output", outputFileName, + }) + if ec != 0 { + utils.PrintlnStdErr("ERR: failed to dump snapshot") + exitWithError = true + return + } + if err := validateOutputFile(outputFileName); err != nil { + utils.PrintlnStdErr("ERR: failed to validate output file:", err) + exitWithError = true + return + } + + fmt.Println("INF: snapshot dumped successfully:", outputFileName) + + registeredCleanup = append(registeredCleanup, func() { + _ = os.Remove(outputFileName) + }) + + fmt.Println("INF: restoring into", dumpHomeDir) + ec = utils.LaunchApp(binary, []string{ + "snapshots", "load", outputFileName, + "--home", dumpHomeDir, + }) + if ec != 0 { + utils.PrintlnStdErr("ERR: failed to load snapshot") + exitWithError = true + return + } + + snapshots, err = loadSnapshotList(binary, dumpHomeDir) + if err != nil { + utils.PrintlnStdErr("ERR: failed to get list after loaded snapshot:", err) + exitWithError = true + return + } else if len(snapshots) == 0 { + utils.PrintlnStdErr("ERR: failed to get list after loaded snapshot") + exitWithError = true + return + } + + ec = utils.LaunchApp(binary, []string{ + "snapshots", "restore", mostRecentSnapshot.HeightStr(), mostRecentSnapshot.FormatStr(), + "--home", dumpHomeDir, + }) + if ec != 0 { + utils.PrintlnStdErr("ERR: failed to restore snapshot") + exitWithError = true + return + } + + if noService { + // launching node for bootstrapping + startArgs := []string{ + "start", + "--home", nodeHomeDirectory, + } + if cmd.Flags().Changed(flagXCrisisSkipAssertInvariants) { + startArgs = append(startArgs, fmt.Sprintf("--%s", flagXCrisisSkipAssertInvariants)) + } + launchCmd := exec.Command(binary, startArgs...) + launchCmd.Stdout = os.Stdout + launchCmd.Stderr = os.Stderr + err = launchCmd.Start() + if err != nil { + utils.PrintlnStdErr("ERR: failed to start node for bootstrapping:", err) + exitWithError = true + return + } + registeredCleanup = append(registeredCleanup, func() { + _ = launchCmd.Process.Kill() + }) + } else { + fmt.Println("INF: restarting service") + ec = utils.LaunchApp("sudo", []string{"systemctl", "restart", serviceName}) + if ec != 0 { + utils.PrintlnStdErr("ERR: failed to restart service") + exitWithError = true + return + } + time.Sleep(5 * time.Second) + } + + rpc, err := types.ReadNodeRpcFromConfigToml(path.Join(nodeHomeDirectory, "config", "config.toml")) + if err != nil { + utils.PrintlnStdErr("ERR: failed to read node rpc from config.toml:", err) + exitWithError = true + return + } + externalRPCs, _ := cmd.Flags().GetStringSlice(flagExternalRpc) + rpcEps := []string{rpc} + for _, externalRPC := range externalRPCs { + rpc := strings.TrimSpace(externalRPC) + if rpc == "" { + continue + } + rpcEps = append(rpcEps, rpc) + } + + for { + resp, err := http.Get(fmt.Sprintf("%s/status", strings.TrimSuffix(rpc, "/"))) + if err == nil && resp.StatusCode == http.StatusOK { + break + } + fmt.Println("INF: waiting node up") + time.Sleep(10 * time.Second) + } + + chanTrustHash := make(chan string, len(rpcEps)) + registeredCleanup = append(registeredCleanup, func() { + close(chanTrustHash) + }) + + for _, rpc := range rpcEps { + go func(rpc string) { + var trustHash string + defer func() { + chanTrustHash <- trustHash + }() + + output, ec := utils.LaunchAppAndGetOutput( + "/bin/bash", + []string{ + "-c", fmt.Sprintf( + `curl -m 30 -s "%s/block?height=%d" | jq -r .result.block_id.hash`, + rpc, + mostRecentSnapshot.height, + ), + }, + ) + if ec != 0 { + utils.PrintlnStdErr(output) + utils.PrintlnStdErr("ERR: failed to get block hash from rpc:", rpc) + return + } + + trustHash = strings.TrimSpace(output) + if !regexp.MustCompile(`^[A-F\d]{64}$`).MatchString(trustHash) { + utils.PrintlnStdErr("ERR: invalid block hash", trustHash, "from rpc", rpc) + trustHash = "" + } + }(rpc) + } + + var trustHash string + for c := 1; c <= len(rpcEps); c++ { + resTrustHash := <-chanTrustHash + if resTrustHash == "" { + continue + } + if trustHash != "" { + // take the first valid trust hash + continue + } + trustHash = resTrustHash + } + + if trustHash == "" { + utils.PrintlnStdErr("ERR: failed to get trust hash from rpc") + exitWithError = true + return + } + + if len(rpcEps) == 1 { + rpcEps = append(rpcEps, rpcEps[0]) + } + sedArgs := []string{ + "-i.bak", + "-E", `s|^(enable[[:space:]]+=[[:space:]]+).*$|\1true| ; s|^(rpc_servers[[:space:]]+=[[:space:]]+).*$|\1\"` + strings.Join(func() []string { + if len(rpcEps) == 1 { + return []string{rpcEps[0], rpcEps[0]} + } + return rpcEps + }(), ",") + `\"| ; s|^(trust_height[[:space:]]+=[[:space:]]+).*$|\1` + mostRecentSnapshot.HeightStr() + `| ; s|^(trust_hash[[:space:]]+=[[:space:]]+).*$|\1"` + trustHash + `"|`, + path.Join(dumpHomeDir, "config", "config.toml"), + } + ec = utils.LaunchApp("sed", sedArgs) + if ec != 0 { + utils.PrintlnStdErr("ERR: failed to launch sed to update config file") + utils.PrintlnStdErr("> sed " + strings.Join(sedArgs, " ")) + exitWithError = true + return + } + + fmt.Println("INF: bootstrapping snapshot") + ec = utils.LaunchApp(binary, []string{ + "tendermint", "bootstrap-state", + "--home", dumpHomeDir, + }) + if ec != 0 { + utils.PrintlnStdErr("ERR: failed to bootstrap snapshot") + exitWithError = true + return + } + + fmt.Println("INF: successfully dumped snapshot into", dumpHomeDir) + }, + } + + cmd.Flags().String(flagBinary, "", "Path to the binary") + cmd.Flags().Duration(flagMaxDuration, 1*time.Hour, "Maximum duration to wait for dumping snapshot") + cmd.Flags().Bool(flagNoService, false, "Do not stop and start service") + cmd.Flags().String(flagServiceName, "", "Custom service name, used to call start/stop") + cmd.Flags().StringSlice(flagExternalRpc, []string{}, "External RPC address used for bootstrapping node") + cmd.Flags().Bool(flagXCrisisSkipAssertInvariants, false, "Skip assert invariants") + cmd.Flags().Bool(flagFixGenesis, false, "Fix `initial_height` in genesis.json") + + return cmd +} diff --git a/cmd/node/dump_snapshot/flags.go b/cmd/node/dump_snapshot/flags.go new file mode 100644 index 0000000..986758a --- /dev/null +++ b/cmd/node/dump_snapshot/flags.go @@ -0,0 +1,53 @@ +package dump_snapshot + +import ( + "fmt" + "github.com/bcdevtools/node-management/utils" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "path" + "strings" +) + +func getServiceName(noService bool, binary string, cmd *cobra.Command) (serviceName string, err error) { + if noService { + return + } + + defer func() { + if err != nil { + serviceName = "" + } + }() + + customServiceName, _ := cmd.Flags().GetString(flagServiceName) + if customServiceName != "" { + if strings.Contains(customServiceName, "/") { + err = fmt.Errorf("service name cannot contain path, provide name only") + return + } + serviceName = customServiceName + } else { + _, binaryName := path.Split(binary) + if binaryName == "" { + err = fmt.Errorf("failed to get service name from binary path, require flag --%s\n", flagServiceName) + return + } + serviceName = binaryName + } + + serviceName = strings.TrimSuffix(serviceName, ".service") + + expectedServiceFile := path.Join("/etc/systemd/system", serviceName+".service") + _, exists, _, errChkSvcF := utils.FileInfo(expectedServiceFile) + if errChkSvcF != nil { + err = errors.Wrapf(errChkSvcF, "failed to check service file %s", expectedServiceFile) + return + } + if !exists { + err = fmt.Errorf("expected service file does not exists [%s], correct service file name by flag --%s", expectedServiceFile, flagServiceName) + return + } + + return +} diff --git a/cmd/node/dump_snapshot/lock.go b/cmd/node/dump_snapshot/lock.go new file mode 100644 index 0000000..442623e --- /dev/null +++ b/cmd/node/dump_snapshot/lock.go @@ -0,0 +1,30 @@ +package dump_snapshot + +import ( + "fmt" + "github.com/bcdevtools/node-management/types" + "github.com/pkg/errors" + "time" +) + +func acquireSingletonInstance(nodeHomeDirectory, dumpHomeDir string) (appOriginalNodeMutex, appDumpNodeMutex *types.AppMutex, err error) { + appOriginalNodeMutex = types.NewAppMutex(nodeHomeDirectory, 4*time.Second) + if acquiredLock, errAcquire := appOriginalNodeMutex.AcquireLockWL(); errAcquire != nil { + err = errors.Wrap(errAcquire, "failed to acquire lock single instance in original node home") + return + } else if !acquiredLock { + err = fmt.Errorf("failed to acquire lock single instance in original node") + return + } + + appDumpNodeMutex = types.NewAppMutex(dumpHomeDir, 8*time.Second) + if acquiredLock, errAcquire := appDumpNodeMutex.AcquireLockWL(); errAcquire != nil { + err = errors.Wrap(errAcquire, "failed to acquire lock single instance in dump node") + return + } else if !acquiredLock { + err = fmt.Errorf("failed to acquire lock single instance in dump node") + return + } + + return +} diff --git a/cmd/node/dump_snapshot/path.go b/cmd/node/dump_snapshot/path.go new file mode 100644 index 0000000..971e087 --- /dev/null +++ b/cmd/node/dump_snapshot/path.go @@ -0,0 +1,71 @@ +package dump_snapshot + +import ( + "fmt" + "github.com/bcdevtools/node-management/types" + "github.com/bcdevtools/node-management/utils" + "github.com/pkg/errors" + "os" + "path" +) + +func prepareDumpNodeHomeDirectory(dumpHomeDir, nodeHomeDir string) error { + _, exists, _, err := utils.FileInfo(dumpHomeDir) + if err != nil { + return errors.Wrap(err, "failed to check dump home directory at "+dumpHomeDir) + } + + if !exists { + fmt.Println("INF: creating dump home directory") + err = os.Mkdir(dumpHomeDir, 0o755) + if err != nil { + return errors.Wrap(err, "failed to create dump home directory at "+dumpHomeDir) + } + } + + configDirOfDump := path.Join(dumpHomeDir, "config") + err = os.Mkdir(configDirOfDump, 0o755) + if err != nil && !os.IsExist(err) { + return errors.Wrap(err, "failed to create config directory at "+configDirOfDump) + } + + dataDirOfDump := path.Join(dumpHomeDir, "data") + err = os.Mkdir(dataDirOfDump, 0o700) + if err != nil && !os.IsExist(err) { + return errors.Wrap(err, "failed to create data directory at "+dataDirOfDump) + } + + privValStateJsonFilePath := path.Join(dataDirOfDump, "priv_validator_state.json") + err = (&types.PrivateValidatorState{ + Height: "0", + }).SaveToJSONFile(privValStateJsonFilePath) + if err != nil { + return errors.Wrap(err, "failed to create empty "+privValStateJsonFilePath) + } + + copyConfig := func(fileName string) error { + src := path.Join(nodeHomeDir, "config", fileName) + dst := path.Join(dumpHomeDir, "config", fileName) + ec := utils.LaunchApp("cp", []string{src, dst}) + if ec != 0 { + return errors.Wrap(err, "failed to copy config file "+fileName) + } + fmt.Println("INF: copied config file:", fileName) + return nil + } + + fmt.Println("INF: Copying config files") + if err = copyConfig("config.toml"); err != nil { + return err + } + if err = copyConfig("app.toml"); err != nil { + return err + } + if err = copyConfig("genesis.json"); err != nil { + return err + } + if err = copyConfig("client.toml"); err != nil { + return err + } + return nil +} diff --git a/cmd/node/dump_snapshot/snapshot.go b/cmd/node/dump_snapshot/snapshot.go new file mode 100644 index 0000000..883423d --- /dev/null +++ b/cmd/node/dump_snapshot/snapshot.go @@ -0,0 +1,62 @@ +package dump_snapshot + +import ( + "fmt" + "github.com/bcdevtools/node-management/utils" + "sort" + "strings" +) + +type snapshot struct { + height int64 + format uint32 + chunks uint +} +type snapshots []snapshot + +func (ss snapshots) Sort() snapshots { + sort.Slice(ss, func(i, j int) bool { + return ss[i].height > ss[j].height + }) + return ss +} + +func (ss snapshot) HeightStr() string { + return fmt.Sprintf("%d", ss.height) +} + +func (ss snapshot) FormatStr() string { + return fmt.Sprintf("%d", ss.format) +} + +func loadSnapshotList(binary, nodeHomeDirectory string) (snapshots, error) { + output, ec := utils.LaunchAppAndGetOutput(binary, []string{"snapshots", "list", "--home", nodeHomeDirectory}) + if ec != 0 { + return nil, fmt.Errorf("failed to list snapshots") + } + + var snapshots []snapshot + for _, line := range strings.Split(output, "\n") { + if !strings.Contains(line, "height:") { + continue + } + if !strings.Contains(line, "format:") { + continue + } + if !strings.Contains(line, "chunks:") { + continue + } + + var s snapshot + _, err := fmt.Sscanf(strings.TrimSpace(line), "height: %d format: %d chunks: %d", &s.height, &s.format, &s.chunks) + if err != nil { + return nil, fmt.Errorf("failed to parse snapshot line: %s", line) + } + if s.height == 0 || s.format == 0 || s.chunks == 0 { + return nil, fmt.Errorf("invalid snapshot line: %s, value %v", line, s) + } + snapshots = append(snapshots, s) + } + + return snapshots, nil +} diff --git a/cmd/node/dump_snapshot/validate.go b/cmd/node/dump_snapshot/validate.go new file mode 100644 index 0000000..6e00e14 --- /dev/null +++ b/cmd/node/dump_snapshot/validate.go @@ -0,0 +1,68 @@ +package dump_snapshot + +import ( + "fmt" + "github.com/bcdevtools/node-management/utils" + "github.com/bcdevtools/node-management/validation" + "github.com/pkg/errors" + "path" + "strings" +) + +func validateBinary(binary string) error { + if binary == "" { + return fmt.Errorf("binary path is required") + } + + if !strings.Contains(binary, "/") { + if !utils.HasBinaryName(binary) { + return fmt.Errorf("specified binary does not exists or not included in PATH: %s", binary) + } + + return nil + } + + _, exists, isDir, err := utils.FileInfo(binary) + if err != nil { + return errors.Wrapf(err, "failed to check binary path: %s", binary) + } + if !exists { + return fmt.Errorf("specified binary does not exists: %s", binary) + } + if isDir { + return fmt.Errorf("specified binary path is a directory: %s", binary) + } + return nil +} + +func validateNodeHomeDirectory(nodeHomeDirectory string) error { + err := validation.PossibleNodeHome(nodeHomeDirectory) + if err != nil { + return errors.Wrapf(err, "invalid node home directory: %s", nodeHomeDirectory) + } + _, exists, _, err := utils.FileInfo(path.Join(nodeHomeDirectory, "data", "application.db")) + if err != nil { + return errors.Wrapf(err, "failed to check application.db in node home directory: %s", nodeHomeDirectory) + } + if !exists { + return fmt.Errorf("node home directory does not contains data") + } + return nil +} + +func validateOutputFile(outputFile string) error { + if outputFile == "" { + return fmt.Errorf("output file path is required") + } + _, exists, _, err := utils.FileInfo(outputFile) + if err != nil { + return errors.Wrapf(err, "failed to check output file path %s", outputFile) + } + if !exists { + return fmt.Errorf("output file path does not exists: %s", outputFile) + } + if !strings.HasSuffix(outputFile, ".tar.gz") { + return fmt.Errorf("output file must be .tar.gz: %s", outputFile) + } + return nil +} diff --git a/cmd/node/root.go b/cmd/node/root.go index d2df0cf..aee5d0a 100644 --- a/cmd/node/root.go +++ b/cmd/node/root.go @@ -2,6 +2,7 @@ package node //goland:noinspection GoSnakeCaseUsage import ( + "github.com/bcdevtools/node-management/cmd/node/dump_snapshot" setup_check "github.com/bcdevtools/node-management/cmd/node/setup-check" "github.com/bcdevtools/node-management/utils" "github.com/bcdevtools/node-management/validation" @@ -22,6 +23,7 @@ func GetNodeCommands() *cobra.Command { GetStateSyncCmd(), GetZipSnapshotCmd(), GetAutoBackupPrivValidatorStateCmd(), + dump_snapshot.GetDumpSnapshotCmd(), ) return cmd diff --git a/cmd/node/state_sync.go b/cmd/node/state_sync.go index 1a03c17..7c8fb86 100644 --- a/cmd/node/state_sync.go +++ b/cmd/node/state_sync.go @@ -5,7 +5,6 @@ import ( "github.com/bcdevtools/node-management/types" "github.com/bcdevtools/node-management/utils" "github.com/bcdevtools/node-management/validation" - "github.com/pelletier/go-toml/v2" "github.com/spf13/cobra" "os" "os/exec" @@ -189,37 +188,11 @@ func GetStateSyncCmd() *cobra.Command { } configFilePath := path.Join(configDirPath, "config.toml") - _, exists, _, err = utils.FileInfo(configFilePath) + stateSyncNodeRpc, err := types.ReadNodeRpcFromConfigToml(configFilePath) if err != nil { - utils.ExitWithErrorMsg("ERR: failed to check config file:", err) + utils.ExitWithErrorMsg("ERR: failed to read node rpc from config file:", err) return } - if !exists { - utils.ExitWithErrorMsg("ERR: config file does not exist:", configFilePath) - return - } - bz, err := os.ReadFile(configFilePath) - if err != nil { - utils.ExitWithErrorMsg("ERR: failed to read config.toml file:", err) - return - } - var config types.ConfigToml - err = toml.Unmarshal(bz, &config) - if err != nil { - utils.ExitWithErrorMsg("ERR: failed to unmarshal config.toml file:", err) - return - } - if config.RPC == nil || config.RPC.LAddr == "" { - utils.ExitWithErrorMsg("ERR: rpc section, address is not set in config.toml") - return - } - stateSyncNodeRpc := strings.TrimSpace(config.RPC.LAddr) - stateSyncNodeRpc = strings.TrimPrefix(stateSyncNodeRpc, "tcp://") - stateSyncNodeRpc = strings.TrimSuffix(stateSyncNodeRpc, "/") - //goland:noinspection HttpUrlsUsage - if !strings.HasPrefix(stateSyncNodeRpc, "http://") { - stateSyncNodeRpc = "http://" + stateSyncNodeRpc - } var modernSed bool launchSed := func(pattern string) { @@ -286,7 +259,12 @@ func GetStateSyncCmd() *cobra.Command { launchSed(`s|^(enable[[:space:]]+=[[:space:]]+).*$|\1true| ; s|^(rpc_servers[[:space:]]+=[[:space:]]+).*$|\1\"` + rpc + "," + rpc + `\"| ; s|^(trust_height[[:space:]]+=[[:space:]]+).*$|\1` + fmt.Sprintf("%d", blockHeight) + `| ; s|^(trust_hash[[:space:]]+=[[:space:]]+).*$|\1"` + trustHash + `"|`) fmt.Println("INF: trust_height, rpc_servers, trust_hash and enable are updated in config file") - startArgs := []string{"start", "--home", nodeHomeDirectory} + startArgs := []string{ + "start", + "--home", nodeHomeDirectory, + "--api.enable=false", + "--grpc.enable=false", + } if cmd.Flags().Changed(flagXCrisisSkipAssertInvariants) { startArgs = append(startArgs, fmt.Sprintf("--%s", flagXCrisisSkipAssertInvariants)) } @@ -299,6 +277,11 @@ func GetStateSyncCmd() *cobra.Command { return } + const minOfMaxDuration = 30 * time.Minute + if maxDuration < minOfMaxDuration { + utils.ExitWithErrorMsgf("ERR: minimum accepted for --%s is %s\n", flagMaxDuration, minOfMaxDuration) + return + } expiry := time.Now().UTC().Add(maxDuration) ensureStateSyncNotExpired := func() { diff --git a/cmd/node/zip_snapshot.go b/cmd/node/zip_snapshot.go index 9aee133..8e0d59e 100644 --- a/cmd/node/zip_snapshot.go +++ b/cmd/node/zip_snapshot.go @@ -6,6 +6,7 @@ import ( "github.com/bcdevtools/node-management/utils" "github.com/spf13/cobra" "os" + "os/exec" "path" "strings" "time" @@ -68,13 +69,34 @@ func GetZipSnapshotCmd() *cobra.Command { return } - _ = os.RemoveAll(path.Join(dataDirPath, "snapshots")) - _ = os.RemoveAll(path.Join(dataDirPath, "tx_index.db")) - - // zip data dir - zipFileNameWithoutExt := fmt.Sprintf("snapshot_%s.tar.lz4", utils.GetDateTimeStringCompatibleWithFileName(time.Now().UTC(), time.DateTime)) + workingDir, err := os.Getwd() // zip data dir + if err != nil { + utils.ExitWithErrorMsg("ERR: failed to get current working directory") + return + } - ec := utils.LaunchApp("/bin/bash", []string{"-c", fmt.Sprintf("tar cvf - %s | lz4 - %s", dataDirPath, zipFileNameWithoutExt)}) + ec := utils.LaunchAppWithSetup( + "/bin/bash", []string{ + "-c", fmt.Sprintf("tar --exclude %s --exclude %s -cvf - %s | lz4 - %s", + "./data/snapshots", + "./data/tx_index.db", + "./data", + path.Join( + workingDir, + fmt.Sprintf( + "snapshot_%s.tar.lz4", + utils.GetDateTimeStringCompatibleWithFileName(time.Now().UTC(), time.DateTime), + ), + ), + ), + }, + func(launchCmd *exec.Cmd) { + launchCmd.Dir = path.Dir(dataDirPath) + launchCmd.Stdin = os.Stdin + launchCmd.Stdout = os.Stdout + launchCmd.Stderr = os.Stderr + }, + ) if ec != 0 { utils.ExitWithErrorMsg("ERR: failed to zip data dir") return diff --git a/constants/varcons.go b/constants/varcons.go index 793a6f5..c63c764 100644 --- a/constants/varcons.go +++ b/constants/varcons.go @@ -4,7 +4,7 @@ package constants //goland:noinspection GoSnakeCaseUsage var ( - VERSION = "1.0.0" + VERSION = "" COMMIT_HASH = "" BUILD_DATE = "" ) diff --git a/types/config_toml.go b/types/config_toml.go index cee351d..5ccdf1c 100644 --- a/types/config_toml.go +++ b/types/config_toml.go @@ -1,5 +1,14 @@ package types +import ( + "fmt" + "github.com/bcdevtools/node-management/utils" + "github.com/pelletier/go-toml/v2" + "github.com/pkg/errors" + "os" + "strings" +) + type P2pConfigToml struct { Seeds string `toml:"seeds"` Laddr string `toml:"laddr"` @@ -34,3 +43,44 @@ type ConfigToml struct { TxIndex *TxIndexConfigToml `toml:"tx_index"` RPC *RpcConfigToml `toml:"rpc"` } + +func ReadNodeRpcFromConfigToml(configFilePath string) (rpc string, err error) { + var exists bool + _, exists, _, err = utils.FileInfo(configFilePath) + if err != nil { + err = errors.Wrap(err, "failed to check "+configFilePath) + return + } + if !exists { + err = fmt.Errorf("file not found: " + configFilePath) + return + } + + var bz []byte + bz, err = os.ReadFile(configFilePath) + if err != nil { + err = errors.Wrap(err, "failed to read "+configFilePath) + return + } + + var config ConfigToml + err = toml.Unmarshal(bz, &config) + if err != nil { + err = errors.Wrap(err, "failed to unmarshal "+configFilePath) + return + } + if config.RPC == nil || config.RPC.LAddr == "" { + err = fmt.Errorf("rpc section, address is not set in " + configFilePath) + return + } + + addr := strings.TrimSpace(config.RPC.LAddr) + addr = strings.TrimPrefix(addr, "tcp://") + addr = strings.TrimSuffix(addr, "/") + //goland:noinspection HttpUrlsUsage + if !strings.HasPrefix(addr, "http://") { + addr = "http://" + addr + } + + return addr, nil +}