40 fixes regarding database path and help for migration (#58)

* working migration

* removed template file

* removed checking error incase it has already been executed

* fixed root and generated domain

* removed services from domain structure and all its references

cli/app/config_cmd.go

@@ -30,7 +30,7 @@ func configCommand() *cli.Command {
 					},
 				},
 				Action: func(ctx *cli.Context) error {
-					_, _, _, err := parseSourceConfig(ctx)
+					_, _, _, err := parseSourceConfig(ctx, ctx.String("shape"))
 					return err
 				},
 			},
@@ -52,7 +52,7 @@ func configCommand() *cli.Command {
 					},
 				},
 				Action: func(ctx *cli.Context) error {
-					pid, cnf, _, err := parseSourceConfig(ctx)
+					pid, cnf, _, err := parseSourceConfig(ctx, ctx.String("shape"))
 					if err != nil {
 						return err
 					}

cli/app/config_test.go

@@ -2,13 +2,15 @@ package app
 
 import (
 	"context"
+	"fmt"
 	"log"
 	"os"
 	"testing"
 	"time"
 
 	_ "embed"
 
+	"github.com/taubyte/tau/config"
 	"gotest.tools/v3/assert"
 )
 
@@ -24,6 +26,7 @@ func TestConfig(t *testing.T) {
 	}
 	defer os.RemoveAll(root)
 
+	fmt.Println("ROOT ", root)
 	os.Mkdir(root+"/storage", 0750)
 	os.Mkdir(root+"/storage/test", 0750)
 	os.Mkdir(root+"/config", 0750)
@@ -37,4 +40,8 @@ func TestConfig(t *testing.T) {
 
 	err = app.RunContext(ctx, []string{os.Args[0], "cnf", "show", "-s", "test", "--root", root})
 	assert.NilError(t, err)
+
+	config.DefaultRoot = root
+	err = app.RunContext(ctx, []string{os.Args[0], "cnf", "show", "-s", "test"})
+	assert.NilError(t, err)
 }

cli/app/gen_config.go

@@ -13,7 +13,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
 	"strings"
 
 	"github.com/libp2p/go-libp2p/core/crypto"
@@ -74,8 +73,7 @@ func generateSourceConfig(ctx *cli.Context) (string, error) {
 				Private: path.Join("keys", "dv_private.pem"),
 				Public:  path.Join("keys", "dv_public.pem"),
 			},
-			Generated: regexp.QuoteMeta(fmt.Sprintf("g.%s", ctx.String("network"))) + `$`,
-			Services:  `^[^.]+\.tau\.` + regexp.QuoteMeta(ctx.String("network")) + `$`,
+			Generated: fmt.Sprintf("g.%s", ctx.String("network")),
 		},
 	}
 

cli/app/helpers.go

@@ -46,7 +46,19 @@ func deleteEmpty(s []string) []string {
 
 func setNetworkDomains(conf *config.Source) {
 	domainSpecs.WhiteListedDomains = conf.Domains.Whitelist.Postfix
-	domainSpecs.TaubyteServiceDomain = regexp.MustCompile(conf.Domains.Services)
+	domainSpecs.TaubyteServiceDomain = regexp.MustCompile(convertToServiceRegex(conf.NetworkFqdn))
 	domainSpecs.SpecialDomain = regexp.MustCompile(conf.Domains.Generated)
 	domainSpecs.TaubyteHooksDomain = regexp.MustCompile(fmt.Sprintf(`https://patrick.tau.%s`, conf.NetworkFqdn))
 }
+
+func convertToServiceRegex(url string) string {
+	urls := strings.Split(url, ".")
+	serviceRegex := `^[^.]+\.tau`
+	var network string
+	for _, _url := range urls {
+		network += fmt.Sprintf(`\.%s`, _url)
+	}
+
+	serviceRegex += network
+	return serviceRegex
+}

cli/app/migrate.go

@@ -0,0 +1,21 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+)
+
+func migrateDatabase(ctx context.Context, shape string, elder bool) error {
+	err := os.Rename(fmt.Sprintf("/tb/storage/databases/%s", shape), fmt.Sprintf("/tb/storage/%s", shape))
+	if err != nil {
+		return fmt.Errorf("migrating %s out of database failed with: %w", shape, err)
+	}
+
+	if !elder {
+		err = os.Rename(fmt.Sprintf("/tb/storage/databases/%s_client", shape), fmt.Sprintf("/tb/storage/%s_client", shape))
+		return fmt.Errorf("migrating %s_client out of database failed with: %w", shape, err)
+	}
+
+	return nil
+}

cli/app/parse_config.go

@@ -22,8 +22,11 @@ import (
 // TODO: move to config as a methods
 
 // Parse from yaml
-func parseSourceConfig(ctx *cli.Context) (string, *config.Node, *config.Source, error) {
+func parseSourceConfig(ctx *cli.Context, shape string) (string, *config.Node, *config.Source, error) {
 	root := ctx.Path("root")
+	if root == "" {
+		root = config.DefaultRoot
+	}
 
 	if !filepath.IsAbs(root) {
 		return "", nil, nil, fmt.Errorf("root folder `%s` is not absolute", root)
@@ -32,7 +35,7 @@ func parseSourceConfig(ctx *cli.Context) (string, *config.Node, *config.Source,
 	configRoot := root + "/config"
 	configPath := ctx.Path("path")
 	if configPath == "" {
-		configPath = path.Join(configRoot, ctx.String("shape")+".yaml")
+		configPath = path.Join(configRoot, shape+".yaml")
 	}
 
 	data, err := os.ReadFile(configPath)
@@ -59,14 +62,14 @@ func parseSourceConfig(ctx *cli.Context) (string, *config.Node, *config.Source,
 
 	protocol := &config.Node{
 		Root:            root,
-		Shape:           ctx.String("shape"),
+		Shape:           shape,
 		P2PAnnounce:     src.P2PAnnounce,
 		P2PListen:       src.P2PListen,
 		Ports:           src.Ports.ToMap(),
 		Location:        src.Location,
 		NetworkFqdn:     src.NetworkFqdn,
 		GeneratedDomain: src.Domains.Generated,
-		ServicesDomain:  src.Domains.Services,
+		ServicesDomain:  convertToServiceRegex(src.NetworkFqdn),
 		HttpListen:      "0.0.0.0:443",
 		Protocols:       src.Protocols,
 		Plugins:         src.Plugins,

cli/app/service_regex_test.go

@@ -0,0 +1,18 @@
+package app
+
+import (
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+var (
+	testUrl       = "example.test.com"
+	expectedRegex = `^[^.]+\.tau\.example\.test\.com`
+)
+
+func TestServiceRegex(t *testing.T) {
+	url := convertToServiceRegex(testUrl)
+	assert.Equal(t, url, expectedRegex)
+
+}

cli/app/start_cmd.go

@@ -2,6 +2,8 @@ package app
 
 import (
 	"fmt"
+	"os"
+	"os/exec"
 
 	"github.com/taubyte/tau/cli/node"
 	"github.com/taubyte/tau/config"
@@ -29,11 +31,27 @@ func startCommand() *cli.Command {
 		},
 
 		Action: func(ctx *cli.Context) error {
-			_, protocolConfig, sourceConfig, err := parseSourceConfig(ctx)
+			shape := ctx.String("shape")
+			_, protocolConfig, sourceConfig, err := parseSourceConfig(ctx, shape)
 			if err != nil {
 				return fmt.Errorf("parsing config failed with: %s", err)
 			}
 
+			// Migration Start
+			if _, err := os.Stat(fmt.Sprintf("/tb/storage/databases/%s", shape)); !os.IsNotExist(err) {
+				err = migrateDatabase(ctx.Context, shape, len(protocolConfig.Protocols) == 0)
+				if err != nil {
+					return fmt.Errorf("migrating shape %s failed with: %w", shape, err)
+				}
+			}
+
+			cmd := exec.Command("sudo", "systemctl", "stop", fmt.Sprintf("odo@%s.service", shape))
+			cmd.CombinedOutput()
+
+			cmd = exec.Command("sudo", "systemctl", "disable", fmt.Sprintf("odo@%s.service", shape))
+			cmd.CombinedOutput()
+			// Migration End
+
 			setNetworkDomains(sourceConfig)
 			return node.Start(ctx.Context, protocolConfig)
 		},

config/p2p_helpers.go

@@ -3,7 +3,6 @@ package config
 import (
 	"context"
 	"fmt"
-	"path"
 	"time"
 
 	"github.com/taubyte/p2p/peer"
@@ -12,9 +11,9 @@ import (
 
 var WaitForSwamDuration = 10 * time.Second
 
-func NewNode(ctx context.Context, config *Node, databaseName string) (peer.Node, error) {
+func NewNode(ctx context.Context, config *Node, storagePath string) (peer.Node, error) {
 	if config.DevMode {
-		return NewLiteNode(ctx, config, databaseName)
+		return NewLiteNode(ctx, config, storagePath)
 	}
 
 	bootstrapParam, err := utils.ConvertBootstrap(config.Peers, config.DevMode)
@@ -24,7 +23,7 @@ func NewNode(ctx context.Context, config *Node, databaseName string) (peer.Node,
 
 	peerNode, err := peer.NewPublic(
 		ctx,
-		path.Join(config.Root, databaseName),
+		storagePath,
 		config.PrivateKey,
 		config.SwarmKey,
 		config.P2PListen,
@@ -43,15 +42,15 @@ func NewNode(ctx context.Context, config *Node, databaseName string) (peer.Node,
 	return peerNode, nil
 }
 
-func NewLiteNode(ctx context.Context, config *Node, databaseName string) (peer.Node, error) {
+func NewLiteNode(ctx context.Context, config *Node, storagePath string) (peer.Node, error) {
 	bootstrapParam, err := utils.ConvertBootstrap(config.Peers, config.DevMode)
 	if err != nil {
 		return nil, fmt.Errorf("getting bootstrap perms in NewLiteNode failed with: %s", err)
 	}
 
 	node, err := peer.NewLitePublic(
 		ctx,
-		path.Join(config.Root, databaseName),
+		storagePath,
 		config.PrivateKey,
 		config.SwarmKey,
 		config.P2PListen,

config/tau.go

@@ -41,7 +41,6 @@ type Plugins struct {
 type Domains struct {
 	Key       DVKey            `yaml:"key"`
 	Whitelist DomainsWhiteList `yaml:"whitelist"`
-	Services  string           `yaml:"services"`
 	Generated string           `yaml:"generated"`
 }
 

protocols/auth/service.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"path"
 
 	"github.com/ipfs/go-log/v2"
 	seerIface "github.com/taubyte/go-interfaces/services/seer"
@@ -41,7 +42,7 @@ func New(ctx context.Context, config *tauConfig.Node) (*AuthService, error) {
 	}
 
 	if config.Node == nil {
-		srv.node, err = tauConfig.NewNode(ctx, config, protocolCommon.Auth)
+		srv.node, err = tauConfig.NewNode(ctx, config, path.Join(config.Root, protocolCommon.Auth))
 		if err != nil {
 			return nil, err
 		}

protocols/hoarder/service.go

@@ -3,6 +3,7 @@ package hoarder
 import (
 	"context"
 	"fmt"
+	"path"
 
 	"github.com/fxamacker/cbor/v2"
 	"github.com/ipfs/go-log/v2"
@@ -39,7 +40,7 @@ func New(ctx context.Context, config *tauConfig.Node) (*Service, error) {
 
 	// TODO move database root to new
 	if config.Node == nil {
-		srv.node, err = tauConfig.NewNode(ctx, config, protocolCommon.Hoarder)
+		srv.node, err = tauConfig.NewNode(ctx, config, path.Join(config.Root, protocolCommon.Hoarder))
 		if err != nil {
 			return nil, fmt.Errorf("config new node failed with: %s", err)
 		}

protocols/monkey/service.go

@@ -3,6 +3,7 @@ package monkey
 import (
 	"context"
 	"fmt"
+	"path"
 	"regexp"
 
 	"github.com/ipfs/go-log/v2"
@@ -66,7 +67,7 @@ func New(ctx context.Context, config *tauConfig.Node) (*Service, error) {
 	}
 
 	if config.Node == nil {
-		srv.node, err = tauConfig.NewLiteNode(ctx, config, protocolCommon.Monkey)
+		srv.node, err = tauConfig.NewLiteNode(ctx, config, path.Join(config.Root, protocolCommon.Monkey))
 		if err != nil {
 			return nil, err
 		}

protocols/patrick/service.go

@@ -3,6 +3,7 @@ package service
 import (
 	"context"
 	"fmt"
+	"path"
 	"time"
 
 	"github.com/ipfs/go-log/v2"
@@ -43,7 +44,7 @@ func New(ctx context.Context, config *tauConfig.Node) (*PatrickService, error) {
 	logger.Info(config)
 
 	if config.Node == nil {
-		srv.node, err = tauConfig.NewNode(ctx, config, protocolsCommon.Patrick)
+		srv.node, err = tauConfig.NewNode(ctx, config, path.Join(config.Root, protocolsCommon.Patrick))
 		if err != nil {
 			return nil, err
 		}

protocols/seer/service.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"path"
 	"regexp"
 	"time"
 
@@ -55,7 +56,7 @@ func New(ctx context.Context, config *tauConfig.Node, opts ...Options) (*Service
 	}
 
 	if config.Node == nil {
-		srv.node, err = tauConfig.NewLiteNode(ctx, config, protocolsCommon.Seer)
+		srv.node, err = tauConfig.NewLiteNode(ctx, config, path.Join(config.Root, protocolsCommon.Seer))
 		if err != nil {
 			return nil, fmt.Errorf("new lite node failed with: %s", err)
 		}

protocols/substrate/service.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 
 	"github.com/ipfs/go-log/v2"
 	"github.com/taubyte/go-interfaces/vm"
@@ -44,7 +45,7 @@ func New(ctx context.Context, config *tauConfig.Node) (*Service, error) {
 	srv.verbose = config.Verbose
 
 	if config.Node == nil {
-		srv.node, err = tauConfig.NewLiteNode(ctx, config, protocolCommon.Substrate)
+		srv.node, err = tauConfig.NewLiteNode(ctx, config, path.Join(config.Root, protocolCommon.Substrate))
 		if err != nil {
 			return nil, fmt.Errorf("creating new lite node failed with: %w", err)
 		}

protocols/tns/service.go

@@ -3,6 +3,7 @@ package tns
 import (
 	"context"
 	"fmt"
+	"path"
 
 	"github.com/ipfs/go-log/v2"
 	"github.com/taubyte/go-interfaces/services/seer"
@@ -33,7 +34,7 @@ func New(ctx context.Context, config *tauConfig.Node) (*Service, error) {
 	}
 
 	if config.Node == nil {
-		srv.node, err = tauConfig.NewNode(ctx, config, protocolsCommon.Tns)
+		srv.node, err = tauConfig.NewNode(ctx, config, path.Join(config.Root, protocolsCommon.Tns))
 		if err != nil {
 			return nil, err
 		}