Merge 8388326 into e228d09

cmd/captplanet/farmer/main.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"github.com/spf13/cobra"
+
+	"storj.io/storj/pkg/cfgstruct"
+	"storj.io/storj/pkg/kademlia"
+	"storj.io/storj/pkg/piecestore/psservice"
+	"storj.io/storj/pkg/process"
+	"storj.io/storj/pkg/provider"
+)
+
+var (
+	rootCmd = &cobra.Command{
+		Use:   "farmer",
+		Short: "Farmer",
+	}
+
+	cfg struct {
+		Identity provider.IdentityConfig
+		Kademlia kademlia.Config
+		Storage  psservice.Config
+	}
+)
+
+func init() {
+	rootCmd.AddCommand(&cobra.Command{
+		Use:   "run",
+		Short: "Run the farmer",
+		RunE:  cmdRun,
+	})
+	cfgstruct.Bind(rootCmd.PersistentFlags(), &cfg)
+}
+
+func cmdRun(cmd *cobra.Command, args []string) (err error) {
+	return cfg.Identity.Run(process.Ctx(cmd), cfg.Kademlia, cfg.Storage)
+}
+
+func main() {
+	process.ExecuteWithConfig(rootCmd, "$HOME/.storj/farmer/config.yaml")
+}

cmd/captplanet/gw/main.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"github.com/spf13/cobra"
+
+	"storj.io/storj/pkg/cfgstruct"
+	"storj.io/storj/pkg/miniogw"
+	"storj.io/storj/pkg/process"
+)
+
+var (
+	rootCmd = &cobra.Command{
+		Use:   "gw",
+		Short: "Gateway",
+	}
+
+	cfg miniogw.Config
+)
+
+func init() {
+	rootCmd.AddCommand(&cobra.Command{
+		Use:   "run",
+		Short: "Run the gateway",
+		RunE:  cmdRun,
+	})
+	cfgstruct.Bind(rootCmd.PersistentFlags(), &cfg)
+}
+
+func cmdRun(cmd *cobra.Command, args []string) (err error) {
+	return cfg.Run(process.Ctx(cmd))
+}
+
+func main() {
+	process.ExecuteWithConfig(rootCmd, "$HOME/.storj/gw/config.yaml")
+}

cmd/captplanet/hc/main.go

@@ -0,0 +1,46 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"github.com/spf13/cobra"
+
+	"storj.io/storj/pkg/cfgstruct"
+	"storj.io/storj/pkg/kademlia"
+	"storj.io/storj/pkg/overlay"
+	"storj.io/storj/pkg/pointerdb"
+	"storj.io/storj/pkg/process"
+	"storj.io/storj/pkg/provider"
+)
+
+var (
+	rootCmd = &cobra.Command{
+		Use:   "hc",
+		Short: "Heavy client",
+	}
+
+	cfg struct {
+		Identity  provider.IdentityConfig
+		Kademlia  kademlia.Config
+		PointerDB pointerdb.Config
+		Overlay   overlay.Config
+	}
+)
+
+func init() {
+	rootCmd.AddCommand(&cobra.Command{
+		Use:   "run",
+		Short: "Run the heavy client",
+		RunE:  cmdRun,
+	})
+	cfgstruct.Bind(rootCmd.PersistentFlags(), &cfg)
+}
+
+func cmdRun(cmd *cobra.Command, args []string) (err error) {
+	return cfg.Identity.Run(process.Ctx(cmd), cfg.Kademlia, cfg.PointerDB, cfg.Overlay)
+}
+
+func main() {
+	process.ExecuteWithConfig(rootCmd, "$HOME/.storj/hc/config.yaml")
+}

cmd/captplanet/main.go

@@ -0,0 +1,27 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"flag"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	monkit "gopkg.in/spacemonkeygo/monkit.v2"
+	"storj.io/storj/pkg/process"
+)
+
+var (
+	mon = monkit.Package()
+
+	rootCmd = &cobra.Command{
+		Use:   "captplanet",
+		Short: "Captain Planet! With our powers combined!",
+	}
+)
+
+func main() {
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	process.ExecuteWithConfig(rootCmd, "$HOME/.storj/capt/config.yaml")
+}

cmd/captplanet/run.go

@@ -0,0 +1,134 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	"storj.io/storj/pkg/cfgstruct"
+	"storj.io/storj/pkg/kademlia"
+	"storj.io/storj/pkg/miniogw"
+	"storj.io/storj/pkg/overlay"
+	"storj.io/storj/pkg/piecestore/psservice"
+	"storj.io/storj/pkg/pointerdb"
+	"storj.io/storj/pkg/process"
+	"storj.io/storj/pkg/provider"
+)
+
+var (
+	runCmd = &cobra.Command{
+		Use:   "run",
+		Short: "Run all providers",
+		RunE:  cmdRun,
+	}
+	runCfg Config
+)
+
+func init() {
+	rootCmd.AddCommand(runCmd)
+	cfgstruct.Bind(runCmd.Flags(), &runCfg)
+}
+
+func cmdRun(cmd *cobra.Command, args []string) (err error) {
+	ctx := process.Ctx(cmd)
+	defer mon.Task()(&ctx)(&err)
+
+	startingPort := runCfg.StartingPort
+
+	errch := make(chan error, runCfg.FarmerCount+2)
+
+	// define heavy client config programmatically
+	type HeavyClient struct {
+		Identity  provider.IdentityConfig
+		Kademlia  kademlia.Config
+		PointerDB pointerdb.Config
+		Overlay   overlay.Config
+	}
+
+	hc := HeavyClient{
+		Identity: provider.IdentityConfig{
+			CertPath: filepath.Join(runCfg.BasePath, "hc", "ident.leaf.cert"),
+			KeyPath:  filepath.Join(runCfg.BasePath, "hc", "ident.leaf.key"),
+			Address:  joinHostPort(runCfg.ListenHost, startingPort+1),
+		},
+		Kademlia: kademlia.Config{
+			TODOListenAddr: joinHostPort(runCfg.ListenHost, startingPort+2),
+			BootstrapAddr:  joinHostPort(runCfg.ListenHost, startingPort+4),
+		},
+		PointerDB: pointerdb.Config{
+			DatabaseURL: "bolt://" + filepath.Join(
+				runCfg.BasePath, "hc", "pointerdb.db"),
+		},
+		Overlay: overlay.Config{
+			DatabaseURL: "bolt://" + filepath.Join(
+				runCfg.BasePath, "hc", "overlay.db"),
+		},
+	}
+
+	// start heavy client
+	go func() {
+		errch <- hc.Identity.Run(ctx, hc.Kademlia, hc.PointerDB, hc.Overlay)
+	}()
+
+	// define and start a bunch of farmers programmatically
+	type Farmer struct {
+		Identity provider.IdentityConfig
+		Kademlia kademlia.Config
+		Storage  psservice.Config
+	}
+
+	for i := 0; i < runCfg.FarmerCount; i++ {
+		basepath := filepath.Join(runCfg.BasePath, fmt.Sprintf("f%d", i))
+		farmer := Farmer{
+			Identity: provider.IdentityConfig{
+				CertPath: filepath.Join(basepath, "ident.leaf.cert"),
+				KeyPath:  filepath.Join(basepath, "ident.leaf.key"),
+				Address:  joinHostPort(runCfg.ListenHost, startingPort+i*2+3),
+			},
+			Kademlia: kademlia.Config{
+				TODOListenAddr: joinHostPort(runCfg.ListenHost, startingPort+i*2+4),
+				BootstrapAddr:  joinHostPort(runCfg.ListenHost, startingPort+1),
+			},
+			Storage: psservice.Config{
+				Path: filepath.Join(basepath, "data"),
+			},
+		}
+		go func() {
+			errch <- farmer.Identity.Run(ctx, farmer.Kademlia, farmer.Storage)
+		}()
+	}
+
+	// start s3 gateway
+	gw := miniogw.Config{
+		IdentityConfig: provider.IdentityConfig{
+			CertPath: filepath.Join(runCfg.BasePath, "gw", "ident.leaf.cert"),
+			KeyPath:  filepath.Join(runCfg.BasePath, "gw", "ident.leaf.key"),
+			Address:  joinHostPort(runCfg.ListenHost, startingPort),
+		},
+		MinioConfig: runCfg.MinioConfig,
+		ClientConfig: miniogw.ClientConfig{
+			OverlayAddr: joinHostPort(
+				runCfg.ListenHost, startingPort+1),
+			PointerDBAddr: joinHostPort(
+				runCfg.ListenHost, startingPort+1),
+		},
+		RSConfig: runCfg.RSConfig,
+	}
+	gw.MinioConfig.MinioDir = filepath.Join(runCfg.BasePath, "gw", "minio")
+
+	// start s3 gateway
+	go func() {
+		errch <- gw.Run(ctx)
+	}()
+
+	return <-errch
+}
+
+func joinHostPort(host string, port int) string {
+	return net.JoinHostPort(host, fmt.Sprint(port))
+}

cmd/captplanet/setup.go

@@ -0,0 +1,80 @@
+// Copyright (C) 2018 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	"storj.io/storj/pkg/cfgstruct"
+	"storj.io/storj/pkg/miniogw"
+	"storj.io/storj/pkg/peertls"
+	"storj.io/storj/pkg/process"
+)
+
+// Config defines broad Captain Planet configuration
+type Config struct {
+	FarmerCount  int    `help:"number of farmers to run" default:"20"`
+	BasePath     string `help:"base path for captain planet storage" default:"$HOME/.storj/capt"`
+	ListenHost   string `help:"the host for providers to listen on" default:"127.0.0.1"`
+	StartingPort int    `help:"all providers will listen on ports consecutively starting with this one" default:"7777"`
+	miniogw.RSConfig
+	miniogw.MinioConfig
+}
+
+var (
+	setupCmd = &cobra.Command{
+		Use:   "setup",
+		Short: "Set up configurations",
+		RunE:  cmdSetup,
+	}
+	setupCfg Config
+)
+
+func init() {
+	rootCmd.AddCommand(setupCmd)
+	cfgstruct.Bind(setupCmd.Flags(), &setupCfg)
+}
+
+func cmdSetup(cmd *cobra.Command, args []string) (err error) {
+	hcPath := filepath.Join(setupCfg.BasePath, "hc")
+	err = os.MkdirAll(hcPath, 0700)
+	if err != nil {
+		return err
+	}
+	identPath := filepath.Join(hcPath, "ident")
+	_, err = peertls.NewTLSFileOptions(identPath, identPath, true, false)
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < setupCfg.FarmerCount; i++ {
+		farmerPath := filepath.Join(setupCfg.BasePath, fmt.Sprintf("f%d", i))
+		err = os.MkdirAll(farmerPath, 0700)
+		if err != nil {
+			return err
+		}
+		identPath = filepath.Join(farmerPath, "ident")
+		_, err = peertls.NewTLSFileOptions(identPath, identPath, true, false)
+		if err != nil {
+			return err
+		}
+	}
+
+	gwPath := filepath.Join(setupCfg.BasePath, "gw")
+	err = os.MkdirAll(gwPath, 0700)
+	if err != nil {
+		return err
+	}
+	identPath = filepath.Join(gwPath, "ident")
+	_, err = peertls.NewTLSFileOptions(identPath, identPath, true, false)
+	if err != nil {
+		return err
+	}
+
+	return process.SaveConfig(cmd)
+}