Dockerize Kelp binary (#213), closes #211

* 1 - initial dockerfile which runs with GUI server

* 2 - cli takes in ccxt-rest-url param on the trade and server commands

* 3 - better error message when ccxt cannot be reached

* 4 - set ccxt-rest-url from server command which uses it to load exchanges in options_metadata.go

* 5 - updated Dockerfile to run any command using the kelp binary

* 6 - ccxt-rest-url param is set on the root command so its common for all subcommands

* 7 - updated Dockerfile to pull tags

* 8 - add volume mounting to the sameple docker command

cmd/exchanges.go

@@ -16,6 +16,7 @@ var exchanagesCmd = &cobra.Command{
 
 func init() {
 	exchanagesCmd.Run = func(ccmd *cobra.Command, args []string) {
+		checkInitRootFlags()
 		// call sdk.GetExchangeList() here so we pre-load exchanges before displaying the table
 		sdk.GetExchangeList()
 		fmt.Printf("  Exchange\t\t\tTested\t\tTrading\t\tDescription\n")

cmd/root.go

@@ -3,9 +3,13 @@ package cmd
 import (
 	"fmt"
 	"log"
+	"net/http"
 	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
+	"github.com/stellar/kelp/support/networking"
+	"github.com/stellar/kelp/support/sdk"
 )
 
 // build flags
@@ -48,9 +52,13 @@ var RootCmd = &cobra.Command{
 	},
 }
 
+var rootCcxtRestURL *string
+
 func init() {
 	validateBuild()
 
+	rootCcxtRestURL = RootCmd.PersistentFlags().String("ccxt-rest-url", "", "URL to use for the CCXT-rest API. Takes precendence over the CCXT_REST_URL param set in the botConfg file for the trade command and passed as a parameter into the Kelp subprocesses started by the GUI (default URL is https://localhost:3000)")
+
 	RootCmd.AddCommand(tradeCmd)
 	if env == envDev {
 		RootCmd.AddCommand(serverCmd)
@@ -61,9 +69,36 @@ func init() {
 	RootCmd.AddCommand(versionCmd)
 }
 
+func checkInitRootFlags() {
+	if *rootCcxtRestURL != "" {
+		*rootCcxtRestURL = strings.TrimSuffix(*rootCcxtRestURL, "/")
+		if !strings.HasPrefix(*rootCcxtRestURL, "http://") && !strings.HasPrefix(*rootCcxtRestURL, "https://") {
+			panic("'ccxt-rest-url' argument must start with either `http://` or `https://`")
+		}
+
+		e := testCcxtURL(*rootCcxtRestURL)
+		if e != nil {
+			panic(e)
+		}
+
+		e = sdk.SetBaseURL(*rootCcxtRestURL)
+		if e != nil {
+			panic(fmt.Errorf("unable to set CCXT-rest URL to '%s': %s", *rootCcxtRestURL, e))
+		}
+	}
+}
+
 func validateBuild() {
 	if version == "" || buildDate == "" || gitBranch == "" || gitHash == "" {
 		fmt.Println("version information not included, please build using the build script (scripts/build.sh)")
 		os.Exit(1)
 	}
 }
+
+func testCcxtURL(ccxtURL string) error {
+	e := networking.JSONRequest(http.DefaultClient, "GET", ccxtURL, "", map[string]string{}, nil, "")
+	if e != nil {
+		return fmt.Errorf("unable to connect to ccxt at the URL: %s", ccxtURL)
+	}
+	return nil
+}

cmd/server.go

@@ -40,6 +40,7 @@ func init() {
 	options.horizonPubnetURI = serverCmd.Flags().String("horizon-pubnet-uri", "https://horizon.stellar.org", "URI to use for the horizon instance connected to the Stellar Public Network (must not contain the word 'test')")
 
 	serverCmd.Run = func(ccmd *cobra.Command, args []string) {
+		checkInitRootFlags()
 		if !strings.Contains(*options.horizonTestnetURI, "test") {
 			panic("'horizon-testnet-uri' argument must contain the word 'test'")
 		}
@@ -48,7 +49,7 @@ func init() {
 		}
 
 		kos := kelpos.GetKelpOS()
-		s, e := backend.MakeAPIServer(kos, *options.horizonTestnetURI, *options.horizonPubnetURI)
+		s, e := backend.MakeAPIServer(kos, *options.horizonTestnetURI, *options.horizonPubnetURI, *rootCcxtRestURL)
 		if e != nil {
 			panic(e)
 		}

cmd/strategies.go

@@ -15,6 +15,7 @@ var strategiesCmd = &cobra.Command{
 
 func init() {
 	strategiesCmd.Run = func(ccmd *cobra.Command, args []string) {
+		checkInitRootFlags()
 		fmt.Printf("  Strategy\tComplexity\tNeeds Config\tDescription\n")
 		fmt.Printf("  --------------------------------------------------------------------------------\n")
 		strategies := plugins.Strategies()

cmd/trade.go

@@ -78,6 +78,8 @@ type inputs struct {
 }
 
 func validateCliParams(l logger.Logger, options inputs) {
+	checkInitRootFlags()
+
 	if *options.operationalBuffer < 0 {
 		panic(fmt.Sprintf("invalid operationalBuffer argument, must be non-negative: %f", *options.operationalBuffer))
 	}
@@ -415,7 +417,7 @@ func runTradeCmd(options inputs) {
 		}
 	}
 
-	if botConfig.CcxtRestURL != nil {
+	if *rootCcxtRestURL == "" && botConfig.CcxtRestURL != nil {
 		e := sdk.SetBaseURL(*botConfig.CcxtRestURL)
 		if e != nil {
 			logger.Fatal(l, fmt.Errorf("unable to set CCXT-rest URL to '%s': %s", *botConfig.CcxtRestURL, e))

gui/backend/api_server.go

@@ -24,6 +24,7 @@ type APIServer struct {
 	kos                   *kelpos.KelpOS
 	horizonTestnetURI     string
 	horizonPubnetURI      string
+	ccxtRestUrl           string
 	apiTestNet            *horizonclient.Client
 	apiPubNet             *horizonclient.Client
 	apiTestNetOld         *horizon.Client
@@ -32,7 +33,7 @@ type APIServer struct {
 }
 
 // MakeAPIServer is a factory method
-func MakeAPIServer(kos *kelpos.KelpOS, horizonTestnetURI string, horizonPubnetURI string) (*APIServer, error) {
+func MakeAPIServer(kos *kelpos.KelpOS, horizonTestnetURI string, horizonPubnetURI string, ccxtRestUrl string) (*APIServer, error) {
 	binPath, e := filepath.Abs(os.Args[0])
 	if e != nil {
 		return nil, fmt.Errorf("could not get binPath of currently running binary: %s", e)
@@ -46,6 +47,7 @@ func MakeAPIServer(kos *kelpos.KelpOS, horizonTestnetURI string, horizonPubnetUR
 	horizonPubnetURI = strings.TrimSuffix(horizonPubnetURI, "/")
 	log.Printf("using horizonTestnetURI: %s\n", horizonTestnetURI)
 	log.Printf("using horizonPubnetURI: %s\n", horizonPubnetURI)
+	log.Printf("using ccxtRestUrl: %s\n", ccxtRestUrl)
 	apiTestNet := &horizonclient.Client{
 		HorizonURL: horizonTestnetURI,
 		HTTP:       http.DefaultClient,
@@ -76,6 +78,7 @@ func MakeAPIServer(kos *kelpos.KelpOS, horizonTestnetURI string, horizonPubnetUR
 		kos:                   kos,
 		horizonTestnetURI:     horizonTestnetURI,
 		horizonPubnetURI:      horizonPubnetURI,
+		ccxtRestUrl:           ccxtRestUrl,
 		apiTestNet:            apiTestNet,
 		apiPubNet:             apiPubNet,
 		apiTestNetOld:         apiTestNetOld,

gui/backend/start_bot.go

@@ -42,6 +42,9 @@ func (s *APIServer) doStartBot(botName string, strategy string, iterations *uint
 	if iterations != nil {
 		command = fmt.Sprintf("%s --iter %d", command, *iterations)
 	}
+	if s.ccxtRestUrl != "" {
+		command = fmt.Sprintf("%s --ccxt-rest-url %s", command, s.ccxtRestUrl)
+	}
 	log.Printf("run command for bot '%s': %s\n", botName, command)
 
 	p, e := s.runKelpCommandBackground(botName, command)

ops/Dockerfile

@@ -0,0 +1,35 @@
+FROM golang:1.12.7
+
+LABEL maintainer="Nikhil Saraf <Github: @nikhilsaraf>"
+
+# install yarn
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+RUN apt update
+RUN apt -y install yarn
+
+# set working dir
+WORKDIR /go/src/github.com/stellar/kelp
+RUN pwd
+
+# install glide
+RUN curl -s https://glide.sh/get | sh
+
+# install kelp
+ENV GOPATH /go
+RUN git clone https://github.com/stellar/kelp.git .
+RUN git fetch --tags
+RUN glide install
+RUN ./scripts/build.sh
+
+# set ulimit
+RUN ulimit -n 10000
+
+# use command line arguments from invocation of docker run against this ENTRYPOINT command - https://stackoverflow.com/a/40312311/1484710
+ENTRYPOINT ["./bin/kelp"]
+
+# sample command to run this container as a daemon process:
+# docker run -v `pwd`/ops:/go/src/github.com/stellar/kelp/bin/ops -d -p 8011:8011 <image> server -p 8011 --ccxt-rest-url=http://host.docker.internal:3000
+# this assumes that you are running ccxt on port 3000 outside this kelp docker container
+# the three port numbers (8011 in the example above) must be the same and must be specified
+# the first part of the -v argument is the directory where you want to save the kelp configs and kelp logs from the bots created in this container

support/sdk/ccxt.go

@@ -20,10 +20,8 @@ var ccxtBaseURL = "http://localhost:3000"
 
 // SetBaseURL allows setting the base URL for ccxt
 func SetBaseURL(baseURL string) error {
-	if strings.HasSuffix(baseURL, "/") {
-		return fmt.Errorf("invalid format for baseURL, should not end with trailing '/': %s", baseURL)
-	}
-	ccxtBaseURL = baseURL
+	ccxtBaseURL = strings.TrimSuffix(baseURL, "/")
+	log.Printf("updated ccxtBaseURL to '%s'\n", ccxtBaseURL)
 	return nil
 }
 
@@ -106,10 +104,10 @@ func loadExchangeList() {
 	e := networking.JSONRequest(http.DefaultClient, "GET", ccxtBaseURL+pathExchanges, "", map[string]string{}, &output, "error")
 	if e != nil {
 		eMsg1 := strings.Contains(e.Error(), "could not execute http request")
-		eMsg2 := strings.Contains(e.Error(), "http://localhost:3000/exchanges: dial tcp")
+		eMsg2 := strings.Contains(e.Error(), ccxtBaseURL+"/exchanges: dial tcp")
 		eMsg3 := strings.Contains(e.Error(), "connection refused")
 		if eMsg1 && eMsg2 && eMsg3 {
-			log.Printf("ccxt-rest is not running on port 3000 so we cannot include those exchanges")
+			log.Printf("ccxt-rest is not running at %s so we cannot include those exchanges: %s", ccxtBaseURL, e.Error())
 		} else {
 			panic(fmt.Errorf("error getting list of supported exchanges by CCXT: %s", e))
 		}