feat: Add network command line option and embedded configs (#137)

* chore: Source control cmd/lilypad

* feat: Add embedded configs

* feat: Add network option

* chore: Remove hardcoded web3 and service defaults

* chore: Lint fix out-of-order import

* docs: Update controller address

* feat: Add GCP deprecation notice (#138)

feat: Add deprecation notice for GCP testnet

* test: Add dev environment

* build: Add network option to stack and Docker builds

.github/workflows/devnet_deploy.yml

@@ -61,6 +61,7 @@ jobs:
             --build-arg="expose_via=cloudflare" \
             --build-arg="cloudflare_token=${CLOUDFLARE_TOKEN_SOLVER}"\
             --build-arg="doppler_config=devnet" \
+            --build-arg="network=devnet" \
             .
           docker tag $ECR_REPOSITORY_SOLVER:latest $ECR_REGISTRY/$ECR_REPOSITORY_SOLVER:latest
           docker push $ECR_REGISTRY/$ECR_REPOSITORY_SOLVER:latest
@@ -122,6 +123,7 @@ jobs:
             -t $ECR_REPOSITORY_JOB_CREATOR \
             -f ./docker/job-creator/Dockerfile \
             --build-arg doppler_config=devnet \
+            --build-arg="network=devnet" \
             .
           docker tag $ECR_REPOSITORY_JOB_CREATOR:latest $ECR_REGISTRY/$ECR_REPOSITORY_JOB_CREATOR:latest
           docker push $ECR_REGISTRY/$ECR_REPOSITORY_JOB_CREATOR:latest

.github/workflows/testnet_deploy_services.yml

@@ -34,6 +34,7 @@ jobs:
             -t $ECR_REPOSITORY_SOLVER \
             -f ./docker/solver/Dockerfile \
             --build-arg="expose_via=cloudflare" \
+            --build-arg="network=testnet" \
             .
           docker tag $ECR_REPOSITORY_SOLVER:latest $ECR_REGISTRY/$ECR_REPOSITORY_SOLVER:latest
           docker push $ECR_REGISTRY/$ECR_REPOSITORY_SOLVER:latest
@@ -94,6 +95,7 @@ jobs:
           docker build \
             -t $ECR_REPOSITORY_JOB_CREATOR \
             -f ./docker/job-creator/Dockerfile \
+            --build-arg="network=testnet" \
             .
           docker tag $ECR_REPOSITORY_JOB_CREATOR:latest $ECR_REGISTRY/$ECR_REPOSITORY_JOB_CREATOR:latest
           docker push $ECR_REGISTRY/$ECR_REPOSITORY_JOB_CREATOR:latest

.gitignore

@@ -6,7 +6,9 @@ hardhat/deployments/localhost
 hardhat/deployments/geth
 hardhat/typechain-types
 node_modules/*
+# Ignore the binary executable
 lilypad
+!cmd/lilypad
 .env
 *.env
 .env.*

cmd/lilypad/jobcreator.go

@@ -17,7 +17,10 @@ func newJobCreatorCmd() *cobra.Command {
 		Long:    "Start the lilypad job creator service.",
 		Example: "",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			options, err := optionsfactory.ProcessOnChainJobCreatorOptions(options, args)
+			optionsfactory.CheckDeprecation(options.Offer.Services, options.Web3)
+
+			network, _ := cmd.Flags().GetString("network")
+			options, err := optionsfactory.ProcessOnChainJobCreatorOptions(options, args, network)
 			if err != nil {
 				return err
 			}

cmd/lilypad/mediator.go

@@ -19,7 +19,10 @@ func newMediatorCmd() *cobra.Command {
 		Long:    "Start the lilypad mediator service.",
 		Example: "",
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			options, err := optionsfactory.ProcessMediatorOptions(options)
+			optionsfactory.CheckDeprecation(options.Services, options.Web3)
+
+			network, _ := cmd.Flags().GetString("network")
+			options, err := optionsfactory.ProcessMediatorOptions(options, network)
 			if err != nil {
 				return err
 			}

cmd/lilypad/resource-provider.go

@@ -18,7 +18,10 @@ func newResourceProviderCmd() *cobra.Command {
 		Long:    "Start the lilypad resource-provider service.",
 		Example: "",
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			options, err := optionsfactory.ProcessResourceProviderOptions(options)
+			optionsfactory.CheckDeprecation(options.Offers.Services, options.Web3)
+
+			network, _ := cmd.Flags().GetString("network")
+			options, err := optionsfactory.ProcessResourceProviderOptions(options, network)
 			if err != nil {
 				return err
 			}

cmd/lilypad/root.go

@@ -21,6 +21,10 @@ func NewRootCmd() *cobra.Command {
 		Short: "Lilypad",
 		Long:  fmt.Sprintf("Lilypad: %s \nCommit: %s \n", VERSION, COMMIT_SHA),
 	}
+
+	var network string
+	RootCmd.PersistentFlags().StringVarP(&network, "network", "n", "testnet", "Sets a target network configuration")
+
 	RootCmd.AddCommand(newSolverCmd())
 	RootCmd.AddCommand(newResourceProviderCmd())
 	RootCmd.AddCommand(newRunCmd())

cmd/lilypad/run.go

@@ -8,12 +8,12 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/fatih/color"
 	"github.com/lilypad-tech/lilypad/pkg/data"
 	"github.com/lilypad-tech/lilypad/pkg/jobcreator"
 	optionsfactory "github.com/lilypad-tech/lilypad/pkg/options"
 	"github.com/lilypad-tech/lilypad/pkg/solver"
 	"github.com/lilypad-tech/lilypad/pkg/system"
-	"github.com/fatih/color"
 	"github.com/spf13/cobra"
 
 	"github.com/theckman/yacspin"
@@ -27,7 +27,10 @@ func newRunCmd() *cobra.Command {
 		Long:    "Run a job on the Lilypad network.",
 		Example: "run cowsay:v0.0.1 -i Message=moo",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			options, err := optionsfactory.ProcessJobCreatorOptions(options, args)
+			optionsfactory.CheckDeprecation(options.Offer.Services, options.Web3)
+
+			network, _ := cmd.Flags().GetString("network")
+			options, err := optionsfactory.ProcessJobCreatorOptions(options, args, network)
 			if err != nil {
 				return err
 			}

cmd/lilypad/solver.go

@@ -1,6 +1,7 @@
 package lilypad
 
 import (
+	"github.com/lilypad-tech/lilypad/pkg/data"
 	optionsfactory "github.com/lilypad-tech/lilypad/pkg/options"
 	"github.com/lilypad-tech/lilypad/pkg/solver"
 	memorystore "github.com/lilypad-tech/lilypad/pkg/solver/store/memory"
@@ -18,7 +19,10 @@ func newSolverCmd() *cobra.Command {
 		Long:    "Start the lilypad solver service.",
 		Example: "",
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			options, err := optionsfactory.ProcessSolverOptions(options)
+			optionsfactory.CheckDeprecation(data.ServiceConfig{}, options.Web3)
+
+			network, _ := cmd.Flags().GetString("network")
+			options, err := optionsfactory.ProcessSolverOptions(options, network)
 			if err != nil {
 				return err
 			}

docker/job-creator/Dockerfile

@@ -1,6 +1,8 @@
 FROM golang:latest as base
 WORKDIR /usr/src/app
 
+ARG network=dev
+
 COPY . .
 RUN go mod download && go mod verify
 RUN go build -v .
@@ -10,7 +12,7 @@ RUN (curl -Ls --tlsv1.2 --proto "=https" --retry 3 https://cli.doppler.com/insta
 
 RUN touch run
 RUN echo "#!/bin/bash" >> run
-RUN echo "doppler run -- lilypad jobcreator" >> run
+RUN echo "doppler run -- lilypad jobcreator --network $network" >> run
 RUN chmod +x run
 
 CMD ["/bin/bash", "./run"]

docker/resource-provider/Dockerfile

@@ -2,6 +2,7 @@ FROM golang:latest as base
 WORKDIR /usr/src/app
 
 ARG doppler_config=dev
+ARG network=dev
 ARG DOPPLER_TOKEN_BACALHAU
 ARG DOPPLER_TOKEN_RESOURCE_PROVIDER
 
@@ -21,7 +22,7 @@ RUN (curl -Ls --tlsv1.2 --proto "=https" --retry 3 https://cli.doppler.com/insta
 RUN touch run
 RUN echo "#!/bin/bash" >> run
 #RUN echo "doppler run --token=$DOPPLER_TOKEN_BACALHAU -p bacalhau -c $doppler_config -- ./stack bacalhau-serve &" >> run
-RUN echo "doppler run --token=$DOPPLER_TOKEN_RESOURCE_PROVIDER -p resource-provider -c $doppler_config -- lilypad resource-provider" >> run
+RUN echo "doppler run --token=$DOPPLER_TOKEN_RESOURCE_PROVIDER -p resource-provider -c $doppler_config -- lilypad resource-provider --network $network" >> run
 RUN chmod +x run
 
 CMD ["/bin/bash", "./run"]

docker/solver/Dockerfile

@@ -4,6 +4,7 @@ FROM golang:latest as base
 WORKDIR /usr/src/app
 
 ARG arch=amd64
+ARG network=dev
 
 COPY . .
 RUN go mod download && go mod verify
@@ -18,11 +19,11 @@ RUN echo "#!/bin/bash" >> run
 FROM base AS expose-cloudflare
 RUN curl -L --output cloudflared.deb https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-${arch}.deb
 RUN dpkg -i cloudflared.deb
-RUN echo "doppler run --command \"cloudflared tunnel run & lilypad solver\"" >> run
+RUN echo "doppler run --command \"cloudflared tunnel run & lilypad solver --network $network\"" >> run
 
 FROM base AS expose-local
 EXPOSE 8080
-RUN echo "doppler run -- lilypad solver" >> run
+RUN echo "doppler run -- lilypad solver --network $network" >> run
 
 FROM expose-$expose_via AS FINAL
 RUN chmod +x run

docs/smart-contract-jobs.md

@@ -8,7 +8,7 @@ It works in tandem with the `lilypad jobcreator` on-chain which will watch the o
 
 You will need to know the contract address for the on-chain job creator so we can submit transactions to it.
 
-The production controller address is `0x8e136587e3e5266d5244f6aa896E5CAf8E969946` and you can ask it for the address of the on-chain job creator `getJobCreatorAddress()`
+The production controller address is `0xF2fD1B9b262982F12446149A27d8901Ac68dcB59` and you can ask it for the address of the on-chain job creator `getJobCreatorAddress()`
 
 Running a job involves 2 phases:
 

go.mod

@@ -3,6 +3,7 @@ module github.com/lilypad-tech/lilypad
 go 1.20
 
 require (
+	github.com/BurntSushi/toml v0.3.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/ethereum/go-ethereum v1.13.4
 	github.com/fatih/color v1.15.0

go.sum

@@ -1,5 +1,6 @@
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=

pkg/data/types.go

@@ -83,8 +83,8 @@ const (
 // by the RP and JC - the solver will find an intersection
 // of these and attach them to the deal
 type ServiceConfig struct {
-	Solver   string   `json:"solver"`
-	Mediator []string `json:"mediator"`
+	Solver   string   `json:"solver" toml:"solver"`
+	Mediator []string `json:"mediator" toml:"mediator"`
 }
 
 // posted to the solver by a job creator

pkg/options/config.go

@@ -0,0 +1,37 @@
+package options
+
+import (
+	"embed"
+	"errors"
+	"fmt"
+
+	"github.com/BurntSushi/toml"
+	"github.com/lilypad-tech/lilypad/pkg/data"
+	"github.com/lilypad-tech/lilypad/pkg/web3"
+)
+
+//go:embed configs
+var fs embed.FS
+
+type Config struct {
+	Web3          web3.Web3Options   `toml:"web3"`
+	ServiceConfig data.ServiceConfig `toml:"services"`
+}
+
+// TODO(bgins) Check for user-defined config files
+func getConfig(network string) (*Config, error) {
+	var config Config
+
+	path := fmt.Sprintf("configs/%s.toml", network)
+	config_toml, err := fs.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("network %s does not exist", network)
+	}
+
+	_, err = toml.Decode(string(config_toml), &config)
+	if err != nil {
+		return nil, errors.New("unable to parse config file")
+	}
+
+	return &config, nil
+}

pkg/options/configs/dev.toml

@@ -0,0 +1,14 @@
+[services]
+solver = "0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC"
+mediator = ["0x90F79bf6EB2c4f870365E785982E1f101E93b906"]
+
+[web3]
+rpc_url = "ws://localhost:8546"
+chain_id = 421614
+controller_address = "0x4ed7c70F96B99c776995fB64377f0d4aB3B0e1C1"
+payments_address = "0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6"
+storage_address = "0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e"
+users_address = "0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82"
+token_address = "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853"
+mediation_address = "0x0B306BF915C4d645ff596e518fAf3F9669b97016"
+jobcreator_address = "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE"

pkg/options/configs/devnet.toml

@@ -0,0 +1,14 @@
+[services]
+solver = "0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC"
+mediator = ["0x90F79bf6EB2c4f870365E785982E1f101E93b906"]
+
+[web3]
+rpc_url = "wss://devnet-chain-ws.lilypad.tech"
+chain_id = 421614
+controller_address = "0x4ed7c70F96B99c776995fB64377f0d4aB3B0e1C1"
+payments_address = "0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6"
+storage_address = "0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e"
+users_address = "0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82"
+token_address = "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853"
+mediation_address = "0x0B306BF915C4d645ff596e518fAf3F9669b97016"
+jobcreator_address = "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE"

pkg/options/configs/testnet.toml

@@ -0,0 +1,14 @@
+[services]
+solver = "0xe05e1b71955da4934938a6b16f97e1db9de6b764"
+mediator = ["0x7B49d6ee530B0A538D26E344f3B02E79ACa96De2"]
+
+[web3]
+rpc_url = "wss://arbitrum-sepolia.infura.io/ws/v3/07472a48382e4679ae818aad0c21cf90"
+chain_id = 421614
+controller_address = "0xF2fD1B9b262982F12446149A27d8901Ac68dcB59"
+payments_address = "0x6f982192B4D225D67ED52A31308a602438882D6b"
+storage_address = "0xEB383625D7837aD49C1909f74d05a2db247CE0ca"
+users_address = "0x8FBAC8549fe9cb0630f7E30929C8256b9b67E68f"
+token_address = "0x0AabEbf21Cc4591475bb64e105267b3d55C7e0f1"
+mediation_address = "0x7B49d6ee530B0A538D26E344f3B02E79ACa96De2"
+jobcreator_address = "0x691ddE4710baFA7D0d7c02da32FeFB513CC870fA"

pkg/options/deprecation.go

@@ -0,0 +1,26 @@
+package options
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/lilypad-tech/lilypad/pkg/data"
+	"github.com/lilypad-tech/lilypad/pkg/web3"
+)
+
+// TODO(bgins) Remove once users have migrated away from old testnet
+func CheckDeprecation(serviceOptions data.ServiceConfig, web3Options web3.Web3Options) {
+	if web3Options.RpcURL == "ws://testnet.lilypad.tech:8546" {
+		web3Options.PrivateKey = "*****"
+		log.SetFlags(0)
+		log.Fatal(fmt.Sprintf(`This testnet has been deprecated. Please remove all environment variables for RPC URL, chain ID, and contract addresses.
+
+Your environment currently contains:
+
+%v
+%v
+The new testnet environment will be set by default, and you will only need to set your private key.
+`, spew.Sdump(web3Options), spew.Sdump(serviceOptions)))
+	}
+}

pkg/options/job-creator.go

@@ -84,7 +84,7 @@ func CheckJobCreatorOptions(options jobcreator.JobCreatorOptions) error {
 	return nil
 }
 
-func ProcessJobCreatorOptions(options jobcreator.JobCreatorOptions, args []string) (jobcreator.JobCreatorOptions, error) {
+func ProcessJobCreatorOptions(options jobcreator.JobCreatorOptions, args []string, network string) (jobcreator.JobCreatorOptions, error) {
 	name := ""
 	if len(args) == 1 {
 		name = args[0]
@@ -99,21 +99,35 @@ func ProcessJobCreatorOptions(options jobcreator.JobCreatorOptions, args []strin
 		return options, err
 	}
 	options.Offer.Module = moduleOptions
-	newWeb3Options, err := ProcessWeb3Options(options.Web3)
+
+	newWeb3Options, err := ProcessWeb3Options(options.Web3, network)
 	if err != nil {
 		return options, err
 	}
 	options.Web3 = newWeb3Options
+
+	newServicesOptions, err := ProcessServicesOptions(options.Offer.Services, network)
+	if err != nil {
+		return options, err
+	}
+	options.Offer.Services = newServicesOptions
+
 	return options, CheckJobCreatorOptions(options)
 }
 
-func ProcessOnChainJobCreatorOptions(options jobcreator.JobCreatorOptions, args []string) (jobcreator.JobCreatorOptions, error) {
-	newWeb3Options, err := ProcessWeb3Options(options.Web3)
+func ProcessOnChainJobCreatorOptions(options jobcreator.JobCreatorOptions, args []string, network string) (jobcreator.JobCreatorOptions, error) {
+	newWeb3Options, err := ProcessWeb3Options(options.Web3, network)
 	if err != nil {
 		return options, err
 	}
 	options.Web3 = newWeb3Options
 
+	newServicesOptions, err := ProcessServicesOptions(options.Offer.Services, network)
+	if err != nil {
+		return options, err
+	}
+	options.Offer.Services = newServicesOptions
+
 	err = CheckWeb3Options(options.Web3)
 	if err != nil {
 		return options, err