separate pods for chainlink and postgres (#535)

client/client.go

@@ -231,7 +231,14 @@ func (m *K8sClient) EnumerateInstances(namespace string, selector string) error
 	if err != nil {
 		return err
 	}
+
 	for id, pod := range podList.Items {
+		// skip if already labeled with instance
+		existingLabels := pod.Labels
+		_, exists := existingLabels["instance"]
+		if exists {
+			continue
+		}
 		if err := m.AddPodLabel(namespace, pod, "instance", strconv.Itoa(id)); err != nil {
 			return err
 		}

e2e/common/test_common.go

@@ -9,6 +9,9 @@ import (
 
 	"github.com/go-resty/resty/v2"
 	"github.com/onsi/gomega"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/smartcontractkit/chainlink-env/chaos"
 	"github.com/smartcontractkit/chainlink-env/client"
 	"github.com/smartcontractkit/chainlink-env/config"
@@ -18,8 +21,6 @@ import (
 	"github.com/smartcontractkit/chainlink-env/pkg/helm/chainlink"
 	"github.com/smartcontractkit/chainlink-env/pkg/helm/ethereum"
 	"github.com/smartcontractkit/chainlink-env/presets"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -143,7 +144,7 @@ func TestConnectWithoutManifest(t *testing.T) {
 	if testEnv.Cfg.InsideK8s {
 		connection = client.RemoteConnection
 	}
-	url, err := testEnv.Fwd.FindPort("chainlink-0:0", "node", "access").As(connection, client.HTTP)
+	url, err := testEnv.Fwd.FindPort("chainlink-0:node-1", "node", "access").As(connection, client.HTTP)
 	require.NoError(t, err)
 	urlGeth, err := testEnv.Fwd.FindPort("geth:0", "geth-network", "http-rpc").As(connection, client.HTTP)
 	require.NoError(t, err)
@@ -262,12 +263,12 @@ func TestWithChaos(t *testing.T) {
 		},
 	}
 	testEnvConfig := GetTestEnvConfig(t)
-	cd, err := chainlink.NewDeployment(1, nil)
-	require.NoError(t, err)
+	cd := chainlink.New(0, nil)
+
 	e := environment.New(testEnvConfig).
 		AddHelm(ethereum.New(nil)).
-		AddHelmCharts(cd)
-	err = e.Run()
+		AddHelm(cd)
+	err := e.Run()
 	require.NoError(t, err)
 	if e.WillUseRemoteRunner() {
 		return
@@ -320,17 +321,18 @@ func TestEmptyEnvironmentStartup(t *testing.T) {
 func TestRolloutRestart(t *testing.T, statefulSet bool) {
 	t.Parallel()
 	testEnvConfig := GetTestEnvConfig(t)
-	cd, err := chainlink.NewDeployment(5, map[string]any{
+	cd := chainlink.New(0, map[string]any{
+		"replicas": 5,
 		"db": map[string]any{
 			"stateful": true,
 			"capacity": "1Gi",
 		},
 	})
-	require.NoError(t, err, "failed to create chainlink deployment")
+
 	e := environment.New(testEnvConfig).
 		AddHelm(ethereum.New(nil)).
-		AddHelmCharts(cd)
-	err = e.Run()
+		AddHelm(cd)
+	err := e.Run()
 	require.NoError(t, err)
 	if e.WillUseRemoteRunner() {
 		return
@@ -354,7 +356,7 @@ func TestRolloutRestart(t *testing.T, statefulSet bool) {
 func TestReplaceHelm(t *testing.T) {
 	t.Parallel()
 	testEnvConfig := GetTestEnvConfig(t)
-	cd, err := chainlink.NewDeployment(1, map[string]any{
+	cd := chainlink.New(0, map[string]any{
 		"chainlink": map[string]any{
 			"resources": map[string]any{
 				"requests": map[string]any{
@@ -366,9 +368,9 @@ func TestReplaceHelm(t *testing.T) {
 			},
 		},
 	})
-	require.NoError(t, err, "failed to create chainlink deployment")
-	e := environment.New(testEnvConfig).AddHelmCharts(cd)
-	err = e.Run()
+
+	e := environment.New(testEnvConfig).AddHelm(cd)
+	err := e.Run()
 	require.NoError(t, err)
 	if e.WillUseRemoteRunner() {
 		return
@@ -377,7 +379,7 @@ func TestReplaceHelm(t *testing.T) {
 		assert.NoError(t, e.Shutdown())
 	})
 	require.NoError(t, err)
-	cd, err = chainlink.NewDeployment(1, map[string]any{
+	cd = chainlink.New(1, map[string]any{
 		"chainlink": map[string]any{
 			"resources": map[string]any{
 				"requests": map[string]any{
@@ -391,7 +393,7 @@ func TestReplaceHelm(t *testing.T) {
 	})
 	require.NoError(t, err)
 	e, err = e.
-		ReplaceHelm("chainlink-0", cd[0])
+		ReplaceHelm("chainlink-0", cd)
 	require.NoError(t, err)
 	err = e.Run()
 	require.NoError(t, err)

environment/environment.go

@@ -378,13 +378,13 @@ func (m *Environment) ReplaceHelm(name string, chart ConnectedChart) (*Environme
 		Interface("Props", chart.GetProps()).
 		Interface("Values", chart.GetValues()).
 		Msg("Chart deployment values")
-	h := cdk8s.NewHelm(m.root, a.Str(name), &cdk8s.HelmProps{
+	h := cdk8s.NewHelm(m.root, a.Str(chart.GetName()), &cdk8s.HelmProps{
 		Chart: a.Str(chart.GetPath()),
 		HelmFlags: &[]*string{
 			a.Str("--namespace"),
 			a.Str(m.Cfg.Namespace),
 		},
-		ReleaseName: a.Str(name),
+		ReleaseName: a.Str(chart.GetName()),
 		Values:      chart.GetValues(),
 	})
 	addDefaultPodAnnotationsAndLabels(h, markNotSafeToEvict(m.Cfg.PreventPodEviction, nil), m.Cfg.PodLabels)

go.mod

@@ -94,11 +94,11 @@ require (
 	go.starlark.net v0.0.0-20220817180228-f738f5508c12 // indirect
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/term v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/term v0.12.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
 	golang.org/x/tools v0.6.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect

go.sum

@@ -455,8 +455,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -524,12 +524,12 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
-golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -538,8 +538,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

pkg/helm/chainlink/chainlink.go

@@ -62,8 +62,9 @@ func (m Chart) ExportData(e *environment.Environment) error {
 		return err
 	}
 	for i := 0; i < len(pods.Items); i++ {
-		internalConnection, err := e.Fwd.FindPort(fmt.Sprintf("%s:%d", m.Name, i), "node", "access").
-			As(client.RemoteConnection, client.HTTP)
+		chainlinkNode := fmt.Sprintf("%s:node-%d", m.Name, i+1)
+		pgNode := fmt.Sprintf("%s-postgres:node-%d", m.Name, i+1)
+		internalConnection, err := e.Fwd.FindPort(chainlinkNode, "node", "access").As(client.RemoteConnection, client.HTTP)
 		if err != nil {
 			return err
 		}
@@ -81,16 +82,15 @@ func (m Chart) ExportData(e *environment.Environment) error {
 		if e.Cfg.InsideK8s {
 			localConnection = internalConnection
 		} else {
-			localConnection, err = e.Fwd.FindPort(fmt.Sprintf("%s:%d", m.Name, i), "node", "access").
-				As(client.LocalConnection, client.HTTP)
+			localConnection, err = e.Fwd.FindPort(chainlinkNode, "node", "access").As(client.LocalConnection, client.HTTP)
 			if err != nil {
 				return err
 			}
 		}
 		e.URLs[NodesInternalURLsKey] = append(e.URLs[NodesInternalURLsKey], internalConnection)
 		e.URLs[NodesLocalURLsKey] = append(e.URLs[NodesLocalURLsKey], localConnection)
 
-		dbLocalConnection, err := e.Fwd.FindPort(fmt.Sprintf("%s:%d", m.Name, i), "chainlink-db", "postgres").
+		dbLocalConnection, err := e.Fwd.FindPort(pgNode, "chainlink-db", "postgres").
 			As(client.LocalConnection, client.HTTP)
 		if err != nil {
 			return err
@@ -123,9 +123,7 @@ func defaultProps() map[string]any {
 		chainlinkImage = fmt.Sprintf("%s/chainlink", internalRepo)
 		postgresImage = fmt.Sprintf("%s/postgres", internalRepo)
 	}
-	env := map[string]any{
-		"CL_DATABASE_URL": "postgresql://postgres:verylongdatabasepassword@0.0.0.0/chainlink?sslmode=disable",
-	}
+	env := make(map[string]string)
 	pyroscopeAuth := os.Getenv(config.EnvVarPyroscopeKey)
 	if pyroscopeAuth != "" {
 		env["CL_PYROSCOPE_AUTH_TOKEN"] = pyroscopeAuth
@@ -176,20 +174,6 @@ func New(index int, props map[string]any) environment.ConnectedChart {
 	return NewVersioned(index, "", props)
 }
 
-// NewDeployment ensures that each chainlink node gets its own helm chart, and should be preferred over New
-// Avoid using replicas when using NewDeployment
-func NewDeployment(deploymentCount int, props map[string]any) ([]environment.ConnectedChart, error) {
-	if props["replicas"] != nil && props["replicas"] != "1" {
-		return nil, fmt.Errorf("don't use replicas with NewDeployment")
-	}
-	charts := make([]environment.ConnectedChart, 0)
-	for i := 0; i < deploymentCount; i++ {
-		charts = append(charts, New(i, props))
-	}
-
-	return charts, nil
-}
-
 // NewVersioned enables you to select a specific helm chart version
 func NewVersioned(index int, helmVersion string, props map[string]any) environment.ConnectedChart {
 	dp := defaultProps()
@@ -200,6 +184,14 @@ func NewVersioned(index int, helmVersion string, props map[string]any) environme
 	}
 	if props["replicas"] != nil && props["replicas"] != "1" {
 		p.HasReplicas = true
+		replicas := props["replicas"].(int)
+		var nodesMap []map[string]string
+		for i := 0; i < replicas; i++ {
+			nodesMap = append(nodesMap, map[string]string{
+				"name": fmt.Sprintf("node-%d", i+1),
+			})
+		}
+		dp["nodes"] = nodesMap
 	}
 	return Chart{
 		Index:   index,

presets/presets.go

@@ -12,30 +12,27 @@ import (
 
 // EVMOneNode local development Chainlink deployment
 func EVMOneNode(config *environment.Config) (*environment.Environment, error) {
-	c, err := chainlink.NewDeployment(1, nil)
-	if err != nil {
-		return nil, err
-	}
+	c := chainlink.New(0, nil)
+
 	return environment.New(config).
 		AddHelm(mockservercfg.New(nil)).
 		AddHelm(mockserver.New(nil)).
 		AddHelm(ethereum.New(nil)).
-		AddHelmCharts(c), nil
+		AddHelm(c), nil
 }
 
 // EVMMinimalLocalBS local development Chainlink deployment,
 // 1 bootstrap + 4 oracles (minimal requirements for OCR) + Blockscout
 func EVMMinimalLocalBS(config *environment.Config) (*environment.Environment, error) {
-	c, err := chainlink.NewDeployment(5, nil)
-	if err != nil {
-		return nil, err
-	}
+	c := chainlink.New(0, map[string]any{
+		"replicas": 5,
+	})
 	return environment.New(config).
 		AddChart(blockscout.New(&blockscout.Props{})).
 		AddHelm(mockservercfg.New(nil)).
 		AddHelm(mockserver.New(nil)).
 		AddHelm(ethereum.New(nil)).
-		AddHelmCharts(c), nil
+		AddHelm(c), nil
 }
 
 // EVMMinimalLocal local development Chainlink deployment,
@@ -63,12 +60,11 @@ HTTPURL = 'http://geth-ethereum-geth:8544'
 
 [EVM.HeadTracker]
 HistoryDepth = 400`
-	c, err := chainlink.NewDeployment(5, map[string]interface{}{
-		"toml": clToml,
+	c := chainlink.New(0, map[string]interface{}{
+		"replicas": 5,
+		"toml":     clToml,
 	})
-	if err != nil {
-		return nil, err
-	}
+
 	return environment.New(config).
 		AddHelm(mockservercfg.New(nil)).
 		AddHelm(mockserver.New(nil)).
@@ -94,7 +90,7 @@ HistoryDepth = 400`
 				},
 			},
 		})).
-		AddHelmCharts(c), nil
+		AddHelm(c), nil
 }
 
 // EVMSoak deployment for a long running soak tests