From cae048da87ce8d6505e5f4ad27fe140fae0c4914 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 13 Mar 2026 14:05:18 -0400 Subject: [PATCH 01/11] Convert legacy VRF tests to devenv --- .github/e2e-tests.yml | 19 -- .github/workflows/devenv-nightly.yml | 6 + devenv/environment.go | 3 + devenv/products/vrf/basic.toml | 20 ++ devenv/products/vrf/configuration.go | 290 +++++++++++++++++++++++++++ devenv/tests/vrf/smoke_test.go | 156 ++++++++++++++ integration-tests/smoke/vrf_test.go | 228 --------------------- 7 files changed, 475 insertions(+), 247 deletions(-) create mode 100644 devenv/products/vrf/basic.toml create mode 100644 devenv/products/vrf/configuration.go create mode 100644 devenv/tests/vrf/smoke_test.go delete mode 100644 integration-tests/smoke/vrf_test.go diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 415bd795b40..c29dac63c45 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -313,25 +313,6 @@ runner-test-matrix: - On Demand VRFV2 Performance Test test_go_project_path: integration-tests - - id: smoke/vrf_test.go:* - path: integration-tests/smoke/vrf_test.go - test_env_type: docker - runs_on: ubuntu-latest - triggers: - - PR E2E Core Tests - - Merge Queue E2E Core Tests - - Nightly E2E Tests - - Push E2E Core Tests - - Workflow Dispatch E2E Core Tests - test_cmd: | - gotestsum \ - --junitfile=/tmp/junit.xml \ - --jsonfile=/tmp/gotest.log \ - --format=github-actions \ - -- -v -run "TestVRFBasic|TestVRFJobReplacement" -timeout 30m -count=1 -parallel=2 github.com/smartcontractkit/chainlink/integration-tests/smoke - pyroscope_env: ci-smoke-vrf-evm-simulated - test_go_project_path: integration-tests - - id: smoke/vrfv2_test.go:* path: integration-tests/smoke/vrfv2_test.go test_env_type: docker diff --git a/.github/workflows/devenv-nightly.yml b/.github/workflows/devenv-nightly.yml index 5bb225a70b4..a293f6b52c7 100644 --- a/.github/workflows/devenv-nightly.yml +++ b/.github/workflows/devenv-nightly.yml @@ -73,6 +73,12 @@ jobs: runner: "ubuntu-latest" tests_dir: "flux" logs_archive_name: "flux" + - display_name: "Test VRF Smoke" + testcmd: "go test -v -timeout 10m -run TestVRFBasic\\|TestVRFJobReplacement" + envcmd: "cl u env.toml,products/vrf/basic.toml" + runner: "ubuntu-latest" + tests_dir: "vrf" + logs_archive_name: "vrf" # smoke tests, medium legacy products - display_name: "Automation Smoke Test, Registry 2.0" testcmd: "go test -v -timeout 30m -run TestRegistry_2_0" diff --git a/devenv/environment.go b/devenv/environment.go index 4ba5f8ec059..fe5eeaac41d 100644 --- a/devenv/environment.go +++ b/devenv/environment.go @@ -19,6 +19,7 @@ import ( "github.com/smartcontractkit/chainlink/devenv/products/flux" "github.com/smartcontractkit/chainlink/devenv/products/keepers" "github.com/smartcontractkit/chainlink/devenv/products/ocr2" + "github.com/smartcontractkit/chainlink/devenv/products/vrf" ) type ProductInfo struct { @@ -48,6 +49,8 @@ func newProduct(name string) (Product, error) { return automation.NewConfigurator(), nil case "keepers": return keepers.NewConfigurator(), nil + case "vrf": + return vrf.NewConfigurator(), nil default: return nil, fmt.Errorf("unknown product type: %s", name) } diff --git a/devenv/products/vrf/basic.toml b/devenv/products/vrf/basic.toml new file mode 100644 index 00000000000..dfeb343ed97 --- /dev/null +++ b/devenv/products/vrf/basic.toml @@ -0,0 +1,20 @@ +[[products]] +name = "vrf" +instances = 1 + +[[nodesets]] + name = "don" + nodes = 1 + override_mode = "each" + + [nodesets.db] + image = "postgres:15.0" + + [[nodesets.node_specs]] + [nodesets.node_specs.node] + image = "public.ecr.aws/chainlink/chainlink:2.30.0" + +[[vrf]] + [vrf.gas_settings] + fee_cap_multiplier = 2 + tip_cap_multiplier = 2 diff --git a/devenv/products/vrf/configuration.go b/devenv/products/vrf/configuration.go new file mode 100644 index 00000000000..f62a8f9b234 --- /dev/null +++ b/devenv/products/vrf/configuration.go @@ -0,0 +1,290 @@ +package vrf + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "math/big" + "os" + "strings" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/google/uuid" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + "github.com/smartcontractkit/chainlink-evm/gethwrappers/generated/blockhash_store" + "github.com/smartcontractkit/chainlink-evm/gethwrappers/generated/solidity_vrf_consumer_interface" + "github.com/smartcontractkit/chainlink-evm/gethwrappers/generated/solidity_vrf_coordinator_interface" + "github.com/smartcontractkit/chainlink-evm/gethwrappers/generated/solidity_vrf_wrapper" + "github.com/smartcontractkit/chainlink-evm/gethwrappers/shared/generated/initial/link_token" + "github.com/smartcontractkit/chainlink-testing-framework/framework/clclient" + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake" + nodeset "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" + + "github.com/smartcontractkit/chainlink/devenv/products" +) + +var L = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(zerolog.DebugLevel).With().Fields(map[string]any{"component": "vrf"}).Logger() + +type Configurator struct { + Config []*VRF `toml:"vrf"` +} + +type VRF struct { + GasSettings *products.GasSettings `toml:"gas_settings"` + Out *Out `toml:"out"` +} + +type Out struct { + ConsumerAddress string `toml:"consumer_address"` + CoordinatorAddress string `toml:"coordinator_address"` + KeyHash string `toml:"key_hash"` + JobID string `toml:"job_id"` + PublicKeyCompressed string `toml:"public_key_compressed"` + ExternalJobID string `toml:"external_job_id"` + ChainID string `toml:"chain_id"` +} + +func NewConfigurator() *Configurator { + return &Configurator{} +} + +func (m *Configurator) Load() error { + cfg, err := products.Load[Configurator]() + if err != nil { + return fmt.Errorf("failed to load product config: %w", err) + } + m.Config = cfg.Config + return nil +} + +func (m *Configurator) Store(path string, instanceIdx int) error { + if err := products.Store(".", m); err != nil { + return fmt.Errorf("failed to store product config: %w", err) + } + return nil +} + +func (m *Configurator) GenerateNodesConfig( + ctx context.Context, + fs *fake.Input, + bc []*blockchain.Input, + ns []*nodeset.Input, +) (string, error) { + return products.DefaultLegacyCLNodeConfig(bc) +} + +func (m *Configurator) GenerateNodesSecrets( + _ context.Context, + _ *fake.Input, + _ []*blockchain.Input, + _ []*nodeset.Input, +) (string, error) { + return "", nil +} + +func (m *Configurator) ConfigureJobsAndContracts( + ctx context.Context, + instanceIdx int, + fs *fake.Input, + bc []*blockchain.Input, + ns []*nodeset.Input, +) error { + L.Info().Msg("Connecting to CL nodes") + cls, err := clclient.New(ns[0].Out.CLNodes) + if err != nil { + return fmt.Errorf("failed to connect to CL nodes: %w", err) + } + + c, auth, rootAddr, err := products.ETHClient(ctx, bc[0].Out.Nodes[0].ExternalWSUrl, m.Config[0].GasSettings.FeeCapMultiplier, m.Config[0].GasSettings.TipCapMultiplier) + if err != nil { + return fmt.Errorf("failed to connect to blockchain: %w", err) + } + + // Deploy Link Token + linkAddr, linkTx, lt, err := link_token.DeployLinkToken(auth, c) + if err != nil { + return fmt.Errorf("could not deploy link token contract: %w", err) + } + _, err = bind.WaitDeployed(ctx, c, linkTx) + if err != nil { + return err + } + L.Info().Str("Address", linkAddr.Hex()).Msg("Deployed link token contract") + + tx, err := lt.GrantMintRole(auth, common.HexToAddress(rootAddr)) + if err != nil { + return fmt.Errorf("could not grant mint role: %w", err) + } + _, err = products.WaitMinedFast(ctx, c, tx.Hash()) + if err != nil { + return err + } + + // Deploy BlockHashStore + bhsAddr, bhsTx, _, err := blockhash_store.DeployBlockhashStore(auth, c) + if err != nil { + return fmt.Errorf("could not deploy blockhash store: %w", err) + } + _, err = bind.WaitDeployed(ctx, c, bhsTx) + if err != nil { + return err + } + L.Info().Str("Address", bhsAddr.Hex()).Msg("Deployed blockhash store") + + // Deploy VRF Coordinator + coordAddr, coordTx, coordinator, err := solidity_vrf_coordinator_interface.DeployVRFCoordinator(auth, c, linkAddr, bhsAddr) + if err != nil { + return fmt.Errorf("could not deploy VRF coordinator: %w", err) + } + _, err = bind.WaitDeployed(ctx, c, coordTx) + if err != nil { + return err + } + L.Info().Str("Address", coordAddr.Hex()).Msg("Deployed VRF coordinator") + + // Deploy VRF Consumer + consumerAddr, consumerTx, _, err := solidity_vrf_consumer_interface.DeployVRFConsumer(auth, c, coordAddr, linkAddr) + if err != nil { + return fmt.Errorf("could not deploy VRF consumer: %w", err) + } + _, err = bind.WaitDeployed(ctx, c, consumerTx) + if err != nil { + return err + } + L.Info().Str("Address", consumerAddr.Hex()).Msg("Deployed VRF consumer") + + // Deploy VRF v1 library + _, vrfLibTx, _, err := solidity_vrf_wrapper.DeployVRF(auth, c) + if err != nil { + return fmt.Errorf("could not deploy VRF library: %w", err) + } + _, err = bind.WaitDeployed(ctx, c, vrfLibTx) + if err != nil { + return err + } + L.Info().Msg("Deployed VRF v1 library") + + // Mint LINK to consumer + L.Info().Msgf("Minting LINK for consumer address: %s", consumerAddr) + tx, err = lt.Mint(auth, consumerAddr, big.NewInt(2e18)) + if err != nil { + return fmt.Errorf("could not mint link to consumer: %w", err) + } + _, err = products.WaitMinedFast(ctx, c, tx.Hash()) + if err != nil { + return err + } + + // Fund CL node transmitter + transmitters := make([]common.Address, 0) + for _, nc := range cls { + addr, cErr := nc.ReadPrimaryETHKey(bc[0].Out.ChainID) + if cErr != nil { + return cErr + } + transmitters = append(transmitters, common.HexToAddress(addr.Attributes.Address)) + } + pkey := products.NetworkPrivateKey() + if pkey == "" { + return errors.New("PRIVATE_KEY environment variable not set") + } + for _, addr := range transmitters { + if cErr := products.FundAddressEIP1559(ctx, c, pkey, addr.String(), 10); cErr != nil { + return cErr + } + } + + // Create VRF key on node + vrfKey, err := cls[0].MustCreateVRFKey() + if err != nil { + return fmt.Errorf("could not create VRF key: %w", err) + } + L.Info().Interface("Key", vrfKey).Msg("Created VRF proving key") + pubKeyCompressed := vrfKey.Data.ID + + // Create VRF job + jobUUID := uuid.New() + pipelineSpec := &clclient.VRFTxPipelineSpec{ + Address: coordAddr.String(), + } + observationSource, err := pipelineSpec.String() + if err != nil { + return fmt.Errorf("could not build VRF pipeline spec: %w", err) + } + + job, err := cls[0].MustCreateJob(&clclient.VRFJobSpec{ + Name: fmt.Sprintf("vrf-%s", jobUUID), + CoordinatorAddress: coordAddr.String(), + MinIncomingConfirmations: 1, + PublicKey: pubKeyCompressed, + ExternalJobID: jobUUID.String(), + EVMChainID: bc[0].ChainID, + ObservationSource: observationSource, + }) + if err != nil { + return fmt.Errorf("could not create VRF job: %w", err) + } + L.Info().Str("JobID", job.Data.ID).Msg("Created VRF job") + + // Register proving key on coordinator + oracleAddr := transmitters[0] + provingKey, err := encodeOnChainVRFProvingKey(vrfKey) + if err != nil { + return fmt.Errorf("could not encode VRF proving key: %w", err) + } + jobIDBytes := encodeOnChainExternalJobID(jobUUID) + + tx, err = coordinator.RegisterProvingKey(auth, big.NewInt(1), oracleAddr, provingKey, jobIDBytes) + if err != nil { + return fmt.Errorf("could not register proving key: %w", err) + } + _, err = products.WaitMinedFast(ctx, c, tx.Hash()) + if err != nil { + return err + } + L.Info().Msg("Registered VRF proving key on coordinator") + + // Compute key hash + keyHash, err := coordinator.HashOfKey(&bind.CallOpts{Context: ctx}, provingKey) + if err != nil { + return fmt.Errorf("could not compute key hash: %w", err) + } + L.Info().Str("KeyHash", hex.EncodeToString(keyHash[:])).Msg("Computed key hash") + + m.Config[0].Out = &Out{ + ConsumerAddress: consumerAddr.String(), + CoordinatorAddress: coordAddr.String(), + KeyHash: hex.EncodeToString(keyHash[:]), + JobID: job.Data.ID, + PublicKeyCompressed: pubKeyCompressed, + ExternalJobID: jobUUID.String(), + ChainID: bc[0].ChainID, + } + return nil +} + +func encodeOnChainVRFProvingKey(vrfKey *clclient.VRFKey) ([2]*big.Int, error) { + uncompressed := vrfKey.Data.Attributes.Uncompressed + provingKey := [2]*big.Int{} + var ok bool + provingKey[0], ok = new(big.Int).SetString(uncompressed[2:66], 16) + if !ok { + return [2]*big.Int{}, errors.New("cannot convert first half of VRF key to *big.Int") + } + provingKey[1], ok = new(big.Int).SetString(uncompressed[66:], 16) + if !ok { + return [2]*big.Int{}, errors.New("cannot convert second half of VRF key to *big.Int") + } + return provingKey, nil +} + +func encodeOnChainExternalJobID(jobID uuid.UUID) [32]byte { + var ji [32]byte + copy(ji[:], strings.Replace(jobID.String(), "-", "", 4)) + return ji +} diff --git a/devenv/tests/vrf/smoke_test.go b/devenv/tests/vrf/smoke_test.go new file mode 100644 index 00000000000..68822d038be --- /dev/null +++ b/devenv/tests/vrf/smoke_test.go @@ -0,0 +1,156 @@ +package vrf + +import ( + "encoding/hex" + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-evm/gethwrappers/generated/solidity_vrf_consumer_interface" + "github.com/smartcontractkit/chainlink-testing-framework/framework" + "github.com/smartcontractkit/chainlink-testing-framework/framework/clclient" + de "github.com/smartcontractkit/chainlink/devenv" + "github.com/smartcontractkit/chainlink/devenv/products" + "github.com/smartcontractkit/chainlink/devenv/products/vrf" +) + +func TestVRFBasic(t *testing.T) { + ctx := t.Context() + outputFile := "../../env-out.toml" + in, err := de.LoadOutput[de.Cfg](outputFile) + require.NoError(t, err) + productCfg, err := products.LoadOutput[vrf.Configurator](outputFile) + require.NoError(t, err) + t.Cleanup(func() { + _, cErr := framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name())) + require.NoError(t, cErr) + }) + + c, auth, _, err := products.ETHClient( + ctx, + in.Blockchains[0].Out.Nodes[0].ExternalWSUrl, + productCfg.Config[0].GasSettings.FeeCapMultiplier, + productCfg.Config[0].GasSettings.TipCapMultiplier, + ) + require.NoError(t, err) + + consumer, err := solidity_vrf_consumer_interface.NewVRFConsumer( + common.HexToAddress(productCfg.Config[0].Out.ConsumerAddress), c, + ) + require.NoError(t, err) + + keyHash := decodeKeyHash(t, productCfg.Config[0].Out.KeyHash) + + tx, err := consumer.TestRequestRandomness(auth, keyHash, big.NewInt(1)) + require.NoError(t, err) + _, err = products.WaitMinedFast(ctx, c, tx.Hash()) + require.NoError(t, err) + + cls, err := clclient.New(in.NodeSets[0].Out.CLNodes) + require.NoError(t, err) + + require.EventuallyWithT(t, func(ct *assert.CollectT) { + jobRuns, err := cls[0].MustReadRunsByJob(productCfg.Config[0].Out.JobID) + assert.NoError(ct, err) + assert.NotEmpty(ct, jobRuns.Data, "Expected the VRF job to have run at least once") + + out, err := consumer.RandomnessOutput(&bind.CallOpts{Context: ctx}) + assert.NoError(ct, err) + assert.NotZero(ct, out.Uint64(), "Expected the VRF job to produce a non-zero randomness output") + }, 2*time.Minute, 2*time.Second) +} + +func TestVRFJobReplacement(t *testing.T) { + ctx := t.Context() + outputFile := "../../env-out.toml" + in, err := de.LoadOutput[de.Cfg](outputFile) + require.NoError(t, err) + productCfg, err := products.LoadOutput[vrf.Configurator](outputFile) + require.NoError(t, err) + t.Cleanup(func() { + _, cErr := framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name())) + require.NoError(t, cErr) + }) + + c, auth, _, err := products.ETHClient( + ctx, + in.Blockchains[0].Out.Nodes[0].ExternalWSUrl, + productCfg.Config[0].GasSettings.FeeCapMultiplier, + productCfg.Config[0].GasSettings.TipCapMultiplier, + ) + require.NoError(t, err) + + consumer, err := solidity_vrf_consumer_interface.NewVRFConsumer( + common.HexToAddress(productCfg.Config[0].Out.ConsumerAddress), c, + ) + require.NoError(t, err) + + keyHash := decodeKeyHash(t, productCfg.Config[0].Out.KeyHash) + + cls, err := clclient.New(in.NodeSets[0].Out.CLNodes) + require.NoError(t, err) + + // First randomness request + tx, err := consumer.TestRequestRandomness(auth, keyHash, big.NewInt(1)) + require.NoError(t, err) + _, err = products.WaitMinedFast(ctx, c, tx.Hash()) + require.NoError(t, err) + + jobID := productCfg.Config[0].Out.JobID + require.EventuallyWithT(t, func(ct *assert.CollectT) { + jobRuns, err := cls[0].MustReadRunsByJob(jobID) + assert.NoError(ct, err) + assert.NotEmpty(ct, jobRuns.Data, "Expected the VRF job to have run at least once") + + out, err := consumer.RandomnessOutput(&bind.CallOpts{Context: ctx}) + assert.NoError(ct, err) + assert.NotZero(ct, out.Uint64(), "Expected the VRF job to produce a non-zero randomness output") + }, 2*time.Minute, 2*time.Second) + + // Delete the job and recreate it + err = cls[0].MustDeleteJob(jobID) + require.NoError(t, err) + + cfg := productCfg.Config[0].Out + pipelineSpec := &clclient.VRFTxPipelineSpec{ + Address: cfg.CoordinatorAddress, + } + observationSource, err := pipelineSpec.String() + require.NoError(t, err) + + newJob, err := cls[0].MustCreateJob(&clclient.VRFJobSpec{ + Name: fmt.Sprintf("vrf-%s", cfg.ExternalJobID), + CoordinatorAddress: cfg.CoordinatorAddress, + MinIncomingConfirmations: 1, + PublicKey: cfg.PublicKeyCompressed, + ExternalJobID: cfg.ExternalJobID, + EVMChainID: cfg.ChainID, + ObservationSource: observationSource, + }) + require.NoError(t, err) + + require.EventuallyWithT(t, func(ct *assert.CollectT) { + jobRuns, err := cls[0].MustReadRunsByJob(newJob.Data.ID) + assert.NoError(ct, err) + assert.NotEmpty(ct, jobRuns.Data, "Expected the recreated VRF job to have run at least once") + + out, err := consumer.RandomnessOutput(&bind.CallOpts{Context: ctx}) + assert.NoError(ct, err) + assert.NotZero(ct, out.Uint64(), "Expected the VRF job to produce a non-zero randomness output") + }, 2*time.Minute, 2*time.Second) +} + +func decodeKeyHash(t *testing.T, keyHashHex string) [32]byte { + t.Helper() + b, err := hex.DecodeString(keyHashHex) + require.NoError(t, err, "Failed to decode key hash hex") + var kh [32]byte + copy(kh[:], b) + return kh +} diff --git a/integration-tests/smoke/vrf_test.go b/integration-tests/smoke/vrf_test.go deleted file mode 100644 index 65aad730f29..00000000000 --- a/integration-tests/smoke/vrf_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package smoke - -import ( - "fmt" - "math/big" - "strconv" - "testing" - "time" - - "github.com/google/uuid" - "github.com/onsi/gomega" - "github.com/rs/zerolog" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" - "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" - "github.com/smartcontractkit/chainlink-testing-framework/seth" - - "github.com/smartcontractkit/chainlink/deployment/environment/nodeclient" - "github.com/smartcontractkit/chainlink/integration-tests/actions" - "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/vrfv1" - "github.com/smartcontractkit/chainlink/integration-tests/contracts" - ethcontracts "github.com/smartcontractkit/chainlink/integration-tests/contracts" - "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" - tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" - "github.com/smartcontractkit/chainlink/integration-tests/utils" -) - -func TestVRFBasic(t *testing.T) { - t.Parallel() - l := logging.GetTestLogger(t) - env, vrfContracts, sethClient := prepareVRFtestEnv(t, l) - - for _, n := range env.ClCluster.Nodes { - nodeKey, err := n.API.MustCreateVRFKey() - require.NoError(t, err, "Creating VRF key shouldn't fail") - l.Debug().Interface("Key JSON", nodeKey).Msg("Created proving key") - pubKeyCompressed := nodeKey.Data.ID - jobUUID := uuid.New() - os := &nodeclient.VRFTxPipelineSpec{ - Address: vrfContracts.Coordinator.Address(), - } - ost, err := os.String() - require.NoError(t, err, "Building observation source spec shouldn't fail") - job, err := n.API.MustCreateJob(&nodeclient.VRFJobSpec{ - Name: fmt.Sprintf("vrf-%s", jobUUID), - CoordinatorAddress: vrfContracts.Coordinator.Address(), - MinIncomingConfirmations: 1, - PublicKey: pubKeyCompressed, - ExternalJobID: jobUUID.String(), - EVMChainID: strconv.FormatInt(sethClient.ChainID, 10), - ObservationSource: ost, - }) - require.NoError(t, err, "Creating VRF Job shouldn't fail") - - oracleAddr, err := n.API.PrimaryEthAddress() - require.NoError(t, err, "Getting primary ETH address of chainlink node shouldn't fail") - provingKey, err := actions.EncodeOnChainVRFProvingKey(*nodeKey) - require.NoError(t, err, "Encoding on-chain VRF Proving key shouldn't fail") - err = vrfContracts.Coordinator.RegisterProvingKey( - big.NewInt(1), - oracleAddr, - provingKey, - actions.EncodeOnChainExternalJobID(jobUUID), - ) - require.NoError(t, err, "Registering the on-chain VRF Proving key shouldn't fail") - encodedProvingKeys := make([][2]*big.Int, 0) - encodedProvingKeys = append(encodedProvingKeys, provingKey) - - requestHash, err := vrfContracts.Coordinator.HashOfKey(testcontext.Get(t), encodedProvingKeys[0]) - require.NoError(t, err, "Getting Hash of encoded proving keys shouldn't fail") - err = vrfContracts.Consumer.RequestRandomness(requestHash, big.NewInt(1)) - require.NoError(t, err, "Requesting randomness shouldn't fail") - - gom := gomega.NewGomegaWithT(t) - timeout := time.Minute * 2 - gom.Eventually(func(g gomega.Gomega) { - jobRuns, err := env.ClCluster.Nodes[0].API.MustReadRunsByJob(job.Data.ID) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Job execution shouldn't fail") - - out, err := vrfContracts.Consumer.RandomnessOutput(testcontext.Get(t)) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Getting the randomness output of the consumer shouldn't fail") - // Checks that the job has actually run - g.Expect(jobRuns.Data).ShouldNot(gomega.BeEmpty(), - fmt.Sprintf("Expected the VRF job to run once or more after %s", timeout)) - - // TODO: This is an imperfect check, given it's a random number, it CAN be 0, but chances are unlikely. - // So we're just checking that the answer has changed to something other than the default (0) - // There's a better formula to ensure that VRF response is as expected, detailed under Technical Walkthrough. - // https://bl.chain.link/chainlink-vrf-on-chain-verifiable-randomness/ - g.Expect(out.Uint64()).ShouldNot(gomega.BeNumerically("==", 0), "Expected the VRF job give an answer other than 0") - l.Debug().Uint64("Output", out.Uint64()).Msg("Randomness fulfilled") - }, timeout, "1s").Should(gomega.Succeed()) - } -} - -func TestVRFJobReplacement(t *testing.T) { - t.Parallel() - l := logging.GetTestLogger(t) - env, contracts, sethClient := prepareVRFtestEnv(t, l) - - for _, n := range env.ClCluster.Nodes { - nodeKey, err := n.API.MustCreateVRFKey() - require.NoError(t, err, "Creating VRF key shouldn't fail") - l.Debug().Interface("Key JSON", nodeKey).Msg("Created proving key") - pubKeyCompressed := nodeKey.Data.ID - jobUUID := uuid.New() - os := &nodeclient.VRFTxPipelineSpec{ - Address: contracts.Coordinator.Address(), - } - ost, err := os.String() - require.NoError(t, err, "Building observation source spec shouldn't fail") - job, err := n.API.MustCreateJob(&nodeclient.VRFJobSpec{ - Name: fmt.Sprintf("vrf-%s", jobUUID), - CoordinatorAddress: contracts.Coordinator.Address(), - MinIncomingConfirmations: 1, - PublicKey: pubKeyCompressed, - ExternalJobID: jobUUID.String(), - EVMChainID: strconv.FormatInt(sethClient.ChainID, 10), - ObservationSource: ost, - }) - require.NoError(t, err, "Creating VRF Job shouldn't fail") - - oracleAddr, err := n.API.PrimaryEthAddress() - require.NoError(t, err, "Getting primary ETH address of chainlink node shouldn't fail") - provingKey, err := actions.EncodeOnChainVRFProvingKey(*nodeKey) - require.NoError(t, err, "Encoding on-chain VRF Proving key shouldn't fail") - err = contracts.Coordinator.RegisterProvingKey( - big.NewInt(1), - oracleAddr, - provingKey, - actions.EncodeOnChainExternalJobID(jobUUID), - ) - require.NoError(t, err, "Registering the on-chain VRF Proving key shouldn't fail") - encodedProvingKeys := make([][2]*big.Int, 0) - encodedProvingKeys = append(encodedProvingKeys, provingKey) - - requestHash, err := contracts.Coordinator.HashOfKey(testcontext.Get(t), encodedProvingKeys[0]) - require.NoError(t, err, "Getting Hash of encoded proving keys shouldn't fail") - err = contracts.Consumer.RequestRandomness(requestHash, big.NewInt(1)) - require.NoError(t, err, "Requesting randomness shouldn't fail") - - gom := gomega.NewGomegaWithT(t) - timeout := time.Minute * 2 - gom.Eventually(func(g gomega.Gomega) { - jobRuns, err := env.ClCluster.Nodes[0].API.MustReadRunsByJob(job.Data.ID) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Job execution shouldn't fail") - - out, err := contracts.Consumer.RandomnessOutput(testcontext.Get(t)) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Getting the randomness output of the consumer shouldn't fail") - // Checks that the job has actually run - g.Expect(jobRuns.Data).ShouldNot(gomega.BeEmpty(), - fmt.Sprintf("Expected the VRF job to run once or more after %s", timeout)) - - g.Expect(out.Uint64()).ShouldNot(gomega.BeNumerically("==", 0), "Expected the VRF job give an answer other than 0") - l.Debug().Uint64("Output", out.Uint64()).Msg("Randomness fulfilled") - }, timeout, "1s").Should(gomega.Succeed()) - - err = n.API.MustDeleteJob(job.Data.ID) - require.NoError(t, err) - - job, err = n.API.MustCreateJob(&nodeclient.VRFJobSpec{ - Name: fmt.Sprintf("vrf-%s", jobUUID), - CoordinatorAddress: contracts.Coordinator.Address(), - MinIncomingConfirmations: 1, - PublicKey: pubKeyCompressed, - ExternalJobID: jobUUID.String(), - EVMChainID: strconv.FormatInt(sethClient.ChainID, 10), - ObservationSource: ost, - }) - require.NoError(t, err, "Recreating VRF Job shouldn't fail") - gom.Eventually(func(g gomega.Gomega) { - jobRuns, err := env.ClCluster.Nodes[0].API.MustReadRunsByJob(job.Data.ID) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Job execution shouldn't fail") - - out, err := contracts.Consumer.RandomnessOutput(testcontext.Get(t)) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Getting the randomness output of the consumer shouldn't fail") - // Checks that the job has actually run - g.Expect(jobRuns.Data).ShouldNot(gomega.BeEmpty(), - fmt.Sprintf("Expected the VRF job to run once or more after %s", timeout)) - g.Expect(out.Uint64()).ShouldNot(gomega.BeNumerically("==", 0), "Expected the VRF job give an answer other than 0") - l.Debug().Uint64("Output", out.Uint64()).Msg("Randomness fulfilled") - }, timeout, "1s").Should(gomega.Succeed()) - } -} - -func prepareVRFtestEnv(t *testing.T, l zerolog.Logger) (*test_env.CLClusterTestEnv, *vrfv1.Contracts, *seth.Client) { - config, err := tc.GetChainAndTestTypeSpecificConfig("Smoke", tc.VRF) - require.NoError(t, err, "Error getting config") - - privateNetwork, err := actions.EthereumNetworkConfigFromConfig(l, &config) - require.NoError(t, err, "Error building ethereum network config") - - env, err := test_env.NewCLTestEnvBuilder(). - WithTestInstance(t). - WithTestConfig(&config). - WithPrivateEthereumNetwork(privateNetwork.EthereumNetworkConfig). - WithCLNodes(1). - WithStandardCleanup(). - Build() - require.NoError(t, err) - - evmNetwork, err := env.GetFirstEvmNetwork() - require.NoError(t, err, "Error getting first evm network") - - sethClient, err := utils.TestAwareSethClient(t, config, evmNetwork) - require.NoError(t, err, "Error getting seth client") - - err = actions.FundChainlinkNodesFromRootAddress(l, sethClient, contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(env.ClCluster.NodeAPIs()), big.NewFloat(*config.Common.ChainlinkNodeFunding)) - require.NoError(t, err, "Failed to fund the nodes") - - t.Cleanup(func() { - // ignore error, we will see failures in the logs anyway - _ = actions.ReturnFundsFromNodes(l, sethClient, contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(env.ClCluster.NodeAPIs())) - }) - - lt, err := ethcontracts.DeployLinkTokenContract(l, sethClient) - require.NoError(t, err, "Deploying Link Token Contract shouldn't fail") - vrfContracts, err := vrfv1.DeployVRFContracts(sethClient, lt.Address()) - require.NoError(t, err, "Deploying VRF Contracts shouldn't fail") - - err = lt.Transfer(vrfContracts.Consumer.Address(), big.NewInt(2e18)) - require.NoError(t, err, "Funding consumer contract shouldn't fail") - _, err = ethcontracts.DeployVRFv1Contract(sethClient) - require.NoError(t, err, "Deploying VRF contract shouldn't fail") - - return env, vrfContracts, sethClient -} From bde539fba1dc4056883a2717b25ac2edb5f4a07a Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 13 Mar 2026 14:44:57 -0400 Subject: [PATCH 02/11] Docs --- devenv/README.md | 118 +++++++++++--- devenv/design.md | 359 +++++++++++++++++++++++++++++++++++++++++ devenv/tests/README.md | 2 +- 3 files changed, 455 insertions(+), 24 deletions(-) create mode 100644 devenv/design.md diff --git a/devenv/README.md b/devenv/README.md index a84ca942a48..5d82c65b1c2 100644 --- a/devenv/README.md +++ b/devenv/README.md @@ -1,47 +1,119 @@ -
- # Chainlink Developer Environment -`NodeSet` + `Anvil` + `Fake Server` + `OCR2 Product Orchestration` +A self-contained Docker-based environment for running Chainlink system-level tests. Spins up an EVM chain (Anvil/Geth), a fake external adapter server, and Chainlink nodes, then deploys product-specific contracts and jobs. -
+For detailed architecture documentation, see [design.md](design.md). -## Run the Environment +## Quickstart -We are using [Justfile](https://github.com/casey/just?tab=readme-ov-file#cross-platform) +### Prerequisites -Change directory to `devenv` then +- **Docker** -- must be running +- **Go** -- version specified in `go.mod` +- **Just** -- task runner ([install guide](https://github.com/casey/just?tab=readme-ov-file#cross-platform)) ```bash -brew install just # click the link above if you are not on OS X -just build-fakes && just cli && cl sh +brew install just # macOS; see link above for other platforms ``` -Then start the observability stack and run the soak test, perform actions in `ccv sh` console +### One-Time Setup + +From the `devenv/` directory: ```bash -up -obs up -f # Click OCR2 Dashboard link to open in another tab -test load # Run the load test, you'll see OCR2 rounds stats +just build-fakes # Build the mock external adapter Docker image +just cli # Install the `cl` CLI tool ``` -## Run with custom CL image +### Running a Test + +Tests use a two-terminal workflow. Pick a product from the reference table below and run: -Use `up env.toml,products/ocr2/basic.toml,env-cl-rebuild.toml` to rebuild custom CL image from your local `chainlink` repository. +**Terminal 1** -- start the environment (from `devenv/`): -## Updating Fakes +```bash +cl u env.toml,products//basic.toml +``` + +**Terminal 2** -- run the test (from `devenv/tests//`): + +```bash +go test -v -run +``` -Fake represent a controlled External Adapter that returns feed values. +## Observability ```bash -just build-fakes # use SDLC registry -just push-fakes # use SDLC registry +cl obs up # Loki + Prometheus + Grafana +cl obs up -f # Full stack (adds Pyroscope, cadvisor, postgres exporter) +cl obs down # Tear down ``` -## Adding Products +### Dashboards + +- [All Logs](http://localhost:3000/explore?schemaVersion=1&panes=%7B%22axv%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bjob%3D%5C%22ctf%5C%22%7D%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22,%22direction%22:%22backward%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D,%22compact%22:false%7D%7D&orgId=1) +- [CL Node Errors](http://localhost:3000/d/a7de535b-3e0f-4066-bed7-d505b6ec9ef1/cl-node-errors?orgId=1&refresh=5s&from=now-15m&to=now&timezone=browser) +- [Load Testing](http://localhost:3000/d/WASPLoadTests/wasp-load-test?orgId=1&from=now-30m&to=now&timezone=browser&var-go_test_name=$__all&var-gen_name=$__all&var-branch=$__all&var-commit=$__all&var-call_group=$__all&refresh=5s) + +### Product Reference + +Each row maps to a CI matrix entry in [devenv-nightly.yml](../.github/workflows/devenv-nightly.yml). + +| Product | envcmd (from `devenv/`) | testcmd (from `devenv/tests//`) | tests_dir | +| -------------- | ------------------------------------------------------------------------------ | ------------------------------------------------------------------ | --------------- | +| Cron | `cl u env.toml,products/cron/basic.toml` | `go test -v -run TestSmoke` | `cron` | +| Direct Request | `cl u env.toml,products/directrequest/basic.toml` | `go test -v -run TestSmoke` | `directrequest` | +| Flux | `cl u env.toml,products/flux/basic.toml` | `go test -v -run TestSmoke` | `flux` | +| VRF | `cl u env.toml,products/vrf/basic.toml` | `go test -v -timeout 10m -run TestVRFBasic\|TestVRFJobReplacement` | `vrf` | +| Automation 2.0 | `cl u env.toml,products/automation/basic.toml` | `go test -v -timeout 30m -run TestRegistry_2_0` | `automation` | +| Automation 2.1 | `cl u env.toml,products/automation/basic.toml` | `go test -v -timeout 30m -run TestRegistry_2_1` | `automation` | +| Keepers | `cl u env.toml,products/keepers/basic.toml` | `go test -v -timeout 1h -run TestKeeperBasic` | `keepers` | +| OCR2 Smoke | `cl u env.toml,products/ocr2/basic.toml` | `go test -v -run TestSmoke` | `ocr2` | +| OCR2 Soak | `cl u env.toml,products/ocr2/basic.toml,products/ocr2/soak.toml; cl obs up -f` | `go test -v -timeout 4h -run TestOCR2Soak/clean` | `ocr2` | + +## Interactive Shell + +Instead of running CLI commands directly, you can use the interactive shell with autocomplete: + +```bash +cl sh +``` -To extend the environment all you need to do is to implement the [interface](interface.go) and add a switch clause in [environment](environment.go) +Then inside the shell: + +```sh +up # start with default OCR2 config +up env.toml,products/vrf/basic.toml # start with a specific product +obs up -f # start full observability stack +test ocr2 TestSmoke # run a test +down # tear down everything +``` + +## Run with Custom CL Image + +Use `env-cl-rebuild.toml` to build a Chainlink image from your local repository: + +```bash +cl u env.toml,products/ocr2/basic.toml,env-cl-rebuild.toml +``` + +Or override the image via environment variable: + +```bash +CHAINLINK_IMAGE=my-registry/chainlink:dev cl u env.toml,products/ocr2/basic.toml +``` + +## Contributing + +### Updating Fakes + +Fakes are mock external adapters that return controlled feed values. See [design.md](design.md#fakes-mock-external-adapters) for details. + +```bash +just build-fakes # Build locally +just push-fakes # Push to ECR +``` -## Running Tests +### Adding Products -Follow the [guide](./tests/README.md) +Implement the [Product interface](interface.go) and add a switch clause in [environment.go](environment.go). See [design.md](design.md#the-product-interface) for the full lifecycle. diff --git a/devenv/design.md b/devenv/design.md new file mode 100644 index 00000000000..3ea386c29b9 --- /dev/null +++ b/devenv/design.md @@ -0,0 +1,359 @@ +# Devenv Architecture + +## Overview + +`devenv` is a self-contained Go module (`github.com/smartcontractkit/chainlink/devenv`) that provides a Docker-based development and testing environment for Chainlink products. It orchestrates local blockchain networks, Chainlink nodes, mock external adapters, and product-specific contract deployments. + +Key design principles: + +- **Dependency isolation** -- devenv does NOT import `github.com/smartcontractkit/chainlink/v2` or any of its child packages. This keeps the test environment decoupled from the core node codebase. +- **TOML-driven configuration** -- all infrastructure and product settings are declared in composable TOML files that merge left-to-right. +- **Two-phase testing** -- environment setup (CLI) and test execution (`go test`) are separate processes, connected by a shared `env-out.toml` output file. +- **Product abstraction** -- each Chainlink product implements a common `Product` interface, making it straightforward to add new products. + +The module depends on the [Chainlink Testing Framework (CTF)](https://github.com/smartcontractkit/chainlink-testing-framework) for Docker orchestration, CL node HTTP clients, and observability tooling. + +## High-Level Architecture + +```mermaid +flowchart TD + subgraph config [TOML Configuration] + envToml["env.toml\n(infra: chain, nodes, fakes)"] + productToml["products/<name>/basic.toml\n(product settings)"] + overrideToml["env-geth.toml etc.\n(optional overrides)"] + end + + subgraph cli [CLI] + clUp["cl up <configs>"] + end + + subgraph envSetup ["NewEnvironment()"] + loadConfig["Load and merge\nTOML configs"] + startInfra["Start infrastructure\n(Anvil, Fake Server)"] + genNodeConfig["Products generate\nCL node config"] + startNodes["Start CL node set\n(shared DB)"] + storeInfra["Store infra output"] + deployProducts["ConfigureJobsAndContracts()\nper product instance"] + storeProducts["Store product output"] + end + + subgraph output [Output] + envOut["env-out.toml\n(addresses, URLs, job IDs)"] + end + + subgraph testPhase [Test Execution] + goTest["go test -v -run TestName"] + loadOutput["Load env-out.toml"] + assertions["Assert on-chain state\nvia gethwrappers +\nCL node API via clclient"] + end + + envToml --> clUp + productToml --> clUp + overrideToml --> clUp + clUp --> loadConfig + loadConfig --> startInfra + startInfra --> genNodeConfig + genNodeConfig --> startNodes + startNodes --> storeInfra + storeInfra --> deployProducts + deployProducts --> storeProducts + storeProducts --> envOut + envOut --> loadOutput + goTest --> loadOutput + loadOutput --> assertions +``` + +## Configuration System + +The configuration system uses composable TOML files merged via the `CTF_CONFIGS` environment variable. + +### Merge Semantics + +When `cl up env.toml,products/ocr2/basic.toml` runs, it sets `CTF_CONFIGS=env.toml,products/ocr2/basic.toml`. The `Load[T]()` function reads each file left-to-right, decoding into the same struct. Later files override earlier keys while preserving keys they do not mention. + +```mermaid +flowchart LR + A["env.toml\n(blockchains, fake_server,\nnodesets)"] -->|merge| C["Merged Config"] + B["products/ocr2/basic.toml\n(products, ocr2 settings,\nnodeset overrides)"] -->|merge| C + C --> D["NewEnvironment()\nuses merged config"] + D --> E["env-out.toml\n(infra + product outputs)"] +``` + +### Config Layers + +| Layer | File | Purpose | +| ------------------- | ---------------------------- | -------------------------------------------------------------------------- | +| Base infrastructure | `env.toml` | Chain type/ID, fake server image, node count and images | +| Product config | `products//basic.toml` | Product name, instances, product-specific settings, node count override | +| Chain override | `env-geth.toml` | Switch from Anvil to Geth | +| Image override | `env-cl-rebuild.toml` | Build CL image from local Dockerfile | +| Runtime output | `env-out.toml` | Generated after `cl up` -- contains deployed addresses, node URLs, job IDs | + +### Root Config Struct + +The root configuration type (`Cfg` in `environment.go`) defines the top-level TOML schema: + +```go +type Cfg struct { + Products []*ProductInfo `toml:"products"` + Blockchains []*blockchain.Input `toml:"blockchains"` + FakeServer *fake.Input `toml:"fake_server"` + NodeSets []*ns.Input `toml:"nodesets"` + JD *jd.Input `toml:"jd"` +} +``` + +Each product configurator has its own struct that gets decoded from the same TOML files (e.g., `[[ocr2]]` sections are decoded by the OCR2 `Configurator`). + +## The Product Interface + +Every product in devenv implements this interface from `interface.go`: + +```go +type Product interface { + Load() error + Store(path string, instanceIdx int) error + GenerateNodesSecrets(ctx, fs, bc, ns) (string, error) + GenerateNodesConfig(ctx, fs, bc, ns) (string, error) + ConfigureJobsAndContracts(ctx, instanceIdx, fs, bc, ns) error +} +``` + +### Product Lifecycle + +```mermaid +sequenceDiagram + participant CLI as cl up + participant Env as NewEnvironment + participant Product as Product Configurator + participant Infra as Docker Infrastructure + participant Chain as Blockchain + + CLI->>Env: Start with merged TOML config + Env->>Infra: Create blockchain network (Anvil/Geth) + Env->>Infra: Create fake data provider + + loop For each product in config + Env->>Product: Load() + Product-->>Env: Product config loaded from TOML + Env->>Product: GenerateNodesConfig() + Product-->>Env: CL node TOML overrides + Env->>Product: GenerateNodesSecrets() + Product-->>Env: CL node secrets overrides + end + + Note over Env,Infra: Merge all product config overrides into node specs + Env->>Infra: Start CL node set (shared DB) + Env->>Env: Store infrastructure output + + loop For each product, for each instance + Env->>Product: ConfigureJobsAndContracts() + Product->>Chain: Deploy contracts (LINK, product contracts) + Product->>Infra: Fund CL nodes, create keys + Product->>Infra: Create jobs on CL nodes + Product->>Chain: Register on-chain config + Env->>Product: Store() + Product-->>Env: Write output (addresses, job IDs) + end +``` + +### Registered Products + +| Name | TOML key | Config dir | Nodes | Contracts deployed | +| -------------- | ---------------- | ------------------------- | ----- | -------------------------------------------------- | +| Cron | `cron` | `products/cron/` | 1 | None (bridge + cron job only) | +| Direct Request | `direct_request` | `products/directrequest/` | 1 | LINK, Oracle, TestAPIConsumer | +| Flux Monitor | `flux` | `products/flux/` | 5 | LINK, FluxAggregator | +| OCR2 | `ocr2` | `products/ocr2/` | 5 | LINK, OCR2Aggregator | +| Automation | `automation` | `products/automation/` | 5 | LINK, Registry (2.0-2.3), Registrar, Upkeeps | +| Keepers | `keepers` | `products/keepers/` | 5 | LINK, KeeperRegistry (1.1-1.3), Registrar, Upkeeps | +| VRF | `vrf` | `products/vrf/` | 1 | LINK, BlockHashStore, VRFCoordinator, VRFConsumer | + +### Adding a New Product + +1. Create `products//configuration.go` implementing the `Product` interface +2. Create `products//basic.toml` with default config +3. Add a `case ""` in `newProduct()` in `environment.go` +4. Create `tests//smoke_test.go` that reads `env-out.toml` and asserts behavior +5. Add a matrix entry in `.github/workflows/devenv-nightly.yml` + +## The `cl` CLI + +The CLI (`cmd/cl/`) is a Cobra-based tool that drives environment lifecycle. + +### Commands + +| Command | Alias | Description | +| --------------------------- | ------ | ------------------------------------------------------------------------------------ | +| `cl up [configs]` | `cl u` | Spin up environment from TOML configs (default: `env.toml,products/ocr2/basic.toml`) | +| `cl down` | `cl d` | Tear down all Docker containers | +| `cl restart [configs]` | `cl r` | Tear down then recreate | +| `cl test ` | | Run `go test` in `tests/` with `-run ` | +| `cl obs up [-f]` | | Start observability stack (Loki/Prometheus/Grafana; `-f` for full) | +| `cl obs down` | | Stop observability stack | +| `cl bs up` | | Start Blockscout block explorer | +| `cl bs down` | | Stop Blockscout | +| `cl shell` / `cl sh` | | Interactive shell with autocomplete | + +### How `cl up` Works + +1. Sets `CTF_CONFIGS` env var from the argument (or defaults to `env.toml,products/ocr2/basic.toml`) +2. Sets `TESTCONTAINERS_RYUK_DISABLED=true` to prevent container cleanup on CLI exit +3. Calls `devenv.NewEnvironment(ctx)` with a 7-minute timeout +4. `NewEnvironment` loads config, starts infra, runs product configurators, writes `env-out.toml` + +### Interactive Shell + +`cl sh` starts a REPL with tab-completion for commands and config file paths. It executes commands by invoking the same Cobra command tree in-process. + +## Fakes (Mock External Adapters) + +Fakes are a lightweight HTTP service that replaces real external adapters and data feeds during testing. The fake server runs as a Docker container on port 9111. + +### Why Fakes Exist + +Chainlink nodes need external data sources (external adapters, price feeds, Mercury endpoints). Instead of depending on real services, fakes provide deterministic, controllable responses that make tests reliable and fast. + +### Routes by Product + +| Product | Route | Behavior | +| -------------- | ------------------------------- | ----------------------------------------------------------- | +| Cron | `POST /cron_response` | Returns `{"data": {"result": 200}}` | +| Direct Request | `POST /direct_request_response` | Returns `{"data": {"result": 200}}` | +| OCR2 | `POST /ea` | Returns current EA value (default 200) | +| OCR2 | `POST /juelsPerFeeCoinSource` | Returns JUELS/LINK ratio | +| OCR2 | `POST /trigger_deviation` | Changes the EA return value (query param `?result=`) | +| Automation | `POST /api/v1/reports/bulk` | Returns mock Mercury/DataStreams reports | +| Automation | `GET /client` | Returns mock Mercury client config | + +### Building and Using Fakes + +```bash +just build-fakes # Build image locally as chainlink-fakes:latest +just push-fakes # Build for linux/amd64 and push to ECR +``` + +In CI, the `FAKE_SERVER_IMAGE` environment variable overrides the image used in `env.toml`. + +## Test Architecture + +Tests follow a two-phase pattern where environment setup and test execution are independent processes. + +```mermaid +sequenceDiagram + participant Dev as Developer + participant CLI as cl up (Terminal 1) + participant Docker as Docker Containers + participant Test as go test (Terminal 2) + participant EnvOut as env-out.toml + + Dev->>CLI: cl u env.toml,products/vrf/basic.toml + CLI->>Docker: Start Anvil, Fake Server, CL Nodes + CLI->>Docker: Deploy contracts, create jobs + CLI->>EnvOut: Write deployed state + + Dev->>Test: go test -v -run TestVRFBasic + Test->>EnvOut: Load config + product output + Test->>Docker: Interact with contracts (gethwrappers) + Test->>Docker: Query CL node API (clclient) + Test->>Test: Assert expected behavior +``` + +### Test File Pattern + +Every smoke test follows the same structure: + +1. **Load output** -- read `../../env-out.toml` to get infrastructure and product config + +```go +in, err := de.LoadOutput[de.Cfg](outputFile) +productCfg, err := products.LoadOutput[.Configurator](outputFile) +``` + +2. **Setup cleanup** -- save container logs on test completion + +```go +t.Cleanup(func() { + framework.SaveContainerLogs(...) +}) +``` + +3. **Create clients** -- ETH client for on-chain interaction, CL client for node API + +```go +c, auth, _, err := products.ETHClient(ctx, wsURL, feeCap, tipCap) +cls, err := clclient.New(in.NodeSets[0].Out.CLNodes) +``` + +4. **Interact with contracts** -- use gethwrappers directly (never through `chainlink/v2` wrappers) + +```go +consumer, err := solidity_vrf_consumer_interface.NewVRFConsumer(addr, c) +``` + +5. **Assert with polling** -- use `require.EventuallyWithT` to poll until expected state + +```go +require.EventuallyWithT(t, func(ct *assert.CollectT) { + // check on-chain state or job runs +}, 2*time.Minute, 2*time.Second) +``` + +### Dependency Rule + +Tests in `devenv/tests/` must NOT import: +- `github.com/smartcontractkit/chainlink/v2` +- `github.com/smartcontractkit/chainlink/integration-tests` +- `github.com/smartcontractkit/chainlink/deployment` + +Allowed imports are: +- `github.com/smartcontractkit/chainlink/devenv` (this module) +- `github.com/smartcontractkit/chainlink-testing-framework/framework` (CTF) +- `github.com/smartcontractkit/chainlink-evm/gethwrappers` (contract bindings) +- `github.com/smartcontractkit/libocr` (OCR bindings) +- Standard library and third-party libraries (testify, go-ethereum, etc.) + +## CI Integration + +System tests run nightly via [`.github/workflows/devenv-nightly.yml`](../.github/workflows/devenv-nightly.yml). + +### Workflow Structure + +The workflow uses a GitHub Actions matrix strategy where each entry defines: + +| Field | Purpose | +| ------------------- | ---------------------------------------------------------------- | +| `display_name` | Human-readable test name | +| `envcmd` | Command to set up the environment (runs from `devenv/`) | +| `testcmd` | Command to run the tests (runs from `devenv/tests//`) | +| `runner` | GitHub Actions runner label | +| `tests_dir` | Subdirectory under `devenv/tests/` | +| `logs_archive_name` | Name for the uploaded log artifact | + +### Execution Flow + +```mermaid +flowchart TD + A[Checkout code] --> B[Setup Docker Buildx] + B --> C[Install Just] + C --> D[AWS OIDC + ECR Login] + D --> E[Setup Go + download deps] + E --> F["Set CHAINLINK_IMAGE\n(nightly build)"] + F --> G["Install cl CLI\n(go install cmd/cl)"] + G --> H["eval envcmd\n(starts environment)"] + H --> I["eval testcmd\n(runs go test)"] + I --> J["Upload logs artifact\n(always)"] +``` + +### Adding a Test to CI + +Add a new entry to the `matrix.include` array: + +```yaml +- display_name: "Test Smoke" + testcmd: "go test -v -timeout 10m -run " + envcmd: "cl u env.toml,products//basic.toml" + runner: "ubuntu-latest" + tests_dir: "" + logs_archive_name: "" +``` diff --git a/devenv/tests/README.md b/devenv/tests/README.md index ccd198f71a0..184d54eb670 100644 --- a/devenv/tests/README.md +++ b/devenv/tests/README.md @@ -7,7 +7,7 @@ Spin up observability stack in case you need performance tests: obs up -f ``` -To run any test, open two terminals and setup corresponding commands, `envcmd` and `testcmd` fields from [here](https://github.com/smartcontractkit/chainlink/blob/develop/.github/workflows/devenv-nightly.yml#L45): +To run any test, open two terminals and setup corresponding commands, `envcmd` and `testcmd` fields from [devenv-nightly](/.github/workflows/devenv-nightly.yml): ```bash $envcmd (from devenv dir) $testcmd (from tests/$product dir, for example "tests/automation") From f686cfc33b4708c45085a90af0c28941871070f5 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 13 Mar 2026 15:02:34 -0400 Subject: [PATCH 03/11] Run nightly devenv tests on certain PRs --- .github/workflows/devenv-nightly.yml | 35 +++++++-- devenv/AGENTS.md | 102 +++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 devenv/AGENTS.md diff --git a/.github/workflows/devenv-nightly.yml b/.github/workflows/devenv-nightly.yml index a293f6b52c7..e7e06405281 100644 --- a/.github/workflows/devenv-nightly.yml +++ b/.github/workflows/devenv-nightly.yml @@ -2,8 +2,9 @@ name: (Nightly) System Tests on: schedule: - - cron: "0 6 * * *" # Run daily at 6 AM + - cron: "0 0 * * *" # Run daily at midnight UTC workflow_dispatch: + pull_request: defaults: run: @@ -14,6 +15,28 @@ concurrency: cancel-in-progress: true jobs: + changes: + name: Detect devenv changes + runs-on: ubuntu-latest + outputs: + devenv-changes: ${{ steps.devenv-changes.outputs.src }} + steps: + - name: Checkout the repo + uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: dorny/paths-filter@9d7afb8d214ad99e78fbd4247752c4caed2b6e4c # v4.0.0 + id: devenv-changes + with: + filters: | + src: + - 'devenv/**/*.go' + - 'devenv/**/*.toml' + - 'devenv/**/go.mod' + - 'devenv/**/go.sum' + - 'devenv/**/*Dockerfile' + - '.github/workflows/devenv-nightly.yml' + summary: name: "Build Information" runs-on: ubuntu-latest @@ -21,7 +44,7 @@ jobs: contents: read steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Print Summary @@ -29,14 +52,16 @@ jobs: IMAGE_VERSION="nightly-$(date +%Y%m%d)" FULL_IMAGE_HINT="/chainlink:${IMAGE_VERSION}" echo "**Image:** \`${FULL_IMAGE_HINT}\`" >> $GITHUB_STEP_SUMMARY - test-nightly: name: ${{ matrix.display_name }} + needs: [changes] permissions: id-token: write contents: read pull-requests: write runs-on: ${{ matrix.runner }} + # If there's a PR that's changing devenv, run the tests + if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.devenv-changes == 'true' }} strategy: fail-fast: false matrix: @@ -149,7 +174,7 @@ jobs: logs_archive_name: "df1-chaos" steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 @@ -181,7 +206,7 @@ jobs: - name: Download Go dependencies run: | go mod download - + - name: Set environment variables run: | IMAGE_VERSION="nightly-$(date +%Y%m%d)" diff --git a/devenv/AGENTS.md b/devenv/AGENTS.md new file mode 100644 index 00000000000..c2eef80f8a6 --- /dev/null +++ b/devenv/AGENTS.md @@ -0,0 +1,102 @@ +# Devenv — AI Agent Guidelines + +This file documents conventions and constraints for the `devenv` module. Follow these rules when generating, modifying, or reviewing code under `devenv/`. + +## Module Isolation + +`devenv` is a standalone Go module: `github.com/smartcontractkit/chainlink/devenv`. + +**Critical rule**: NEVER import `github.com/smartcontractkit/chainlink/v2` or any of its children (e.g. `chainlink/v2/core/...`, `chainlink/integration-tests/...`, `chainlink/deployment/...`). This is enforced by `depguard` in `.golangci.yml` and will fail CI. + +### Allowed Dependencies + +| Dependency | Use for | +|---|---| +| `github.com/smartcontractkit/chainlink-testing-framework/framework` | Docker orchestration, `clclient` (CL node HTTP API), observability | +| `github.com/smartcontractkit/chainlink-evm/gethwrappers` | On-chain contract interaction (deploy, call, transact) | +| `github.com/smartcontractkit/libocr` | OCR-specific contract wrappers | +| `github.com/ethereum/go-ethereum` | ETH client, `bind`, ABI, `common`, `crypto` | +| `github.com/stretchr/testify` | Test assertions (`require`, `assert`) | +| `github.com/google/uuid` | UUID generation | +| Standard library | Everything else | + +### Denied Packages + +These are enforced by depguard and will cause lint failures: + +| Denied | Use instead | +|---|---| +| `github.com/smartcontractkit/chainlink/v2` (and children) | Local implementations or CTF equivalents | +| `github.com/BurntSushi/toml` | `github.com/pelletier/go-toml/v2` | +| `github.com/smartcontractkit/chainlink-integrations/evm` | `github.com/smartcontractkit/chainlink-evm` | +| `github.com/gofrs/uuid`, `github.com/satori/go.uuid` | `github.com/google/uuid` | +| `github.com/test-go/testify/*` | `github.com/stretchr/testify/*` | +| `go.uber.org/multierr` | `errors.Join` (standard library) | +| `gopkg.in/guregu/null.v1/v2/v3` | `gopkg.in/guregu/null.v4` | +| `github.com/go-gorm/gorm` | `github.com/jmoiron/sqlx` | + +## Product Interface + +Every Chainlink product in devenv implements the `Product` interface defined in [interface.go](interface.go). Read that file for the exact method signatures. + +### Adding a New Product + +1. Create `products//configuration.go` implementing `Product` — see any existing product (e.g. [products/cron/configuration.go](products/cron/configuration.go)) as a reference +2. Create `products//basic.toml` with default TOML config +3. Register in [environment.go](environment.go) — add a `case ""` in `newProduct()` +4. Create `tests//smoke_test.go` +5. Add a matrix entry in [`.github/workflows/devenv-nightly.yml`](../.github/workflows/devenv-nightly.yml) + +### Product Lifecycle + +The environment calls product methods in this order: + +1. `Load()` — parse product config from merged TOML +2. `GenerateNodesConfig()` — return CL node TOML overrides +3. `GenerateNodesSecrets()` — return CL node secrets overrides +4. *(infrastructure starts: blockchain, fake server, CL nodes)* +5. `ConfigureJobsAndContracts()` — deploy contracts, create keys/jobs, fund nodes +6. `Store()` — write deployed state (addresses, job IDs) to `env-out.toml` + +See [environment.go](environment.go) `NewEnvironment()` for the full orchestration flow. + +## Test Conventions + +Tests use a two-phase pattern: environment setup (via `cl` CLI) then test execution (via `go test`). + +### Test File Structure + +See [tests/cron/smoke_test.go](tests/cron/smoke_test.go) for the simplest example of the standard pattern. Every smoke test follows the same structure: + +1. Load infrastructure output via `de.LoadOutput[de.Cfg]` +2. Load product output via `products.LoadOutput[.Configurator]` +3. Save container logs in `t.Cleanup` +4. Create clients (ETH and/or CL node) +5. Interact with contracts and assert results + +### Key Patterns + +- Output file path from `tests//` is always `../../env-out.toml` +- Use `products.ETHClient()` for Ethereum client creation with gas settings +- Use `products.WaitMinedFast()` for fast transaction confirmation on Anvil +- Use `clclient.New()` for CL node API access (job runs, keys, jobs) +- Use `require.EventuallyWithT` for async assertions (typical: 2 min timeout, 2 s interval) +- Use gethwrappers from `chainlink-evm/gethwrappers` directly for contract bindings + +## Configuration + +- Base infra: `env.toml` (blockchain, fake server, node set) +- Product config: `products//basic.toml` +- CLI merges configs left-to-right: `cl u env.toml,products//basic.toml` +- Output written to `env-out.toml` after environment starts + +## Formatting and Linting + +- **Formatter**: goimports with local prefix `github.com/smartcontractkit/chainlink` +- **Linter config**: `devenv/.golangci.yml` +- **Run linter**: `golangci-lint run` from the `devenv/` directory +- **nolint directives**: must include both an explanation and a specific linter name + +## CI + +Tests run via [`.github/workflows/devenv-nightly.yml`](../.github/workflows/devenv-nightly.yml) using a matrix with `envcmd` and `testcmd` pairs. When adding a new test, copy an existing matrix entry and update the product name, test command, and directory fields. From 1b688b95a61f08fd185bed205d74f1f7288eb9a1 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 13 Mar 2026 15:06:29 -0400 Subject: [PATCH 04/11] Add debug monitoring --- .github/workflows/devenv-nightly.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/devenv-nightly.yml b/.github/workflows/devenv-nightly.yml index e7e06405281..b6d0e416408 100644 --- a/.github/workflows/devenv-nightly.yml +++ b/.github/workflows/devenv-nightly.yml @@ -173,6 +173,14 @@ jobs: tests_dir: "ocr2" logs_archive_name: "df1-chaos" steps: + # DEBUG: Run monitor for tests + - name: Monitor + uses: kalverra/octometrics-action@v0.0.10 + with: + job_name: ${{ matrix.display_name }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Checkout code uses: actions/checkout@v6 with: From 0ecd09ae59802ba718f357ddf3f4339ce516cc22 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 13 Mar 2026 15:35:58 -0400 Subject: [PATCH 05/11] Add migration guide --- .github/workflows/devenv-nightly.yml | 5 +- devenv/AGENTS.md | 38 ++++--- devenv/Justfile | 5 +- devenv/MIGRATION_GUIDE.md | 163 +++++++++++++++++++++++++++ devenv/products/vrf/configuration.go | 24 ++-- devenv/tests/vrf/smoke_test.go | 2 +- 6 files changed, 202 insertions(+), 35 deletions(-) create mode 100644 devenv/MIGRATION_GUIDE.md diff --git a/.github/workflows/devenv-nightly.yml b/.github/workflows/devenv-nightly.yml index b6d0e416408..f5696093334 100644 --- a/.github/workflows/devenv-nightly.yml +++ b/.github/workflows/devenv-nightly.yml @@ -29,7 +29,6 @@ jobs: id: devenv-changes with: filters: | - src: - 'devenv/**/*.go' - 'devenv/**/*.toml' - 'devenv/**/go.mod' @@ -187,7 +186,7 @@ jobs: fetch-depth: 0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Install Just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 @@ -195,7 +194,7 @@ jobs: just-version: "1.40.0" - name: Configure AWS credentials using OIDC - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: role-to-assume: ${{ secrets.AWS_OIDC_IAM_ROLE_SDLC_ECR_READONLY_ARN }} aws-region: us-west-2 diff --git a/devenv/AGENTS.md b/devenv/AGENTS.md index c2eef80f8a6..1dcc1ac92f5 100644 --- a/devenv/AGENTS.md +++ b/devenv/AGENTS.md @@ -10,30 +10,30 @@ This file documents conventions and constraints for the `devenv` module. Follow ### Allowed Dependencies -| Dependency | Use for | -|---|---| +| Dependency | Use for | +| ------------------------------------------------------------------- | ------------------------------------------------------------------ | | `github.com/smartcontractkit/chainlink-testing-framework/framework` | Docker orchestration, `clclient` (CL node HTTP API), observability | -| `github.com/smartcontractkit/chainlink-evm/gethwrappers` | On-chain contract interaction (deploy, call, transact) | -| `github.com/smartcontractkit/libocr` | OCR-specific contract wrappers | -| `github.com/ethereum/go-ethereum` | ETH client, `bind`, ABI, `common`, `crypto` | -| `github.com/stretchr/testify` | Test assertions (`require`, `assert`) | -| `github.com/google/uuid` | UUID generation | -| Standard library | Everything else | +| `github.com/smartcontractkit/chainlink-evm/gethwrappers` | On-chain contract interaction (deploy, call, transact) | +| `github.com/smartcontractkit/libocr` | OCR-specific contract wrappers | +| `github.com/ethereum/go-ethereum` | ETH client, `bind`, ABI, `common`, `crypto` | +| `github.com/stretchr/testify` | Test assertions (`require`, `assert`) | +| `github.com/google/uuid` | UUID generation | +| Standard library | Everything else | ### Denied Packages These are enforced by depguard and will cause lint failures: -| Denied | Use instead | -|---|---| -| `github.com/smartcontractkit/chainlink/v2` (and children) | Local implementations or CTF equivalents | -| `github.com/BurntSushi/toml` | `github.com/pelletier/go-toml/v2` | -| `github.com/smartcontractkit/chainlink-integrations/evm` | `github.com/smartcontractkit/chainlink-evm` | -| `github.com/gofrs/uuid`, `github.com/satori/go.uuid` | `github.com/google/uuid` | -| `github.com/test-go/testify/*` | `github.com/stretchr/testify/*` | -| `go.uber.org/multierr` | `errors.Join` (standard library) | -| `gopkg.in/guregu/null.v1/v2/v3` | `gopkg.in/guregu/null.v4` | -| `github.com/go-gorm/gorm` | `github.com/jmoiron/sqlx` | +| Denied | Use instead | +| --------------------------------------------------------- | ------------------------------------------- | +| `github.com/smartcontractkit/chainlink/v2` (and children) | Local implementations or CTF equivalents | +| `github.com/BurntSushi/toml` | `github.com/pelletier/go-toml/v2` | +| `github.com/smartcontractkit/chainlink-integrations/evm` | `github.com/smartcontractkit/chainlink-evm` | +| `github.com/gofrs/uuid`, `github.com/satori/go.uuid` | `github.com/google/uuid` | +| `github.com/test-go/testify/*` | `github.com/stretchr/testify/*` | +| `go.uber.org/multierr` | `errors.Join` (standard library) | +| `gopkg.in/guregu/null.v1/v2/v3` | `gopkg.in/guregu/null.v4` | +| `github.com/go-gorm/gorm` | `github.com/jmoiron/sqlx` | ## Product Interface @@ -47,6 +47,8 @@ Every Chainlink product in devenv implements the `Product` interface defined in 4. Create `tests//smoke_test.go` 5. Add a matrix entry in [`.github/workflows/devenv-nightly.yml`](../.github/workflows/devenv-nightly.yml) +If migrating an existing test from `integration-tests/smoke/`, follow the full step-by-step process in [MIGRATION_GUIDE.md](MIGRATION_GUIDE.md). + ### Product Lifecycle The environment calls product methods in this order: diff --git a/devenv/Justfile b/devenv/Justfile index 5bbf2fde7a6..1e7e093ab9e 100644 --- a/devenv/Justfile +++ b/devenv/Justfile @@ -15,4 +15,7 @@ push-fakes registry: # Rebuild CLI cli: - cd cmd/cl && go install -ldflags="-X main.Version=1.0.0" . && cd - > /dev/null || exit 1 \ No newline at end of file + cd cmd/cl && go install -ldflags="-X main.Version=1.0.0" . && cd - > /dev/null || exit 1 + +lint: + golangci-lint run ./... --fix \ No newline at end of file diff --git a/devenv/MIGRATION_GUIDE.md b/devenv/MIGRATION_GUIDE.md new file mode 100644 index 00000000000..9f4976c564f --- /dev/null +++ b/devenv/MIGRATION_GUIDE.md @@ -0,0 +1,163 @@ +# Migrating Smoke Tests to devenv + +Step-by-step guide for converting a test from `integration-tests/smoke/` to the `devenv/` pattern. Read [AGENTS.md](AGENTS.md) first for module constraints and conventions. + +## Scope + +This guide covers migrating a single Chainlink product's smoke tests. If the product already has a configurator in `devenv/products/`, skip to Step 4. + +## Pre-work: Analyze the Source Test + +Read the old test file in `integration-tests/smoke/_test.go` and identify: + +1. **Product name** -- what Chainlink product is being tested (e.g. VRF, Flux, Direct Request) +2. **Contracts deployed** -- which Solidity contracts are deployed during test setup +3. **Jobs and keys created** -- which CL node job types and key types are used +4. **Helper functions** -- any calls to `integration-tests/actions` or `integration-tests/contracts` that need local replacements +5. **Forbidden imports** -- list every import from `github.com/smartcontractkit/chainlink/v2`, `integration-tests/`, or `deployment/` -- all of these must be replaced + +For each forbidden import, find the replacement: + +| Old import pattern | Replacement | +| --------------------------------------- | ---------------------------------------------------- | +| `integration-tests/contracts/` | Direct gethwrapper from `chainlink-evm/gethwrappers` | +| `integration-tests/actions/` | Reimplement locally in the configurator | +| `integration-tests/client` | `chainlink-testing-framework/framework/clclient` | +| `chainlink/v2/core/services/...` | Not needed -- use `clclient` job spec types | +| `chainlink/v2/core/gethwrappers/...` | `chainlink-evm/gethwrappers` | + +## Step 1: Create the Product TOML Config + +Create `devenv/products//basic.toml`. + +Reference file: [products/directrequest/basic.toml](products/directrequest/basic.toml) (simplest single-node product). + +Key decisions: +- `[[products]]` `name` field must match the switch case you will add in Step 3 +- `nodes` count depends on the product (1 for single-node products like cron/DR/VRF, 5 for multi-node like OCR2/flux) +- The product-specific TOML section (e.g. `[[vrf]]`) key must match the `toml` struct tag on the `Configurator.Config` field you create in Step 2 +- Include `gas_settings` if the product deploys contracts + +## Step 2: Create the Product Configurator + +Create `devenv/products//configuration.go`. + +Reference files: +- [products/directrequest/configuration.go](products/directrequest/configuration.go) -- single-node product with contracts, bridge, and job +- [products/vrf/configuration.go](products/vrf/configuration.go) -- single-node product with multiple contracts and local helper functions +- [products/flux/configuration.go](products/flux/configuration.go) -- multi-node product + +### Struct definitions + +Define three structs: + +1. `Configurator` -- top-level, with `Config []*` and a `toml` tag matching the TOML section key +2. `` -- product config fields (typically `GasSettings` + `Out`) +3. `Out` -- all deployed state the tests will need (contract addresses, job IDs, key hashes, chain IDs, etc.). Every field must have a `toml` tag. + +### Boilerplate methods + +`Load()`, `Store()`, `GenerateNodesConfig()`, and `GenerateNodesSecrets()` are nearly identical across products. Copy from any existing product and adjust the type parameter in `products.Load[Configurator]()`. + +### ConfigureJobsAndContracts + +This is where the real migration work happens. Port the setup logic from the old test file: + +1. Connect to CL nodes via `clclient.New(ns[0].Out.CLNodes)` +2. Create ETH client via `products.ETHClient()` +3. Deploy contracts using gethwrappers directly (e.g. `link_token.DeployLinkToken`, then `bind.WaitDeployed`) +4. Use `products.WaitMinedFast()` for transaction confirmations after deployment +5. Fund CL node transmitter addresses via `products.FundAddressEIP1559()` +6. Create keys/bridges/jobs via `clclient` methods (`MustCreateVRFKey`, `MustCreateBridge`, `MustCreateJob`, etc.) +7. Store all outputs the test will need in `m.Config[0].Out` + +### Replacing forbidden helpers + +When old tests use functions from `integration-tests/actions`: +- Read the source of the helper function +- Reimplement it locally as an unexported function in your `configuration.go` +- Example: VRF migration reimplemented `EncodeOnChainVRFProvingKey` and `EncodeOnChainExternalJobID` as local helpers (see [products/vrf/configuration.go](products/vrf/configuration.go)) + +### Contract wrappers + +Old tests often use wrappers from `integration-tests/contracts` that add convenience methods around gethwrappers. In devenv, use the gethwrappers directly: + +- Find the underlying gethwrapper package in `chainlink-evm/gethwrappers/generated/` or `chainlink-evm/gethwrappers/shared/generated/` +- Call `Deploy(auth, client, ...)` directly +- Call contract methods on the returned instance directly + +## Step 3: Register the Product + +Edit [environment.go](environment.go): + +1. Add an import for the new package: `"github.com/smartcontractkit/chainlink/devenv/products/"` +2. Add a case to the `newProduct()` switch matching the product name from your TOML config + +## Step 4: Create the Smoke Test + +Create `devenv/tests//smoke_test.go`. + +Reference files: +- [tests/cron/smoke_test.go](tests/cron/smoke_test.go) -- simplest test (no contracts, just job run polling) +- [tests/vrf/smoke_test.go](tests/vrf/smoke_test.go) -- test with contract interaction, key hash decoding, and job replacement + +Every test must: + +1. Load infra output: `de.LoadOutput[de.Cfg]("../../env-out.toml")` +2. Load product output: `products.LoadOutput[.Configurator]("../../env-out.toml")` +3. Register cleanup: `t.Cleanup(func() { framework.SaveContainerLogs(...) })` +4. Create clients as needed (`products.ETHClient`, `clclient.New`) +5. Interact with contracts via gethwrappers (never `chainlink/v2` wrappers) +6. Assert with `require.EventuallyWithT` for async results (typical: 2 min timeout, 2 s poll) + +The test file should only contain assertion logic. All setup (contract deployment, job creation, funding) belongs in the configurator. + +## Step 5: Verify the Build + +From the `devenv/` directory, run targeted builds on the new packages only: + +```bash +go build ./products//... +go build ./tests//... +go vet ./products//... +go vet ./tests//... +``` + +Verify no forbidden imports slipped in and linting: + +```bash +just lint +``` + +Both grep commands should return no results. + +## Step 6: Remove Old Test from CI + +Open `.github/e2e-tests.yml` and find the entry matching the old test file path (e.g. `path: integration-tests/smoke/vrf_test.go`). Delete the entire entry block including its `id`, `path`, `test_env_type`, `triggers`, `test_cmd`, and all other fields. + +## Step 7: Add New Test to CI + +Open `.github/workflows/devenv-nightly.yml` and add a matrix entry in the `matrix.include` array of the `test-nightly` job. Copy an existing entry for a product of similar complexity and update: + +- `display_name` -- human-readable name +- `testcmd` -- the `go test` command with `-run` filter matching your test function names +- `envcmd` -- the `cl u` command pointing to your TOML configs +- `runner` -- `ubuntu-latest` for simple tests, larger runners for resource-heavy tests +- `tests_dir` -- the subdirectory name under `devenv/tests/` +- `logs_archive_name` -- name for the uploaded log artifact + +If the `testcmd` includes multiple test names separated by `|`, escape the pipe as `\\|` in YAML. + +## Step 8: Delete the Old Test File + +Delete `integration-tests/smoke/_test.go`. If this was the last test in that file, also check if there are any shared helpers in the same package that are now unused and can be removed. + +## Common Pitfalls + +1. **Forbidden imports** -- The most common failure. Grep for `chainlink/v2`, `integration-tests`, and `deployment` before committing. The `depguard` linter in `devenv/.golangci.yml` enforces this. +2. **TOML key mismatch** -- The `toml` struct tag on `Configurator.Config` must exactly match the TOML section name (e.g. `toml:"vrf"` matches `[[vrf]]`). A mismatch means the config silently loads as empty. +3. **Missing `toml` tags on `Out` fields** -- Every field in the `Out` struct needs a `toml` tag or it won't be persisted/loaded from `env-out.toml`. +4. **`WaitMinedFast` vs `bind.WaitDeployed`** -- Use `bind.WaitDeployed` for contract deployment transactions (returns the deployed address). Use `products.WaitMinedFast` for all other transactions (state changes, transfers, registrations). +5. **Test imports product package** -- The test in `tests//` imports the product package from `products//` only for the `Configurator` type. All contract interaction in tests uses gethwrappers directly. +6. **Output file path** -- Tests run from `devenv/tests//`, so the output file is `../../env-out.toml` (two levels up to `devenv/`). +7. **Package name** -- The test file's `package` declaration should match the directory name (e.g. `package vrf` in `tests/vrf/`). diff --git a/devenv/products/vrf/configuration.go b/devenv/products/vrf/configuration.go index f62a8f9b234..f1c96d458de 100644 --- a/devenv/products/vrf/configuration.go +++ b/devenv/products/vrf/configuration.go @@ -40,13 +40,13 @@ type VRF struct { } type Out struct { - ConsumerAddress string `toml:"consumer_address"` - CoordinatorAddress string `toml:"coordinator_address"` - KeyHash string `toml:"key_hash"` - JobID string `toml:"job_id"` + ConsumerAddress string `toml:"consumer_address"` + CoordinatorAddress string `toml:"coordinator_address"` + KeyHash string `toml:"key_hash"` + JobID string `toml:"job_id"` PublicKeyCompressed string `toml:"public_key_compressed"` - ExternalJobID string `toml:"external_job_id"` - ChainID string `toml:"chain_id"` + ExternalJobID string `toml:"external_job_id"` + ChainID string `toml:"chain_id"` } func NewConfigurator() *Configurator { @@ -257,13 +257,13 @@ func (m *Configurator) ConfigureJobsAndContracts( L.Info().Str("KeyHash", hex.EncodeToString(keyHash[:])).Msg("Computed key hash") m.Config[0].Out = &Out{ - ConsumerAddress: consumerAddr.String(), - CoordinatorAddress: coordAddr.String(), - KeyHash: hex.EncodeToString(keyHash[:]), - JobID: job.Data.ID, + ConsumerAddress: consumerAddr.String(), + CoordinatorAddress: coordAddr.String(), + KeyHash: hex.EncodeToString(keyHash[:]), + JobID: job.Data.ID, PublicKeyCompressed: pubKeyCompressed, - ExternalJobID: jobUUID.String(), - ChainID: bc[0].ChainID, + ExternalJobID: jobUUID.String(), + ChainID: bc[0].ChainID, } return nil } diff --git a/devenv/tests/vrf/smoke_test.go b/devenv/tests/vrf/smoke_test.go index 68822d038be..5113e01e5b2 100644 --- a/devenv/tests/vrf/smoke_test.go +++ b/devenv/tests/vrf/smoke_test.go @@ -125,7 +125,7 @@ func TestVRFJobReplacement(t *testing.T) { require.NoError(t, err) newJob, err := cls[0].MustCreateJob(&clclient.VRFJobSpec{ - Name: fmt.Sprintf("vrf-%s", cfg.ExternalJobID), + Name: "vrf-" + cfg.ExternalJobID, CoordinatorAddress: cfg.CoordinatorAddress, MinIncomingConfirmations: 1, PublicKey: cfg.PublicKeyCompressed, From 3fc4e24be58eab7cfde445c0d2dbeb6c6ca567a5 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 13 Mar 2026 15:51:22 -0400 Subject: [PATCH 06/11] Skip soak tests on PRs --- .github/workflows/devenv-nightly.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/devenv-nightly.yml b/.github/workflows/devenv-nightly.yml index f5696093334..fe58b5d78f8 100644 --- a/.github/workflows/devenv-nightly.yml +++ b/.github/workflows/devenv-nightly.yml @@ -231,6 +231,8 @@ jobs: - name: Run tests working-directory: devenv/tests/${{ matrix.tests_dir }} + # Skip soak tests on PRs + if: ${{ github.event_name != 'pull_request' || !contains(matrix.display_name, 'Soak') }} env: CURRENT_TEST: ${{ matrix.testcmd }} run: | From ab1677478290d853eb028398a5f32a1b9812708a Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 13 Mar 2026 16:09:54 -0400 Subject: [PATCH 07/11] Fix logic typ --- .github/workflows/devenv-nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/devenv-nightly.yml b/.github/workflows/devenv-nightly.yml index fe58b5d78f8..8fbe2376bcd 100644 --- a/.github/workflows/devenv-nightly.yml +++ b/.github/workflows/devenv-nightly.yml @@ -181,6 +181,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout code + # Skip soak tests on PRs + if: ${{ github.event_name != 'pull_request' || (github.event_name == 'pull_request' && !contains(matrix.display_name, 'Soak')) }} uses: actions/checkout@v6 with: fetch-depth: 0 @@ -231,8 +233,6 @@ jobs: - name: Run tests working-directory: devenv/tests/${{ matrix.tests_dir }} - # Skip soak tests on PRs - if: ${{ github.event_name != 'pull_request' || !contains(matrix.display_name, 'Soak') }} env: CURRENT_TEST: ${{ matrix.testcmd }} run: | From b63f4013b2958cd800d35925e0b7764dcfa36428 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 13 Mar 2026 16:13:12 -0400 Subject: [PATCH 08/11] Paths filter --- .github/workflows/devenv-nightly.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/devenv-nightly.yml b/.github/workflows/devenv-nightly.yml index 8fbe2376bcd..6365578a9c6 100644 --- a/.github/workflows/devenv-nightly.yml +++ b/.github/workflows/devenv-nightly.yml @@ -19,7 +19,7 @@ jobs: name: Detect devenv changes runs-on: ubuntu-latest outputs: - devenv-changes: ${{ steps.devenv-changes.outputs.src }} + devenv-changes: ${{ steps.devenv-changes.outputs.changes }} steps: - name: Checkout the repo uses: actions/checkout@v6 @@ -29,6 +29,7 @@ jobs: id: devenv-changes with: filters: | + changes: - 'devenv/**/*.go' - 'devenv/**/*.toml' - 'devenv/**/go.mod' From 6b73921958e278fc7be7362b4f07a25ce031ad3e Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 13 Mar 2026 16:19:26 -0400 Subject: [PATCH 09/11] Longer running PRs fix --- .github/workflows/devenv-nightly.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/devenv-nightly.yml b/.github/workflows/devenv-nightly.yml index 6365578a9c6..2f7d49503e8 100644 --- a/.github/workflows/devenv-nightly.yml +++ b/.github/workflows/devenv-nightly.yml @@ -160,19 +160,28 @@ jobs: runner: "ubuntu24.04-16cores-64GB" tests_dir: "automation" logs_archive_name: "automation-soak" + run_on_pr: false - display_name: "Test DF1 (OCR2) Soak" testcmd: "go test -v -timeout 4h -run TestOCR2Soak/clean" envcmd: "cl u env.toml,products/ocr2/basic.toml,products/ocr2/soak.toml; cl obs up -f" runner: "ubuntu24.04-16cores-64GB" tests_dir: "ocr2" logs_archive_name: "df1-soak" + run_on_pr: false - display_name: "Test DF1 (OCR2) Chaos" testcmd: "go test -v -timeout 30m -run TestOCR2Chaos" envcmd: "cl u env.toml,products/ocr2/basic.toml; cl obs up -f" runner: "ubuntu24.04-16cores-64GB" tests_dir: "ocr2" logs_archive_name: "df1-chaos" + run_on_pr: false steps: + - name: Skip soak tests on PRs + if: ${{ github.event_name != 'pull_request' || (github.event_name == 'pull_request' && matrix.run_on_pr == false) }} + run: | + echo "Skipping soak tests on PRs" + exit 0 + # DEBUG: Run monitor for tests - name: Monitor uses: kalverra/octometrics-action@v0.0.10 @@ -182,8 +191,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout code - # Skip soak tests on PRs - if: ${{ github.event_name != 'pull_request' || (github.event_name == 'pull_request' && !contains(matrix.display_name, 'Soak')) }} uses: actions/checkout@v6 with: fetch-depth: 0 From de5a3e253b3afb557753e0ece745c51889efc362 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 13 Mar 2026 16:24:56 -0400 Subject: [PATCH 10/11] Longer running PRs fix...again --- .github/workflows/devenv-nightly.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/devenv-nightly.yml b/.github/workflows/devenv-nightly.yml index 2f7d49503e8..059a3453dd4 100644 --- a/.github/workflows/devenv-nightly.yml +++ b/.github/workflows/devenv-nightly.yml @@ -176,14 +176,20 @@ jobs: logs_archive_name: "df1-chaos" run_on_pr: false steps: - - name: Skip soak tests on PRs - if: ${{ github.event_name != 'pull_request' || (github.event_name == 'pull_request' && matrix.run_on_pr == false) }} + - name: Check if job should run + id: gate + working-directory: . run: | - echo "Skipping soak tests on PRs" - exit 0 + if [[ "${{ github.event_name }}" == "pull_request" && "${{ matrix.run_on_pr }}" == "false" ]]; then + echo "should_run=false" >> "$GITHUB_OUTPUT" + echo "Skipping ${{ matrix.display_name }} on PR" + else + echo "should_run=true" >> "$GITHUB_OUTPUT" + fi # DEBUG: Run monitor for tests - name: Monitor + if: steps.gate.outputs.should_run == 'true' uses: kalverra/octometrics-action@v0.0.10 with: job_name: ${{ matrix.display_name }} @@ -191,29 +197,35 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout code + if: steps.gate.outputs.should_run == 'true' uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Docker Buildx + if: steps.gate.outputs.should_run == 'true' uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Install Just + if: steps.gate.outputs.should_run == 'true' uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 with: just-version: "1.40.0" - name: Configure AWS credentials using OIDC + if: steps.gate.outputs.should_run == 'true' uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: role-to-assume: ${{ secrets.AWS_OIDC_IAM_ROLE_SDLC_ECR_READONLY_ARN }} aws-region: us-west-2 - name: Authenticate to ECR + if: steps.gate.outputs.should_run == 'true' id: login-ecr uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Set up Go + if: steps.gate.outputs.should_run == 'true' uses: actions/setup-go@v6 # v6 with: cache: true @@ -221,16 +233,19 @@ jobs: cache-dependency-path: devenv/go.sum - name: Download Go dependencies + if: steps.gate.outputs.should_run == 'true' run: | go mod download - name: Set environment variables + if: steps.gate.outputs.should_run == 'true' run: | IMAGE_VERSION="nightly-$(date +%Y%m%d)" FULL_IMAGE="${{ secrets.REGISTRY_SDLC }}/chainlink:${IMAGE_VERSION}" echo "CHAINLINK_IMAGE=${FULL_IMAGE}" >> $GITHUB_ENV - name: Setup environment + if: steps.gate.outputs.should_run == 'true' env: FAKE_SERVER_IMAGE: ${{ secrets.FAKE_SERVER_IMAGE }} run: | @@ -240,6 +255,7 @@ jobs: eval $ENV_CMD - name: Run tests + if: steps.gate.outputs.should_run == 'true' working-directory: devenv/tests/${{ matrix.tests_dir }} env: CURRENT_TEST: ${{ matrix.testcmd }} @@ -249,7 +265,7 @@ jobs: eval $TESTCMD - name: Upload Logs - if: always() + if: always() && steps.gate.outputs.should_run == 'true' uses: actions/upload-artifact@v4 with: name: container-logs-${{ matrix.logs_archive_name }} From 9e29628474c93bb1e59a9337d716f935d5c0b46a Mon Sep 17 00:00:00 2001 From: skudasov Date: Thu, 19 Mar 2026 10:55:32 +0100 Subject: [PATCH 11/11] fix cron trigger --- .github/workflows/devenv-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/devenv-nightly.yml b/.github/workflows/devenv-nightly.yml index 059a3453dd4..cd881f7bb4e 100644 --- a/.github/workflows/devenv-nightly.yml +++ b/.github/workflows/devenv-nightly.yml @@ -2,7 +2,7 @@ name: (Nightly) System Tests on: schedule: - - cron: "0 0 * * *" # Run daily at midnight UTC + - cron: "0 6 * * *" # Run daily at 6 AM workflow_dispatch: pull_request: