Merge pull request #4205 from smartcontractkit/release/0.10.4-rc4

Release/0.10.4 rc4

.github/workflows/continuous-integration-workflow.yml

@@ -42,6 +42,8 @@ jobs:
         uses: docker://postgres
         with:
           args: psql -v ON_ERROR_STOP=1 --username postgres -h postgres -c "CREATE USER chainlink NOSUPERUSER CREATEDB;"
+      - name: Install Postgres for CLI tools
+        run: wget --quiet -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - && echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |sudo tee  /etc/apt/sources.list.d/pgdg.list && sudo apt update && sudo apt install -y postgresql-client-13
       - name: Cache Yarn dependencies
         uses: actions/cache@v2
         with:

.github/workflows/dependency-check.yml

@@ -20,8 +20,10 @@ jobs:
       - name: Check vulnerabilities
         uses: sonatype-nexus-community/nancy-github-action@b492e6567a301a914bd227d44bf10dcfe85438ad
         with:
-          nancyCommand: sleuth -e bba60acb-c7b5-4621-af69-f4085a8301d0,d373dc3f-aa88-483b-b501-20fe5382cc80,5def94e5-b89c-4a94-b9c6-ae0e120784c2
+          nancyCommand: sleuth -e bba60acb-c7b5-4621-af69-f4085a8301d0,d373dc3f-aa88-483b-b501-20fe5382cc80,5def94e5-b89c-4a94-b9c6-ae0e120784c2,dcf6da03-f9dd-4a4e-b792-0262de36a0b1
           # Check the dependency by entering the OSS index UUID here:
           # https://ossindex.sonatype.org/vuln/bba60acb-c7b5-4621-af69-f4085a8301d0
           #
           # To get more detail locally run `go list -json -m all | nancy sleuth`
+          # dcf6da03-f9dd-4a4e-b792-0262de36a0b1 is because of gogo/protobuf@1.3.1
+          # which is used by go-libp2p-core, need them to upgrade to 1.3.2 before we can remove it.

.gitignore

@@ -29,10 +29,15 @@ tools/clroot/db.sqlite3-wal
 .DS_Store
 .envrc
 .env*
+.idea 
 
 # codeship
 *.aes
 dockercfg
 env
 credentials.env
 gcr_creds.env
+
+# DB backups
+
+cl_backup_*.tar.gz

CODEOWNERS

@@ -1 +1,3 @@
 core @se3000 @RyanRHall @spooktheducks @samsondav @j16r @connorwstein
+
+/operator-ui/ @DeividasK

VERSION

@@ -1 +1 @@
-0.10.3
+0.10.4

core/adapters/bridge.go

@@ -8,13 +8,11 @@ import (
 	"net/http"
 	"net/url"
 
-	uuid "github.com/satori/go.uuid"
 	"github.com/smartcontractkit/chainlink/core/store"
 	"github.com/smartcontractkit/chainlink/core/store/models"
 	"github.com/smartcontractkit/chainlink/core/utils"
 
 	"github.com/pkg/errors"
-	"github.com/tidwall/gjson"
 )
 
 // Bridge adapter is responsible for connecting the task pipeline to external
@@ -42,31 +40,10 @@ func (ba *Bridge) Perform(input models.RunInput, store *store.Store) models.RunO
 	} else if input.Status().PendingBridge() {
 		return models.NewRunOutputInProgress(input.Data())
 	}
-	meta := getMeta(store, input.JobRunID())
-	return ba.handleNewRun(input, meta, store)
+	return ba.handleNewRun(input, store)
 }
 
-func getMeta(store *store.Store, jobRunID uuid.UUID) *models.JSON {
-	jobRun, err := store.ORM.FindJobRun(jobRunID)
-	if err != nil {
-		return nil
-	} else if jobRun.RunRequest.TxHash == nil || jobRun.RunRequest.BlockHash == nil {
-		return nil
-	}
-	meta := fmt.Sprintf(`
-		{
-			"initiator": {
-				"transactionHash": "%s",
-				"blockHash": "%s"
-			}
-		}`,
-		jobRun.RunRequest.TxHash.Hex(),
-		jobRun.RunRequest.BlockHash.Hex(),
-	)
-	return &models.JSON{Result: gjson.Parse(meta)}
-}
-
-func (ba *Bridge) handleNewRun(input models.RunInput, meta *models.JSON, store *store.Store) models.RunOutput {
+func (ba *Bridge) handleNewRun(input models.RunInput, store *store.Store) models.RunOutput {
 	data, err := models.Merge(input.Data(), ba.Params)
 	if err != nil {
 		return models.NewRunOutputError(baRunResultError("handling data param", err))
@@ -82,7 +59,7 @@ func (ba *Bridge) handleNewRun(input models.RunInput, meta *models.JSON, store *
 	// Some node operators may run external adapters on their own hardware
 	httpConfig.AllowUnrestrictedNetworkAccess = true
 
-	body, err := ba.postToExternalAdapter(input, meta, responseURL, httpConfig)
+	body, err := ba.postToExternalAdapter(input, responseURL, httpConfig)
 	if err != nil {
 		return models.NewRunOutputError(baRunResultError("post to external adapter", err))
 	}
@@ -120,7 +97,6 @@ func (ba *Bridge) responseToRunResult(body []byte, input models.RunInput) models
 
 func (ba *Bridge) postToExternalAdapter(
 	input models.RunInput,
-	meta *models.JSON,
 	bridgeResponseURL *url.URL,
 	config utils.HTTPRequestConfig,
 ) ([]byte, error) {
@@ -129,7 +105,7 @@ func (ba *Bridge) postToExternalAdapter(
 		return nil, errors.Wrap(err, "error merging bridge params with input params")
 	}
 
-	outgoing := bridgeOutgoing{JobRunID: input.JobRunID().String(), Data: data, Meta: meta}
+	outgoing := bridgeOutgoing{JobRunID: input.JobRunID().String(), Data: data}
 	if bridgeResponseURL != nil {
 		outgoing.ResponseURL = bridgeResponseURL.String()
 	}
@@ -169,10 +145,9 @@ func baRunResultError(str string, err error) error {
 }
 
 type bridgeOutgoing struct {
-	JobRunID    string       `json:"id"`
-	Data        models.JSON  `json:"data"`
-	Meta        *models.JSON `json:"meta,omitempty"`
-	ResponseURL string       `json:"responseURL,omitempty"`
+	JobRunID    string      `json:"id"`
+	Data        models.JSON `json:"data"`
+	ResponseURL string      `json:"responseURL,omitempty"`
 }
 
 var zeroURL = new(url.URL)

core/adapters/bridge_test.go

@@ -1,17 +1,13 @@
 package adapters_test
 
 import (
-	"encoding/hex"
 	"fmt"
 	"net/http"
 	"testing"
 
-	"github.com/smartcontractkit/chainlink/core/services/eth"
-
 	uuid "github.com/satori/go.uuid"
 	"github.com/smartcontractkit/chainlink/core/adapters"
 	"github.com/smartcontractkit/chainlink/core/internal/cltest"
-	"github.com/smartcontractkit/chainlink/core/store"
 	"github.com/smartcontractkit/chainlink/core/store/models"
 
 	"github.com/stretchr/testify/assert"
@@ -24,13 +20,11 @@ func TestBridge_PerformEmbedsParamsInData(t *testing.T) {
 	store.Config.Set("BRIDGE_RESPONSE_URL", cltest.WebURL(t, ""))
 
 	data := ""
-	meta := false
 	token := ""
 	mock, cleanup := cltest.NewHTTPMockServer(t, http.StatusOK, "POST", `{"pending": true}`,
 		func(h http.Header, b string) {
 			body := cltest.JSONFromString(t, b)
 			data = body.Get("data").String()
-			meta = body.Get("meta").Exists()
 			token = h.Get("Authorization")
 		},
 	)
@@ -44,55 +38,6 @@ func TestBridge_PerformEmbedsParamsInData(t *testing.T) {
 	result := ba.Perform(input, store)
 	require.NoError(t, result.Error())
 	assert.Equal(t, `{"bodyParam":true,"result":"100"}`, data)
-	assert.False(t, meta)
-	assert.Equal(t, "Bearer "+bt.OutgoingToken, token)
-}
-
-func setupJobRunAndStore(t *testing.T, txHash []byte, blockHash []byte) (*store.Store, uuid.UUID, func()) {
-	rpcClient, gethClient, _, assertMocksCalled := cltest.NewEthMocksWithStartupAssertions(t)
-	app, cleanup := cltest.NewApplication(t,
-		eth.NewClientWith(rpcClient, gethClient),
-	)
-	app.Store.Config.Set("BRIDGE_RESPONSE_URL", cltest.WebURL(t, ""))
-	require.NoError(t, app.Start())
-	jr := app.MustCreateJobRun(txHash, blockHash)
-
-	return app.Store, jr.ID, func() {
-		assertMocksCalled()
-		cleanup()
-	}
-}
-
-func TestBridge_IncludesMetaIfJobRunIsInDB(t *testing.T) {
-	txHashHex := "d6432b8321d9988e664f23cfce392dff8221da36a44ebb622160156dcef4abb9"
-	blockHashHex := "d5150a4f602af1de7ff51f02c5b55b130693596c68f00b7796ac2b0f51175675"
-	txHash, _ := hex.DecodeString(txHashHex)
-	blockHash, _ := hex.DecodeString(blockHashHex)
-	store, jobRunID, cleanup := setupJobRunAndStore(t, txHash, blockHash)
-	defer cleanup()
-
-	data := ""
-	meta := ""
-	token := ""
-	mock, cleanup := cltest.NewHTTPMockServer(t, http.StatusOK, "POST", `{"pending": true}`,
-		func(h http.Header, b string) {
-			body := cltest.JSONFromString(t, b)
-			data = body.Get("data").String()
-			meta = body.Get("meta").String()
-			token = h.Get("Authorization")
-		},
-	)
-	defer cleanup()
-
-	_, bt := cltest.NewBridgeType(t, "auctionBidding", mock.URL)
-	params := cltest.JSONFromString(t, `{"bodyParam": true}`)
-	ba := &adapters.Bridge{BridgeType: *bt, Params: params}
-
-	input := cltest.NewRunInputWithResultAndJobRunID("100", jobRunID)
-	result := ba.Perform(input, store)
-	require.NoError(t, result.Error())
-	assert.Equal(t, `{"bodyParam":true,"result":"100"}`, data)
-	assert.Equal(t, fmt.Sprintf(`{"initiator":{"transactionHash":"0x%s","blockHash":"0x%s"}}`, txHashHex, blockHashHex), meta)
 	assert.Equal(t, "Bearer "+bt.OutgoingToken, token)
 }
 

core/adapters/random.go

@@ -3,10 +3,12 @@ package adapters
 import (
 	"fmt"
 
+	"github.com/smartcontractkit/chainlink/core/internal/gethwrappers/generated/solidity_vrf_coordinator_interface"
 	"github.com/smartcontractkit/chainlink/core/services/vrf"
 	"github.com/smartcontractkit/chainlink/core/store"
 	"github.com/smartcontractkit/chainlink/core/store/models"
 	"github.com/smartcontractkit/chainlink/core/store/models/vrfkey"
+	"github.com/smartcontractkit/chainlink/core/utils"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -57,7 +59,8 @@ type Random struct {
 	//
 	// This is just a hex string because Random is instantiated by json.Unmarshal.
 	// (See adapters.For function.)
-	PublicKey string `json:"publicKey"`
+	PublicKey          string              `json:"publicKey"`
+	CoordinatorAddress models.EIP55Address `json:"coordinatorAddress"`
 }
 
 // TaskType returns the type of Adapter.
@@ -67,6 +70,14 @@ func (ra *Random) TaskType() models.TaskType {
 
 // Perform returns the the proof for the VRF output given seed, or an error.
 func (ra *Random) Perform(input models.RunInput, store *store.Store) models.RunOutput {
+	shouldFulfill, err := checkFulfillment(ra, input, store)
+	if err != nil {
+		return models.NewRunOutputError(errors.Wrapf(err, "unable to determine if fulfillment needed"))
+	}
+	if !shouldFulfill {
+		return models.NewRunOutputError(errors.New("randomness request already fulfilled"))
+	}
+
 	key, i, err := getInputs(ra, input, store)
 	if err != nil {
 		return models.NewRunOutputError(err)
@@ -217,3 +228,34 @@ func extractHex(input models.RunInput, key string) ([]byte, error) {
 	}
 	return hexutil.Decode(rawValue.String())
 }
+
+// checkFulfillment checks to see if the randomness request has already been fulfilled or not
+func checkFulfillment(ra *Random, input models.RunInput, store *store.Store) (bool, error) {
+	if len(ra.CoordinatorAddress) == 0 {
+		return true, nil // only perform this check if the optional address field is present
+	}
+
+	contract, err := solidity_vrf_coordinator_interface.NewVRFCoordinator(
+		ra.CoordinatorAddress.Address(),
+		store.EthClient,
+	)
+	if err != nil {
+		return false, errors.Wrapf(
+			err, "unable to create vrf coordinator wrapper, address: %s", ra.CoordinatorAddress.Hex(),
+		)
+	}
+	requestID, err := extractHex(input, "requestID")
+	if err != nil {
+		return false, err
+	}
+	requestID32 := [32]byte{}
+	copy(requestID32[:], requestID)
+
+	callback, err := contract.Callbacks(nil, requestID32)
+	if err != nil {
+		return false, err
+	}
+
+	// If seedAndBlockNumber is non-zero then the response has not yet been fulfilled
+	return !utils.IsEmpty(callback.SeedAndBlockNum[:]), nil
+}

core/adapters/random_test.go

@@ -9,6 +9,9 @@ import (
 	"github.com/smartcontractkit/chainlink/core/adapters"
 	"github.com/smartcontractkit/chainlink/core/internal/cltest"
 	tvrf "github.com/smartcontractkit/chainlink/core/internal/cltest/vrf"
+	"github.com/smartcontractkit/chainlink/core/internal/gethwrappers/generated/solidity_vrf_coordinator_interface"
+	"github.com/smartcontractkit/chainlink/core/internal/mocks"
+	"github.com/smartcontractkit/chainlink/core/services/eth"
 	"github.com/smartcontractkit/chainlink/core/services/vrf"
 	"github.com/smartcontractkit/chainlink/core/store/models"
 	"github.com/smartcontractkit/chainlink/core/utils"
@@ -23,6 +26,7 @@ import (
 // NB: For changes to the VRF solidity code to be reflected here, "go generate"
 // must be run in core/services/vrf.
 func TestRandom_Perform(t *testing.T) {
+	t.Parallel()
 	store, cleanup := cltest.NewStore(t)
 	defer cleanup()
 	publicKey := cltest.StoredVRFKey(t, store)
@@ -63,3 +67,63 @@ func TestRandom_Perform(t *testing.T) {
 	result = adapter.Perform(*input, store)
 	require.Error(t, result.Error(), "must reject if keyHash doesn't match")
 }
+
+func TestRandom_Perform_CheckFulfillment(t *testing.T) {
+	t.Parallel()
+	store, cleanup := cltest.NewStore(t)
+	defer cleanup()
+
+	ethMock := new(mocks.Client)
+	store.EthClient = ethMock
+
+	publicKey := cltest.StoredVRFKey(t, store)
+	address := cltest.NewEIP55Address()
+	hash := utils.MustHash("a random string")
+	seed := big.NewInt(0x10)
+	blockNum := 10
+	jsonInput, err := models.JSON{}.MultiAdd(models.KV{
+		"seed":      utils.Uint64ToHex(seed.Uint64()),
+		"keyHash":   publicKey.MustHash().Hex(),
+		"blockHash": hash.Hex(),
+		"blockNum":  blockNum,
+		"requestID": utils.AddHexPrefix(common.Bytes2Hex([]byte{1, 2, 3})),
+	})
+	require.NoError(t, err)
+	input := models.NewRunInput(uuid.Nil, uuid.Nil, jsonInput, models.RunStatusUnstarted)
+
+	abi := eth.MustGetABI(solidity_vrf_coordinator_interface.VRFCoordinatorABI)
+	registryMock := cltest.NewContractMockReceiver(t, ethMock, abi, address.Address())
+
+	for _, test := range []struct {
+		name                   string
+		addressParamPresent    bool
+		seedAndBlockNumPresent bool
+		shouldFulfill          bool
+	}{
+		{"both missing", false, false, true},
+		{"address missing, seed/block present", false, true, true},
+		{"address present, seed/block missing", true, false, false},
+		{"both present", true, true, true},
+	} {
+		test := test
+		t.Run(test.name, func(tt *testing.T) {
+			adapter := adapters.Random{PublicKey: publicKey.String()}
+			response := solidity_vrf_coordinator_interface.Callbacks{
+				CallbackContract: cltest.NewAddress(),
+				RandomnessFee:    big.NewInt(100),
+			}
+
+			if test.seedAndBlockNumPresent {
+				response.SeedAndBlockNum = [32]byte{1, 2, 3}
+			}
+			if test.addressParamPresent {
+				adapter.CoordinatorAddress = address
+				registryMock.MockResponse("callbacks", response).Once()
+			}
+
+			result := adapter.Perform(*input, store)
+			require.Equal(tt, test.shouldFulfill, result.Error() == nil)
+			ethMock.AssertExpectations(t)
+		})
+	}
+}