driver, runner and CI for interoperable test-vector based testing.

.circleci/config.yml

@@ -113,7 +113,7 @@ jobs:
   test: &test
     description: |
       Run tests with gotestsum.
-    parameters:
+    parameters: &test-params
       executor:
         type: executor
         default: golang
@@ -161,6 +161,7 @@ jobs:
           name: go test
           environment:
             LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >>
+            SKIP_CONFORMANCE: "1"
           command: |
             mkdir -p /tmp/test-reports/<< parameters.test-suite-name >>
             mkdir -p /tmp/test-artifacts
@@ -191,6 +192,63 @@ jobs:
     <<: *test
   test-window-post:
     <<: *test
+  test-conformance:
+    description: |
+      Run tests using a corpus of interoperable test vectors for Filecoin 
+      implementations to test their correctness and compliance with the Filecoin
+      specifications.
+    parameters:
+      <<: *test-params
+      vectors-branch:
+        type: string
+        default: ""
+        description: |
+          Branch on github.com/filecoin-project/test-vectors to checkout and
+          test with. If empty (the default) the commit defined by the git
+          submodule is used.
+    executor: << parameters.executor >>
+    steps:
+      - install-deps
+      - prepare
+      - run:
+          command: make deps lotus
+          no_output_timeout: 30m
+      - download-params
+      - when:
+          condition:
+            not:
+              equal: [ "", << parameters.vectors-branch >> ]
+          steps:
+            - run:
+                name: checkout vectors branch
+                command: |
+                  cd extern/test-vectors
+                  git fetch
+                  git checkout origin/<< parameters.vectors-branch >>
+            - run:
+                name: go get vectors branch
+                command: go get github.com/filecoin-project/test-vectors@<< parameters.vectors-branch >>
+      - go/install-gotestsum:
+          gobin: $HOME/.local/bin
+          version: 0.5.2
+      - run:
+          name: go test
+          environment:
+            SKIP_CONFORMANCE: "0"
+          command: |
+            mkdir -p /tmp/test-reports
+            mkdir -p /tmp/test-artifacts
+            gotestsum \
+              --format pkgname-and-test-fails \
+              --junitfile /tmp/test-reports/junit.xml \
+              -- \
+              -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
+            go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
+          no_output_timeout: 30m
+      - store_test_results:
+          path: /tmp/test-reports
+      - store_artifacts:
+          path: /tmp/test-artifacts/conformance-coverage.html
 
   build-macos:
     description: build darwin lotus binary
@@ -356,6 +414,14 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+$/
+      - test-conformance:
+          test-suite-name: conformance
+          packages: "./conformance"
+      - test-conformance:
+          name: test-conformance-bleeding-edge
+          test-suite-name: conformance-bleeding-edge
+          packages: "./conformance"
+          vectors-branch: master
       - build-debug
       - build-all:
           requires:

.gitmodules

@@ -5,3 +5,6 @@
 [submodule "extern/serialization-vectors"]
 	path = extern/serialization-vectors
 	url = https://github.com/filecoin-project/serialization-vectors
+[submodule "extern/test-vectors"]
+	path = extern/test-vectors
+	url = https://github.com/filecoin-project/test-vectors.git

conformance/driver.go

@@ -0,0 +1,71 @@
+package conformance
+
+import (
+	"context"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/puppet"
+
+	"github.com/filecoin-project/test-vectors/chaos"
+	"github.com/filecoin-project/test-vectors/schema"
+
+	"github.com/ipfs/go-cid"
+)
+
+var (
+	// BaseFee to use in the VM.
+	// TODO make parametrisable through vector.
+	BaseFee = abi.NewTokenAmount(100)
+)
+
+type Driver struct {
+	ctx    context.Context
+	vector *schema.TestVector
+}
+
+func NewDriver(ctx context.Context, vector *schema.TestVector) *Driver {
+	return &Driver{ctx: ctx, vector: vector}
+}
+
+// ExecuteMessage executes a conformance test vector message in a temporary VM.
+func (d *Driver) ExecuteMessage(msg *types.Message, preroot cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) (*vm.ApplyRet, cid.Cid, error) {
+	vmOpts := &vm.VMOpts{
+		StateBase:      preroot,
+		Epoch:          epoch,
+		Rand:           &testRand{}, // TODO always succeeds; need more flexibility.
+		Bstore:         bs,
+		Syscalls:       mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
+		CircSupplyCalc: nil,
+		BaseFee:        BaseFee,
+	}
+
+	lvm, err := vm.NewVM(vmOpts)
+	if err != nil {
+		return nil, cid.Undef, err
+	}
+
+	invoker := vm.NewInvoker()
+
+	// add support for the puppet and chaos actors.
+	if puppetOn, ok := d.vector.Selector["puppet_actor"]; ok && puppetOn == "true" {
+		invoker.Register(puppet.PuppetActorCodeID, puppet.Actor{}, puppet.State{})
+	}
+	if chaosOn, ok := d.vector.Selector["chaos_actor"]; ok && chaosOn == "true" {
+		invoker.Register(chaos.ChaosActorCodeCID, chaos.Actor{}, chaos.State{})
+	}
+
+	lvm.SetInvoker(invoker)
+
+	ret, err := lvm.ApplyMessage(d.ctx, msg)
+	if err != nil {
+		return nil, cid.Undef, err
+	}
+
+	root, err := lvm.Flush(d.ctx)
+	return ret, root, err
+}