add support for coverage reports

runtime/coverage.go

@@ -0,0 +1,61 @@
+/*
+ * Cadence - The resource-oriented smart contract programming language
+ *
+ * Copyright 2019-2020 Dapper Labs, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package runtime
+
+import (
+	"github.com/onflow/cadence/runtime/ast"
+)
+
+// LocationCoverage records coverage information for a location
+//
+type LocationCoverage struct {
+	LineHits map[int]int `json:"line_hits"`
+}
+
+func (c *LocationCoverage) AddLineHit(line int) {
+	c.LineHits[line]++
+}
+
+func NewLocationCoverage() *LocationCoverage {
+	return &LocationCoverage{
+		LineHits: map[int]int{},
+	}
+}
+
+// CoverageReport is a collection of coverage per location
+//
+type CoverageReport struct {
+	Coverage map[ast.LocationID]*LocationCoverage `json:"coverage"`
+}
+
+func (r *CoverageReport) AddLineHit(location ast.Location, line int) {
+	locationID := location.ID()
+	locationCoverage := r.Coverage[locationID]
+	if locationCoverage == nil {
+		locationCoverage = NewLocationCoverage()
+		r.Coverage[locationID] = locationCoverage
+	}
+	locationCoverage.AddLineHit(line)
+}
+
+func NewCoverageReport() *CoverageReport {
+	return &CoverageReport{
+		Coverage: map[ast.LocationID]*LocationCoverage{},
+	}
+}

runtime/coverage_test.go

@@ -0,0 +1,109 @@
+/*
+ * Cadence - The resource-oriented smart contract programming language
+ *
+ * Copyright 2019-2020 Dapper Labs, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package runtime
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/onflow/cadence"
+)
+
+func TestRuntimeCoverage(t *testing.T) {
+
+	t.Parallel()
+
+	runtime := NewInterpreterRuntime()
+
+	importedScript := []byte(`
+      pub fun answer(): Int {
+        var i = 0
+        while i < 42 {
+          i = i + 1
+        }
+        return i
+      }
+    `)
+
+	script := []byte(`
+      import "imported"
+
+      pub fun main(): Int {
+          let answer = answer()
+          if answer != 42 {
+            panic("?!")
+          }
+          return answer
+        }
+    `)
+
+	runtimeInterface := &testRuntimeInterface{
+		getCode: func(location Location) (bytes []byte, err error) {
+			switch location {
+			case StringLocation("imported"):
+				return importedScript, nil
+			default:
+				return nil, fmt.Errorf("unknown import location: %s", location)
+			}
+		},
+	}
+
+	nextTransactionLocation := newTransactionLocationGenerator()
+
+	coverageReport := NewCoverageReport()
+
+	runtime.SetCoverageReport(coverageReport)
+
+	value, err := runtime.ExecuteScript(script, nil, runtimeInterface, nextTransactionLocation())
+	require.NoError(t, err)
+
+	assert.Equal(t, cadence.NewInt(42), value)
+
+	actual, err := json.Marshal(coverageReport)
+	require.NoError(t, err)
+
+	require.JSONEq(t,
+		`
+        {
+          "coverage": {
+            "S.imported": {
+              "line_hits": {
+                "3": 1,
+                "4": 1,
+                "5": 42,
+                "7": 1
+              }
+            },
+            "t.00": {
+              "line_hits": {
+                "5": 1,
+                "6": 1,
+                "9": 1
+              }
+            }
+          }
+        }
+        `,
+		string(actual),
+	)
+}

runtime/runtime.go

@@ -56,6 +56,11 @@ type Runtime interface {
 	//
 	// This function returns an error if the program contains any syntax or semantic errors.
 	ParseAndCheckProgram(code []byte, runtimeInterface Interface, location Location) (*sema.Checker, error)
+
+	// SetCoverageReport activates reporting coverage in the given report.
+	// Passing nil disables coverage reporting (default).
+	//
+	SetCoverageReport(coverageReport *CoverageReport)
 }
 
 var typeDeclarations = append(
@@ -108,7 +113,9 @@ func reportMetric(
 }
 
 // interpreterRuntime is a interpreter-based version of the Flow runtime.
-type interpreterRuntime struct{}
+type interpreterRuntime struct {
+	coverageReport *CoverageReport
+}
 
 type Option func(Runtime)
 
@@ -121,6 +128,10 @@ func NewInterpreterRuntime(options ...Option) Runtime {
 	return runtime
 }
 
+func (r *interpreterRuntime) SetCoverageReport(coverageReport *CoverageReport) {
+	r.coverageReport = coverageReport
+}
+
 func (r *interpreterRuntime) ExecuteScript(
 	script []byte,
 	arguments [][]byte,
@@ -647,6 +658,9 @@ func (r *interpreterRuntime) newInterpreter(
 		interpreter.WithImportLocationHandler(
 			r.importLocationHandler(runtimeInterface),
 		),
+		interpreter.WithOnStatementHandler(
+			r.onStatementHandler(),
+		),
 	}
 
 	defaultOptions = append(defaultOptions,
@@ -1734,6 +1748,18 @@ func (r *interpreterRuntime) newAuthAccountContractsRemoveFunction(
 	)
 }
 
+func (r *interpreterRuntime) onStatementHandler() interpreter.OnStatementFunc {
+	if r.coverageReport == nil {
+		return nil
+	}
+
+	return func(statement *interpreter.Statement) {
+		location := statement.Interpreter.Checker.Location
+		line := statement.Statement.StartPosition().Line
+		r.coverageReport.AddLineHit(location, line)
+	}
+}
+
 func compositeTypesToIDValues(types []*sema.CompositeType) *interpreter.ArrayValue {
 	typeIDValues := make([]interpreter.Value, len(types))