Merge branch 'gomips'

.github/workflows/test.yml

@@ -16,19 +16,20 @@ jobs:
         with:
           submodules: true
       - name: unicorn commit hash
+        working-directory: ./diffmips/unicorn
         run: |
           git rev-parse HEAD > /tmp/unicorn-commit-hash.txt
       - name: cached libunicorn
         uses: actions/cache@v3
         with:
           path: |
-            ./unicorn/build
+            ./diffmips/unicorn/build
           key:
             unicorn-build-{{ hashFiles('/tmp/unicorn-commit-hash.txt') }}
           restore-keys: |
             unicorn-build-{{ hashFiles('/tmp/unicorn-commit-hash.txt') }}
       - name: install libunicorn
-        working-directory: .
+        working-directory: ./diffmips
         run: make libunicorn
       - uses: actions/cache@v3
         with:
@@ -45,11 +46,17 @@ jobs:
           key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-${{ matrix.go-version }}-
-      - name: golangci-lint
+      - name: main golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           version: v1.52
-          working-directory: mipsevm
+          working-directory: ./
+          skip-cache: true  # we already have go caching
+      - name: unicorntest golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.52
+          working-directory: ./diffmips/unicorntest
           skip-cache: true  # we already have go caching
       - name: Build examples
         working-directory: ./example
@@ -60,3 +67,6 @@ jobs:
       - name: mipsevm tests
         working-directory: ./mipsevm
         run: go test ./...
+      - name: diffmips tests
+        working-directory: ./diffmips/unicorntest
+        run: go test ./...

.gitignore

@@ -8,3 +8,4 @@ venv
 example/bin
 contracts/out
 state.json
+*.json

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "unicorn"]
-	path = unicorn
+	path = diffmips/unicorn
 	url = https://github.com/unicorn-engine/unicorn.git

Makefile

@@ -1,62 +1,12 @@
 SHELL := /bin/bash
 
-build: submodules libunicorn contracts
+build: contracts
 .PHONY: build
 
-submodules:
-	# CI will checkout submodules on its own (and fails on these commands)
-	if [[ -z "$$GITHUB_ENV" ]]; then \
-		git submodule init; \
-		git submodule update; \
-	fi
-.PHONY: submodules
-
-# Approximation, use `make libunicorn_rebuild` to force.
-unicorn/build: unicorn/CMakeLists.txt
-	mkdir -p unicorn/build
-	cd unicorn/build && cmake .. -DUNICORN_ARCH=mips -DCMAKE_BUILD_TYPE=Release
-	# Not sure why, but the second invocation is needed for fresh installs on MacOS.
-	if [ "$(shell uname)" == "Darwin" ]; then \
-		cd unicorn/build && cmake .. -DUNICORN_ARCH=mips -DCMAKE_BUILD_TYPE=Release; \
-	fi
-
-# Rebuild whenever anything in the unicorn/ directory changes.
-unicorn/build/libunicorn.so: unicorn/build unicorn
-	cd unicorn/build && make -j8
-	# The Go linker / runtime expects dynamic libraries in the unicorn/ dir.
-	find ./unicorn/build -name "libunicorn.*" | xargs -L 1 -I {} cp {} ./unicorn/
-	# Update timestamp on libunicorn.so to make it more recent than the build/ dir.
-	# On Mac this will create a new empty file (dyn libraries are .dylib), but works
-	# fine for the purpose of avoiding recompilation.
-	touch unicorn/build/libunicorn.so
-
-libunicorn: unicorn/build/libunicorn.so
-.PHONY: libunicorn
-
-libunicorn_rebuild:
-	touch unicorn/CMakeLists.txt
-	make libunicorn
-.PHONY: libunicorn_rebuild
-
-# Must be a definition and not a rule, otherwise it gets only called once and
-# not before each test as we wish.
-define clear_cache
-	rm -rf /tmp/cannon
-	mkdir -p /tmp/cannon
-endef
-
-clear_cache:
-	$(call clear_cache)
-.PHONY: clear_cache
-
-clean:
-	rm -f unicorn/libunicorn.*
-.PHONY: clean
-
 contracts:
 	cd contracts && forge build
 .PHONY: contracts
 
-test: libunicorn
+test:
 	cd mipsevm && go test -v ./...
 .PHONY: test

README.md

@@ -24,36 +24,18 @@ contracts -- A MIPS emulator implementation, using merkleized state and a pre-im
 example   -- Example programs that can be run and proven with Cannon.
 extra     -- Extra scripts and legacy contracts, deprecated.
 mipsevm   -- Go tooling to test the onchain MIPS implementation, and generate proof data.
-unicorn   -- Sub-module, used by mipsevm for offchain MIPS emulation.
+diffmips  -- MIPS diff testing, to ensure correctness of the main Cannon implementation, with isolated dependencies. 
 ```
 
 ## Building
 
-### `unicorn`
-
-To build unicorn from source (git sub-module), run:
-```
-make libunicorn
-```
-
 ### `contracts`
 
 The contracts are compiled with [`forge`](https://github.com/foundry-rs/foundry).
 ```
 make contracts
 ```
 
-### `mipsevm`
-
-This requires `unicorn` to be built, as well as the `contracts` for testing.
-
-To test:
-```
-make test
-```
-
-Also see notes in `mipsevm/go.mod` about the Unicorn dependency, if you wish to use Cannon as a Go library.
-
 ## License
 
 MIT, see [`LICENSE`](./LICENSE) file.

cmd/json.go

@@ -2,12 +2,16 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
 )
 
 func loadJSON[X any](inputPath string) (*X, error) {
+	if inputPath == "" {
+		return nil, errors.New("no path specified")
+	}
 	f, err := os.OpenFile(inputPath, os.O_RDONLY, 0)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open file %q: %w", inputPath, err)

cmd/load_elf.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/urfave/cli/v2"
 
-	"cannon/mipsevm"
+	"github.com/ethereum-optimism/cannon/mipsevm"
 )
 
 var (
@@ -28,6 +28,12 @@ var (
 		Value:    "state.json",
 		Required: false,
 	}
+	LoadELFMetaFlag = &cli.PathFlag{
+		Name:     "meta",
+		Usage:    "Write metadata file, for symbol lookup during program execution. None if empty.",
+		Value:    "meta.json",
+		Required: false,
+	}
 )
 
 func LoadELF(ctx *cli.Context) error {
@@ -36,6 +42,9 @@ func LoadELF(ctx *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to open ELF file %q: %w", elfPath, err)
 	}
+	if elfProgram.Machine != elf.EM_MIPS {
+		return fmt.Errorf("ELF is not big-endian MIPS R3000, but got %q", elfProgram.Machine.String())
+	}
 	state, err := mipsevm.LoadELF(elfProgram)
 	if err != nil {
 		return fmt.Errorf("failed to load ELF data into VM state: %w", err)
@@ -53,6 +62,13 @@ func LoadELF(ctx *cli.Context) error {
 			return fmt.Errorf("failed to apply patch %s: %w", typ, err)
 		}
 	}
+	meta, err := mipsevm.MakeMetadata(elfProgram)
+	if err != nil {
+		return fmt.Errorf("failed to compute program metadata: %w", err)
+	}
+	if err := writeJSON[*mipsevm.Metadata](ctx.Path(LoadELFMetaFlag.Name), meta, false); err != nil {
+		return fmt.Errorf("failed to output metadata: %w", err)
+	}
 	return writeJSON[*mipsevm.State](ctx.Path(LoadELFOutFlag.Name), state, true)
 }
 
@@ -65,5 +81,6 @@ var LoadELFCommand = &cli.Command{
 		LoadELFPathFlag,
 		LoadELFPatchFlag,
 		LoadELFOutFlag,
+		LoadELFMetaFlag,
 	},
 }

cmd/matcher.go

@@ -5,7 +5,7 @@ import (
 	"strconv"
 	"strings"
 
-	"cannon/mipsevm"
+	"github.com/ethereum-optimism/cannon/mipsevm"
 )
 
 type StepMatcher func(st *mipsevm.State) bool
@@ -15,6 +15,14 @@ type StepMatcherFlag struct {
 	matcher StepMatcher
 }
 
+func MustStepMatcherFlag(pattern string) *StepMatcherFlag {
+	out := new(StepMatcherFlag)
+	if err := out.Set(pattern); err != nil {
+		panic(err)
+	}
+	return out
+}
+
 func (m *StepMatcherFlag) Set(value string) error {
 	m.repr = value
 	if value == "" || value == "never" {

cmd/run.go

@@ -12,7 +12,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/urfave/cli/v2"
 
-	"cannon/mipsevm"
+	"github.com/ethereum-optimism/cannon/mipsevm"
 	"github.com/ethereum-optimism/cannon/preimage"
 )
 
@@ -62,6 +62,18 @@ var (
 		Value:    new(StepMatcherFlag),
 		Required: false,
 	}
+	RunMetaFlag = &cli.PathFlag{
+		Name:     "meta",
+		Usage:    "path to metadata file for symbol lookup for enhanced debugging info durign execution.",
+		Value:    "meta.json",
+		Required: false,
+	}
+	RunInfoAtFlag = &cli.GenericFlag{
+		Name:     "info-at",
+		Usage:    "step pattern to print info at: " + patternHelp,
+		Value:    MustStepMatcherFlag("%1000"),
+		Required: false,
+	}
 )
 
 type Proof struct {
@@ -170,13 +182,7 @@ func Run(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	mu, err := mipsevm.NewUnicorn()
-	if err != nil {
-		return fmt.Errorf("failed to create unicorn emulator: %w", err)
-	}
-	if err := mipsevm.LoadUnicorn(state, mu); err != nil {
-		return fmt.Errorf("failed to load state into unicorn emulator: %w", err)
-	}
+
 	l := Logger(os.Stderr, log.LvlInfo)
 	outLog := &mipsevm.LoggingWriter{Name: "program std-out", Log: l}
 	errLog := &mipsevm.LoggingWriter{Name: "program std-err", Log: l}
@@ -206,11 +212,21 @@ func Run(ctx *cli.Context) error {
 	stopAt := ctx.Generic(RunStopAtFlag.Name).(*StepMatcherFlag).Matcher()
 	proofAt := ctx.Generic(RunProofAtFlag.Name).(*StepMatcherFlag).Matcher()
 	snapshotAt := ctx.Generic(RunSnapshotAtFlag.Name).(*StepMatcherFlag).Matcher()
-
-	us, err := mipsevm.NewUnicornState(mu, state, po, outLog, errLog)
-	if err != nil {
-		return fmt.Errorf("failed to setup instrumented VM state: %w", err)
+	infoAt := ctx.Generic(RunInfoAtFlag.Name).(*StepMatcherFlag).Matcher()
+
+	var meta *mipsevm.Metadata
+	if metaPath := ctx.Path(RunMetaFlag.Name); metaPath == "" {
+		l.Info("no metadata file specified, defaulting to empty metadata")
+		meta = &mipsevm.Metadata{Symbols: nil} // provide empty metadata by default
+	} else {
+		if m, err := loadJSON[mipsevm.Metadata](metaPath); err != nil {
+			return fmt.Errorf("failed to load metadata: %w", err)
+		} else {
+			meta = m
+		}
 	}
+
+	us := mipsevm.NewInstrumentedState(state, po, outLog, errLog)
 	proofFmt := ctx.String(RunProofFmtFlag.Name)
 	snapshotFmt := ctx.String(RunSnapshotFmtFlag.Name)
 
@@ -222,12 +238,18 @@ func Run(ctx *cli.Context) error {
 	for !state.Exited {
 		step := state.Step
 
-		//if infoAt(state) {
-		//	s := lookupSymbol(state.PC)
-		//	var sy elf.Symbol
-		//	l.Info("", "insn", state.Memory.GetMemory(state.PC), "pc", state.PC, "symbol", sy.Name)
-		//	// print name
-		//}
+		name := meta.LookupSymbol(state.PC)
+		if infoAt(state) {
+			l.Info("processing",
+				"step", step,
+				"pc", mipsevm.HexU32(state.PC),
+				"insn", mipsevm.HexU32(state.Memory.GetMemory(state.PC)),
+				"name", name,
+			)
+		}
+		if name == "runtime.notesleep" { // don't loop forever when we get stuck because of an unexpected bad program
+			return fmt.Errorf("got stuck in Go sleep at step %d", step)
+		}
 
 		if stopAt(state) {
 			break
@@ -262,7 +284,7 @@ func Run(ctx *cli.Context) error {
 				return fmt.Errorf("failed to write proof data: %w", err)
 			}
 		} else {
-			_, err = us.Step(false)
+			_, err = stepFn(false)
 			if err != nil {
 				return fmt.Errorf("failed at step %d (PC: %08x): %w", step, state.PC, err)
 			}
@@ -288,5 +310,7 @@ var RunCommand = &cli.Command{
 		RunSnapshotAtFlag,
 		RunSnapshotFmtFlag,
 		RunStopAtFlag,
+		RunMetaFlag,
+		RunInfoAtFlag,
 	},
 }