FEAT : Benchmark for integration tests

integration_tests/BenchMarks.txt

@@ -0,0 +1,14 @@
+====================================================================
+| File                              | PythonVM (ms)   | GoVM (ms)  |
+====================================================================
+| fib.cairo                         | 744             | 130        |
+| squash_dict.cairo                 | 694             | 138        |
+| set_add.cairo                     | 713             | 129        |
+| simple.cairo                      | 729             | 124        |
+| test.starknet_with_keccak.cairo   | 1682            | 143        |
+| alloc.cairo                       | 716             | 127        |
+| dict.cairo                        | 714             | 123        |
+| factorial.cairo                   | 1312            | 119        |
+| find_element.cairo                | 668             | 132        |
+| hintrefs.cairo                    | 750             | 144        |
+====================================================================

integration_tests/cairozero_test.go

@@ -5,8 +5,10 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/NethermindEth/cairo-vm-go/pkg/vm"
 	"github.com/consensys/gnark-crypto/ecc/stark-curve/fp"
@@ -69,6 +71,8 @@ func TestCairoZeroFiles(t *testing.T) {
 	filter := Filter{}
 	filter.init()
 
+	benchmarkMap := make(map[string][2]int)
+
 	for _, dirEntry := range testFiles {
 		if dirEntry.IsDir() || isGeneratedFile(dirEntry.Name()) {
 			continue
@@ -93,18 +97,20 @@ func TestCairoZeroFiles(t *testing.T) {
 			continue
 		}
 
-		pyTraceFile, pyMemoryFile, err := runPythonVm(dirEntry.Name(), compiledOutput)
+		elapsed_py, pyTraceFile, pyMemoryFile, err := runPythonVm(dirEntry.Name(), compiledOutput)
 		if err != nil {
 			t.Error(err)
 			continue
 		}
 
-		traceFile, memoryFile, _, err := runVm(compiledOutput)
+		elapsed_go, traceFile, memoryFile, _, err := runVm(compiledOutput)
 		if err != nil {
 			t.Error(err)
 			continue
 		}
 
+		benchmarkMap[dirEntry.Name()] = [2]int{int(elapsed_py.Milliseconds()), int(elapsed_go.Milliseconds())}
+
 		pyTrace, pyMemory, err := decodeProof(pyTraceFile, pyMemoryFile)
 		if err != nil {
 			t.Error(err)
@@ -127,10 +133,73 @@ func TestCairoZeroFiles(t *testing.T) {
 		}
 	}
 
+	WriteBenchMarksToFile(benchmarkMap)
+
 	clean(root1)
 	clean(root2)
 }
 
+// Save the Benchmarks for the integration tests in `BenchMarks.txt`
+func WriteBenchMarksToFile(benchmarkMap map[string][2]int) {
+	headers := []string{"File", "PythonVM (ms)", "GoVM (ms)"}
+	columnWidths := []int{33, 15, 10}
+
+	totalWidth := 0
+	for _, width := range columnWidths {
+		totalWidth += width + 3
+	}
+
+	totalWidth += 1
+
+	border := strings.Repeat("=", totalWidth)
+
+	var sb strings.Builder
+
+	sb.WriteString(border + "\n")
+	sb.WriteString(formatRow(headers, columnWidths) + "\n")
+	sb.WriteString(border + "\n")
+
+	for key, values := range benchmarkMap {
+		row := []string{key}
+		for _, value := range values {
+			row = append(row, strconv.Itoa(value))
+		}
+		sb.WriteString(formatRow(row, columnWidths) + "\n")
+	}
+
+	sb.WriteString(border + "\n")
+
+	fileName := "BenchMarks.txt"
+	file, err := os.Create(fileName)
+	if err != nil {
+		fmt.Println("Error creating file: ", err)
+		return
+	}
+	defer file.Close()
+
+	_, err = file.WriteString(sb.String())
+	if err != nil {
+		fmt.Println("Error writing to file: ", err)
+	} else {
+		fmt.Println("Benchmarks successfully written to:", fileName)
+	}
+}
+
+// format a row with borders and spaces based on the column widths
+func formatRow(row []string, widths []int) string {
+	var sb strings.Builder
+	sb.WriteString("|")
+	for i, col := range row {
+		sb.WriteString(" " + padRight(col, widths[i]) + " |")
+	}
+	return sb.String()
+}
+
+// pad a string with spaces to the right to match the given width
+func padRight(str string, width int) string {
+	return fmt.Sprintf("%-"+fmt.Sprintf("%d", width)+"s", str)
+}
+
 const (
 	compiledSuffix = "_compiled.json"
 	pyTraceSuffix  = "_py_trace"
@@ -168,7 +237,7 @@ func compileZeroCode(path string) (string, error) {
 
 // given a path to a compiled cairo zero file, execute it using the
 // python vm and returns the trace and memory files location
-func runPythonVm(testFilename, path string) (string, string, error) {
+func runPythonVm(testFilename, path string) (time.Duration, string, string, error) {
 	traceOutput := swapExtenstion(path, pyTraceSuffix)
 	memoryOutput := swapExtenstion(path, pyMemorySuffix)
 
@@ -193,19 +262,24 @@ func runPythonVm(testFilename, path string) (string, string, error) {
 
 	cmd := exec.Command("cairo-run", args...)
 
+	start := time.Now()
+
 	res, err := cmd.CombinedOutput()
+
+	elapsed := time.Since(start)
+
 	if err != nil {
-		return "", "", fmt.Errorf(
+		return 0, "", "", fmt.Errorf(
 			"cairo-run %s: %w\n%s", path, err, string(res),
 		)
 	}
 
-	return traceOutput, memoryOutput, nil
+	return elapsed, traceOutput, memoryOutput, nil
 }
 
 // given a path to a compiled cairo zero file, execute
 // it using our vm
-func runVm(path string) (string, string, string, error) {
+func runVm(path string) (time.Duration, string, string, string, error) {
 	traceOutput := swapExtenstion(path, traceSuffix)
 	memoryOutput := swapExtenstion(path, memorySuffix)
 
@@ -232,14 +306,19 @@ func runVm(path string) (string, string, string, error) {
 		path,
 	)
 
+	start := time.Now()
+
 	res, err := cmd.CombinedOutput()
+
+	elapsed := time.Since(start)
+
 	if err != nil {
-		return "", "", string(res), fmt.Errorf(
+		return 0, "", "", string(res), fmt.Errorf(
 			"cairo-vm run %s: %w\n%s", path, err, string(res),
 		)
 	}
 
-	return traceOutput, memoryOutput, string(res), nil
+	return elapsed, traceOutput, memoryOutput, string(res), nil
 
 }
 
@@ -318,7 +397,7 @@ func TestFailingRangeCheck(t *testing.T) {
 	compiledOutput, err := compileZeroCode("./builtin_tests/range_check.small.cairo")
 	require.NoError(t, err)
 
-	_, _, _, err = runVm(compiledOutput)
+	_, _, _, _, err = runVm(compiledOutput)
 	require.ErrorContains(t, err, "check write: 2**128 <")
 
 	clean("./builtin_tests/")
@@ -328,7 +407,7 @@ func TestBitwise(t *testing.T) {
 	compiledOutput, err := compileZeroCode("./builtin_tests/bitwise_builtin_test.starknet_with_keccak.cairo")
 	require.NoError(t, err)
 
-	_, _, _, err = runVm(compiledOutput)
+	_, _, _, _, err = runVm(compiledOutput)
 	require.NoError(t, err)
 
 	clean("./builtin_tests/")
@@ -338,7 +417,7 @@ func TestPedersen(t *testing.T) {
 	compiledOutput, err := compileZeroCode("./builtin_tests/pedersen_test.small.cairo")
 	require.NoError(t, err)
 
-	_, _, output, err := runVm(compiledOutput)
+	_, _, _, output, err := runVm(compiledOutput)
 	require.NoError(t, err)
 	require.Contains(t, output, "Program output:\n  2089986280348253421170679821480865132823066470938446095505822317253594081284")
 
@@ -349,7 +428,7 @@ func TestPoseidon(t *testing.T) {
 	compiledOutput, err := compileZeroCode("./builtin_tests/poseidon_test.starknet_with_keccak.cairo")
 	require.NoError(t, err)
 
-	_, _, output, err := runVm(compiledOutput)
+	_, _, _, output, err := runVm(compiledOutput)
 	require.NoError(t, err)
 	require.Contains(t, output, "Program output:\n  442682200349489646213731521593476982257703159825582578145778919623645026501\n  2233832504250924383748553933071188903279928981104663696710686541536735838182\n  2512222140811166287287541003826449032093371832913959128171347018667852712082\n")
 	require.Contains(t, output, "3016509350703874362933565866148509373957094754875411937434637891208784994231\n  3015199725895936530535660185611704199044060139852899280809302949374221328865\n  3062378460350040063467318871602229987911299744598148928378797834245039883769\n")
@@ -360,7 +439,7 @@ func TestECDSA(t *testing.T) {
 	compiledOutput, err := compileZeroCode("./builtin_tests/ecdsa_test.starknet_with_keccak.cairo")
 	require.NoError(t, err)
 
-	_, _, _, err = runVm(compiledOutput)
+	_, _, _, _, err = runVm(compiledOutput)
 	require.NoError(t, err)
 
 	clean("./builtin_tests/")
@@ -370,7 +449,7 @@ func TestEcOp(t *testing.T) {
 	compiledOutput, err := compileZeroCode("./builtin_tests/ecop.starknet_with_keccak.cairo")
 	require.NoError(t, err)
 
-	_, _, _, err = runVm(compiledOutput)
+	_, _, _, _, err = runVm(compiledOutput)
 	// todo(rodro): This test is failing due to the lack of hint processing. It should be address soon
 	require.Error(t, err)
 
@@ -381,7 +460,7 @@ func TestKeccak(t *testing.T) {
 	compiledOutput, err := compileZeroCode("./builtin_tests/keccak_test.starknet_with_keccak.cairo")
 	require.NoError(t, err)
 
-	_, _, output, err := runVm(compiledOutput)
+	_, _, _, output, err := runVm(compiledOutput)
 	require.NoError(t, err)
 	require.Contains(t, output, "Program output:\n  1304102964824333531548398680304964155037696012322029952943772\n  688749063493959345342507274897412933692859993314608487848187\n  986714560881445649520443980361539218531403996118322524237197\n  1184757872753521629808292433475729390634371625298664050186717\n  719230200744669084408849842242045083289669818920073250264351\n  1543031433416778513637578850638598357854418012971636697855068\n  63644822371671650271181212513090078620238279557402571802224\n  879446821229338092940381117330194802032344024906379963157761\n")