From 6d939e42512553336c04b339d6edc3f466990e1c Mon Sep 17 00:00:00 2001 From: Shourya Goel Date: Fri, 28 Jun 2024 22:29:38 +0530 Subject: [PATCH] FEAT : Benchmark for integration tests (#477) * Added Benchmark for integration tests * Relocated time measuring code * nit * Minor design changes * Using tabwriter --- integration_tests/BenchMarks.txt | 81 +++++++++++++++++++++ integration_tests/cairozero_test.go | 105 ++++++++++++++++++++++++---- 2 files changed, 171 insertions(+), 15 deletions(-) create mode 100644 integration_tests/BenchMarks.txt diff --git a/integration_tests/BenchMarks.txt b/integration_tests/BenchMarks.txt new file mode 100644 index 00000000..460afc40 --- /dev/null +++ b/integration_tests/BenchMarks.txt @@ -0,0 +1,81 @@ +=========================================================================================================================== +| File | PythonVM (ms) | GoVM (ms) | +=========================================================================================================================== +| get_point_from_x.small.cairo | 971 | 125 | +--------------------------------------------------------------------------------------------------------------------------- +| is_quad_residue.small.cairo | 963 | 152 | +--------------------------------------------------------------------------------------------------------------------------- +| reduce_v1.small.cairo | 931 | 130 | +--------------------------------------------------------------------------------------------------------------------------- +| uint256_unsigned_div_rem.small.cairo | 1061 | 132 | +--------------------------------------------------------------------------------------------------------------------------- +| set_add.small.cairo | 749 | 117 | +--------------------------------------------------------------------------------------------------------------------------- +| split64.small.cairo | 1014 | 117 | +--------------------------------------------------------------------------------------------------------------------------- +| uint256_signedNN.small.cairo | 752 | 114 | +--------------------------------------------------------------------------------------------------------------------------- +| split_felt.small.cairo | 841 | 145 | +--------------------------------------------------------------------------------------------------------------------------- +| split_int.small.cairo | 1037 | 133 | +--------------------------------------------------------------------------------------------------------------------------- +| uint256_add.small.cairo | 921 | 110 | +--------------------------------------------------------------------------------------------------------------------------- +| signed_div_rem.small.cairo | 816 | 117 | +--------------------------------------------------------------------------------------------------------------------------- +| sqrt.small.cairo | 960 | 119 | +--------------------------------------------------------------------------------------------------------------------------- +| uint256_sqrt.small.cairo | 1195 | 115 | +--------------------------------------------------------------------------------------------------------------------------- +| assert_not_zero.cairo | 586 | 100 | +--------------------------------------------------------------------------------------------------------------------------- +| blake.starknet_with_keccak.cairo | 2498 | 144 | +--------------------------------------------------------------------------------------------------------------------------- +| import_secp256R1P.small.cairo | 699 | 123 | +--------------------------------------------------------------------------------------------------------------------------- +| memcpy.cairo | 704 | 124 | +--------------------------------------------------------------------------------------------------------------------------- +| unsigned_div_rem.small.cairo | 843 | 118 | +--------------------------------------------------------------------------------------------------------------------------- +| factorial.cairo | 1161 | 123 | +--------------------------------------------------------------------------------------------------------------------------- +| hintrefs.cairo | 676 | 127 | +--------------------------------------------------------------------------------------------------------------------------- +| cmp.small.cairo | 897 | 110 | +--------------------------------------------------------------------------------------------------------------------------- +| dict_squash.small.cairo | 1240 | 166 | +--------------------------------------------------------------------------------------------------------------------------- +| div_mod_n.small.cairo | 921 | 113 | +--------------------------------------------------------------------------------------------------------------------------- +| find_element.small.cairo | 939 | 124 | +--------------------------------------------------------------------------------------------------------------------------- +| uint256_mul_div_mod.small.cairo | 1032 | 119 | +--------------------------------------------------------------------------------------------------------------------------- +| verify_zero.small.cairo | 781 | 112 | +--------------------------------------------------------------------------------------------------------------------------- +| ec.small.cairo | 1042 | 129 | +--------------------------------------------------------------------------------------------------------------------------- +| memset.cairo | 701 | 128 | +--------------------------------------------------------------------------------------------------------------------------- +| pow.small.cairo | 887 | 112 | +--------------------------------------------------------------------------------------------------------------------------- +| search_sorted_lower.small.cairo | 836 | 134 | +--------------------------------------------------------------------------------------------------------------------------- +| alloc.cairo | 666 | 126 | +--------------------------------------------------------------------------------------------------------------------------- +| simple.cairo | 596 | 110 | +--------------------------------------------------------------------------------------------------------------------------- +| assert_250_bits.small.cairo | 979 | 120 | +--------------------------------------------------------------------------------------------------------------------------- +| assert_not_equal.cairo | 638 | 101 | +--------------------------------------------------------------------------------------------------------------------------- +| dict.cairo | 678 | 115 | +--------------------------------------------------------------------------------------------------------------------------- +| is_positive.small.cairo | 884 | 128 | +--------------------------------------------------------------------------------------------------------------------------- +| usort.small.cairo | 797 | 119 | +--------------------------------------------------------------------------------------------------------------------------- +| verify_ecdsa_signature.small.cairo | 758 | 117 | +--------------------------------------------------------------------------------------------------------------------------- +| fib.cairo | 634 | 116 | +=========================================================================================================================== diff --git a/integration_tests/cairozero_test.go b/integration_tests/cairozero_test.go index 0564ffe2..04653c83 100644 --- a/integration_tests/cairozero_test.go +++ b/integration_tests/cairozero_test.go @@ -5,8 +5,11 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" + "text/tabwriter" + "time" "github.com/NethermindEth/cairo-vm-go/pkg/vm" "github.com/consensys/gnark-crypto/ecc/stark-curve/fp" @@ -69,6 +72,8 @@ func TestCairoZeroFiles(t *testing.T) { filter := Filter{} filter.init() + benchmarkMap := make(map[string][2]int) + for _, dirEntry := range testFiles { if dirEntry.IsDir() || isGeneratedFile(dirEntry.Name()) { continue @@ -91,18 +96,20 @@ func TestCairoZeroFiles(t *testing.T) { continue } - pyTraceFile, pyMemoryFile, err := runPythonVm(dirEntry.Name(), compiledOutput) + elapsedPy, pyTraceFile, pyMemoryFile, err := runPythonVm(dirEntry.Name(), compiledOutput) if err != nil { t.Error(err) continue } - traceFile, memoryFile, _, err := runVm(compiledOutput) + elapsedGo, traceFile, memoryFile, _, err := runVm(compiledOutput) if err != nil { t.Error(err) continue } + benchmarkMap[dirEntry.Name()] = [2]int{int(elapsedPy.Milliseconds()), int(elapsedGo.Milliseconds())} + pyTrace, pyMemory, err := decodeProof(pyTraceFile, pyMemoryFile) if err != nil { t.Error(err) @@ -125,10 +132,68 @@ func TestCairoZeroFiles(t *testing.T) { } } + WriteBenchMarksToFile(benchmarkMap) + clean(root1) clean(root2) } +// Save the Benchmarks for the integration tests in `BenchMarks.txt` +func WriteBenchMarksToFile(benchmarkMap map[string][2]int) { + totalWidth := 123 + + border := strings.Repeat("=", totalWidth) + separator := strings.Repeat("-", totalWidth) + + var sb strings.Builder + w := tabwriter.NewWriter(&sb, 40, 0, 0, ' ', tabwriter.Debug) + + sb.WriteString(border + "\n") + fmt.Fprintln(w, "| File \t PythonVM (ms) \t GoVM (ms) \t") + w.Flush() + sb.WriteString(border + "\n") + + iterator := 0 + totalFiles := len(benchmarkMap) + + for key, values := range benchmarkMap { + row := "| " + key + "\t " + + for iter, value := range values { + row = row + strconv.Itoa(value) + "\t" + if iter == 0 { + row = row + " " + } + } + + fmt.Fprintln(w, row) + w.Flush() + + if iterator < totalFiles-1 { + sb.WriteString(separator + "\n") + } + + iterator++ + } + + sb.WriteString(border + "\n") + + fileName := "BenchMarks.txt" + file, err := os.Create(fileName) + if err != nil { + fmt.Println("Error creating file: ", err) + return + } + defer file.Close() + + _, err = file.WriteString(sb.String()) + if err != nil { + fmt.Println("Error writing to file: ", err) + } else { + fmt.Println("Benchmarks successfully written to:", fileName) + } +} + const ( compiledSuffix = "_compiled.json" pyTraceSuffix = "_py_trace" @@ -166,7 +231,7 @@ func compileZeroCode(path string) (string, error) { // given a path to a compiled cairo zero file, execute it using the // python vm and returns the trace and memory files location -func runPythonVm(testFilename, path string) (string, string, error) { +func runPythonVm(testFilename, path string) (time.Duration, string, string, error) { traceOutput := swapExtenstion(path, pyTraceSuffix) memoryOutput := swapExtenstion(path, pyMemorySuffix) @@ -191,19 +256,24 @@ func runPythonVm(testFilename, path string) (string, string, error) { cmd := exec.Command("cairo-run", args...) + start := time.Now() + res, err := cmd.CombinedOutput() + + elapsed := time.Since(start) + if err != nil { - return "", "", fmt.Errorf( + return 0, "", "", fmt.Errorf( "cairo-run %s: %w\n%s", path, err, string(res), ) } - return traceOutput, memoryOutput, nil + return elapsed, traceOutput, memoryOutput, nil } // given a path to a compiled cairo zero file, execute // it using our vm -func runVm(path string) (string, string, string, error) { +func runVm(path string) (time.Duration, string, string, string, error) { traceOutput := swapExtenstion(path, traceSuffix) memoryOutput := swapExtenstion(path, memorySuffix) @@ -230,14 +300,19 @@ func runVm(path string) (string, string, string, error) { path, ) + start := time.Now() + res, err := cmd.CombinedOutput() + + elapsed := time.Since(start) + if err != nil { - return "", "", string(res), fmt.Errorf( + return 0, "", "", string(res), fmt.Errorf( "cairo-vm run %s: %w\n%s", path, err, string(res), ) } - return traceOutput, memoryOutput, string(res), nil + return elapsed, traceOutput, memoryOutput, string(res), nil } @@ -316,7 +391,7 @@ func TestFailingRangeCheck(t *testing.T) { compiledOutput, err := compileZeroCode("./builtin_tests/range_check.small.cairo") require.NoError(t, err) - _, _, _, err = runVm(compiledOutput) + _, _, _, _, err = runVm(compiledOutput) require.ErrorContains(t, err, "check write: 2**128 <") clean("./builtin_tests/") @@ -326,7 +401,7 @@ func TestBitwise(t *testing.T) { compiledOutput, err := compileZeroCode("./builtin_tests/bitwise_builtin_test.starknet_with_keccak.cairo") require.NoError(t, err) - _, _, _, err = runVm(compiledOutput) + _, _, _, _, err = runVm(compiledOutput) require.NoError(t, err) clean("./builtin_tests/") @@ -336,7 +411,7 @@ func TestPedersen(t *testing.T) { compiledOutput, err := compileZeroCode("./builtin_tests/pedersen_test.small.cairo") require.NoError(t, err) - _, _, output, err := runVm(compiledOutput) + _, _, _, output, err := runVm(compiledOutput) require.NoError(t, err) require.Contains(t, output, "Program output:\n 2089986280348253421170679821480865132823066470938446095505822317253594081284") @@ -347,7 +422,7 @@ func TestPoseidon(t *testing.T) { compiledOutput, err := compileZeroCode("./builtin_tests/poseidon_test.starknet_with_keccak.cairo") require.NoError(t, err) - _, _, output, err := runVm(compiledOutput) + _, _, _, output, err := runVm(compiledOutput) require.NoError(t, err) require.Contains(t, output, "Program output:\n 442682200349489646213731521593476982257703159825582578145778919623645026501\n 2233832504250924383748553933071188903279928981104663696710686541536735838182\n 2512222140811166287287541003826449032093371832913959128171347018667852712082\n") require.Contains(t, output, "3016509350703874362933565866148509373957094754875411937434637891208784994231\n 3015199725895936530535660185611704199044060139852899280809302949374221328865\n 3062378460350040063467318871602229987911299744598148928378797834245039883769\n") @@ -358,7 +433,7 @@ func TestECDSA(t *testing.T) { compiledOutput, err := compileZeroCode("./builtin_tests/ecdsa_test.starknet_with_keccak.cairo") require.NoError(t, err) - _, _, _, err = runVm(compiledOutput) + _, _, _, _, err = runVm(compiledOutput) require.NoError(t, err) clean("./builtin_tests/") @@ -368,7 +443,7 @@ func TestEcOp(t *testing.T) { compiledOutput, err := compileZeroCode("./builtin_tests/ecop.starknet_with_keccak.cairo") require.NoError(t, err) - _, _, _, err = runVm(compiledOutput) + _, _, _, _, err = runVm(compiledOutput) // todo(rodro): This test is failing due to the lack of hint processing. It should be address soon require.Error(t, err) @@ -379,7 +454,7 @@ func TestKeccak(t *testing.T) { compiledOutput, err := compileZeroCode("./builtin_tests/keccak_test.starknet_with_keccak.cairo") require.NoError(t, err) - _, _, output, err := runVm(compiledOutput) + _, _, _, output, err := runVm(compiledOutput) require.NoError(t, err) require.Contains(t, output, "Program output:\n 1304102964824333531548398680304964155037696012322029952943772\n 688749063493959345342507274897412933692859993314608487848187\n 986714560881445649520443980361539218531403996118322524237197\n 1184757872753521629808292433475729390634371625298664050186717\n 719230200744669084408849842242045083289669818920073250264351\n 1543031433416778513637578850638598357854418012971636697855068\n 63644822371671650271181212513090078620238279557402571802224\n 879446821229338092940381117330194802032344024906379963157761\n")