From 1dd7b8c40ca794cfe4bd72baf39f89b4ae952ce8 Mon Sep 17 00:00:00 2001 From: ascandone Date: Tue, 9 Sep 2025 12:02:32 +0200 Subject: [PATCH 01/10] breaking: change assertion name --- .../mixed-source-prefer-single-source.num.specs.json | 2 +- .../experimental/transfer-example.num.specs.json | 4 ++-- .../testdata/numscript-cookbook/top-up-max.num.specs.json | 2 +- ...olor-restrict-balance-when-missing-funds.num.specs.json | 6 ++---- .../experimental/oneof/oneof-all-failing.num.specs.json | 6 ++---- .../script-tests/insufficient-funds.num.specs.json | 2 +- .../overdraft-when-not-enough-funds.num.specs.json | 2 +- .../save-from-account__save-causes-failure.num.specs.json | 2 +- .../source-allotment-invalid-amt.num.specs.json | 2 +- .../testdata/script-tests/track-balances2.num.specs.json | 2 +- internal/specs_format/__snapshots__/runner_test.snap | 2 +- internal/specs_format/index.go | 7 ++++--- internal/specs_format/run_test.go | 6 +++--- internal/specs_format/runner_test.go | 6 +++--- specs.schema.json | 2 +- 15 files changed, 25 insertions(+), 28 deletions(-) diff --git a/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json index 431cca04..5cae81f7 100644 --- a/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json @@ -75,7 +75,7 @@ "USD": 4 } }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } diff --git a/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json index fd4844de..974638e5 100644 --- a/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json @@ -31,7 +31,7 @@ "wallet": { "EUR": 50 }, "bank_account": { "EUR": -100 } }, - "expect.missingFunds": true + "expect.errMissingFunds": true }, { "it": "should not authorize transfer if bank account does not display enough balance and wallet does", @@ -39,7 +39,7 @@ "wallet": { "EUR": 100 }, "bank_account": { "EUR": -50 } }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } diff --git a/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json index 0ef59acf..2700c7c2 100644 --- a/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json @@ -49,7 +49,7 @@ "EUR/2": 51 } }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json b/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json index ad2f94d1..beaa0a9c 100644 --- a/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json +++ b/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json @@ -1,7 +1,5 @@ { - "featureFlags": [ - "experimental-asset-colors" - ], + "featureFlags": ["experimental-asset-colors"], "testCases": [ { "it": "-", @@ -11,7 +9,7 @@ "COIN_RED": 1 } }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json b/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json index 6e799080..6726fe80 100644 --- a/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json +++ b/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json @@ -1,7 +1,5 @@ { - "featureFlags": [ - "experimental-oneof" - ], + "featureFlags": ["experimental-oneof"], "testCases": [ { "it": "-", @@ -10,7 +8,7 @@ "empty2": {}, "empty3": {} }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json b/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json index 53642e8b..f7c8a455 100644 --- a/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json +++ b/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json @@ -15,7 +15,7 @@ "payment": "payments:001", "seller": "users:002" }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json b/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json index d6ff9698..3a4be108 100644 --- a/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json +++ b/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json @@ -7,7 +7,7 @@ "COIN": 1 } }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json b/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json index 00f4e4ab..d1dbf52c 100644 --- a/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json +++ b/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json @@ -7,7 +7,7 @@ "USD/2": 30 } }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json b/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json index 42603ca8..3bfd8350 100644 --- a/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json +++ b/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json @@ -7,7 +7,7 @@ "COIN": 1 } }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json b/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json index 9a740b96..b97d525f 100644 --- a/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json +++ b/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json @@ -7,7 +7,7 @@ "COIN": 60 } }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } diff --git a/internal/specs_format/__snapshots__/runner_test.snap b/internal/specs_format/__snapshots__/runner_test.snap index 0cfee566..9e13d9e9 100755 --- a/internal/specs_format/__snapshots__/runner_test.snap +++ b/internal/specs_format/__snapshots__/runner_test.snap @@ -52,7 +52,7 @@ GIVEN: - Expected + Received  -expect.missingFunds +expect.errMissingFunds - true + false diff --git a/internal/specs_format/index.go b/internal/specs_format/index.go index f4f27d6a..f03629c5 100644 --- a/internal/specs_format/index.go +++ b/internal/specs_format/index.go @@ -26,7 +26,8 @@ type TestCase struct { Meta interpreter.AccountsMetadata `json:"metadata,omitempty"` // Expectations - ExpectMissingFunds bool `json:"expect.missingFunds,omitempty"` + ExpectMissingFunds bool `json:"expect.errMissingFunds,omitempty"` + ExpectPostings []interpreter.Posting `json:"expect.postings,omitempty"` ExpectTxMeta map[string]string `json:"expect.txMetadata,omitempty"` ExpectAccountsMeta interpreter.AccountsMetadata `json:"expect.metadata,omitempty"` @@ -102,7 +103,7 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp if !testCase.ExpectMissingFunds { failedAssertions = append(failedAssertions, AssertionMismatch[any]{ - Assertion: "expect.missingFunds", + Assertion: "expect.errMissingFunds", Expected: false, Got: true, }) @@ -112,7 +113,7 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp if testCase.ExpectMissingFunds { failedAssertions = append(failedAssertions, AssertionMismatch[any]{ - Assertion: "expect.missingFunds", + Assertion: "expect.errMissingFunds", Expected: true, Got: false, }) diff --git a/internal/specs_format/run_test.go b/internal/specs_format/run_test.go index 16582ca0..504b70c8 100644 --- a/internal/specs_format/run_test.go +++ b/internal/specs_format/run_test.go @@ -163,7 +163,7 @@ func TestRunWithMissingBalance(t *testing.T) { "it": "t1", "variables": { "source": "src", "amount": "42" }, "balances": { "src": { "USD": 1 } }, - "expect.missingFunds": false, + "expect.errMissingFunds": false, "expect.postings": null } ] @@ -196,7 +196,7 @@ func TestRunWithMissingBalance(t *testing.T) { Meta: interpreter.AccountsMetadata{}, FailedAssertions: []specs_format.AssertionMismatch[any]{ { - Assertion: "expect.missingFunds", + Assertion: "expect.errMissingFunds", Expected: false, Got: true, }, @@ -250,7 +250,7 @@ func TestRunWithMissingBalanceWhenExpectedPostings(t *testing.T) { Meta: interpreter.AccountsMetadata{}, FailedAssertions: []specs_format.AssertionMismatch[any]{ { - Assertion: "expect.missingFunds", + Assertion: "expect.errMissingFunds", Got: true, Expected: false, }, diff --git a/internal/specs_format/runner_test.go b/internal/specs_format/runner_test.go index 21973bb0..cd6f49c5 100644 --- a/internal/specs_format/runner_test.go +++ b/internal/specs_format/runner_test.go @@ -101,14 +101,14 @@ func TestComplexAssertions(t *testing.T) { "dest": { "EUR": 100 } } }, - "expect.missingFunds": true + "expect.errMissingFunds": true }, { "it": "tpassing", "balances": { "alice": { "USD/2": 0 } }, - "expect.missingFunds": true + "expect.errMissingFunds": true } ] } @@ -194,7 +194,7 @@ func TestRuntimeErr(t *testing.T) { "testCases": [ { "it": "runs", - "expect.missingFunds": false + "expect.errMissingFunds": false } ] } diff --git a/specs.schema.json b/specs.schema.json index 79a2aaf8..d1e02784 100644 --- a/specs.schema.json +++ b/specs.schema.json @@ -64,7 +64,7 @@ "$ref": "#/definitions/AccountsMetadata" }, - "expect.missingFunds": { + "expect.errMissingFunds": { "type": "boolean" } } From 69a3112aaa540860056ab52433bdcc82588b24d0 Mon Sep 17 00:00:00 2001 From: ascandone Date: Tue, 9 Sep 2025 12:42:32 +0200 Subject: [PATCH 02/10] feat: add errMissingFunds assertion --- internal/specs_format/index.go | 36 +++++++++++++++++------- internal/specs_format/run_test.go | 46 +++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/internal/specs_format/index.go b/internal/specs_format/index.go index f03629c5..1aea2e10 100644 --- a/internal/specs_format/index.go +++ b/internal/specs_format/index.go @@ -26,7 +26,8 @@ type TestCase struct { Meta interpreter.AccountsMetadata `json:"metadata,omitempty"` // Expectations - ExpectMissingFunds bool `json:"expect.errMissingFunds,omitempty"` + ExpectMissingFunds bool `json:"expect.errMissingFunds,omitempty"` + ExpectNegativeAmount bool `json:"expect.errNegativeAmount,omitempty"` ExpectPostings []interpreter.Posting `json:"expect.postings,omitempty"` ExpectTxMeta map[string]string `json:"expect.txMetadata,omitempty"` @@ -96,24 +97,39 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp var failedAssertions []AssertionMismatch[any] if err != nil { - _, ok := err.(interpreter.MissingFundsErr) - if !ok { + switch err.(type) { + case interpreter.MissingFundsErr: + if !testCase.ExpectMissingFunds { + failedAssertions = append(failedAssertions, AssertionMismatch[any]{ + Assertion: "expect.errMissingFunds", + Expected: false, + Got: true, + }) + } + case interpreter.NegativeAmountErr: + if !testCase.ExpectNegativeAmount { + failedAssertions = append(failedAssertions, AssertionMismatch[any]{ + Assertion: "expect.errNegativeAmount", + Expected: false, + Got: true, + }) + } + default: return SpecsResult{}, err } + } else { - if !testCase.ExpectMissingFunds { + if testCase.ExpectMissingFunds { failedAssertions = append(failedAssertions, AssertionMismatch[any]{ Assertion: "expect.errMissingFunds", - Expected: false, - Got: true, + Expected: true, + Got: false, }) } - } else { - - if testCase.ExpectMissingFunds { + if testCase.ExpectNegativeAmount { failedAssertions = append(failedAssertions, AssertionMismatch[any]{ - Assertion: "expect.errMissingFunds", + Assertion: "expect.errNegativeAmount", Expected: true, Got: false, }) diff --git a/internal/specs_format/run_test.go b/internal/specs_format/run_test.go index 504b70c8..8fdf947e 100644 --- a/internal/specs_format/run_test.go +++ b/internal/specs_format/run_test.go @@ -306,3 +306,49 @@ func TestNullPostingsIsNoop(t *testing.T) { }, out) } + +func TestNegativeAmt(t *testing.T) { + exampleProgram := parser.Parse(` + vars { number $amt } + send [USD $amt] ( + source = @world + destination = @dest + ) + `) + + j := `{ + "testCases": [ + { + "it": "t1", + "variables": { "amt": "-100" }, + "expect.errNegativeAmount": true + } + ] + }` + + var specs specs_format.Specs + err := json.Unmarshal([]byte(j), &specs) + require.Nil(t, err) + + out, err := specs_format.Check(exampleProgram.Value, specs) + require.Nil(t, err) + + require.Equal(t, specs_format.SpecsResult{ + Total: 1, + Failing: 0, + Passing: 1, + Cases: []specs_format.TestCaseResult{ + { + It: "t1", + Pass: true, + Vars: interpreter.VariablesMap{ + "amt": "-100", + }, + Balances: interpreter.Balances{}, + Meta: interpreter.AccountsMetadata{}, + FailedAssertions: nil, + }, + }, + }, out) + +} From 6e5e2c29065a2c7f23b20b8217cbe0a7186175a5 Mon Sep 17 00:00:00 2001 From: ascandone Date: Tue, 9 Sep 2025 13:06:40 +0200 Subject: [PATCH 03/10] breaking: renamed assertion --- .../numscript-cookbook/experimental/top-up.num.specs.json | 2 +- internal/specs_format/__snapshots__/runner_test.snap | 2 +- internal/specs_format/index.go | 8 ++++---- internal/specs_format/runner_test.go | 2 +- specs.schema.json | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/interpreter/testdata/numscript-cookbook/experimental/top-up.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/experimental/top-up.num.specs.json index fbaddc23..05fcf681 100644 --- a/internal/interpreter/testdata/numscript-cookbook/experimental/top-up.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/experimental/top-up.num.specs.json @@ -10,7 +10,7 @@ { "it": "should send the missing amount to an overdraft account", "balances": { "alice": { "EUR": -100 } }, - "expect.volumes": { + "expect.endBalances": { "alice": { "EUR": 0 }, "world": { "EUR": -100 } }, diff --git a/internal/specs_format/__snapshots__/runner_test.snap b/internal/specs_format/__snapshots__/runner_test.snap index 9e13d9e9..abaf0dfa 100755 --- a/internal/specs_format/__snapshots__/runner_test.snap +++ b/internal/specs_format/__snapshots__/runner_test.snap @@ -57,7 +57,7 @@ GIVEN: - true + false -expect.volumes +expect.endBalances  {  "alice": { diff --git a/internal/specs_format/index.go b/internal/specs_format/index.go index 1aea2e10..048b6dad 100644 --- a/internal/specs_format/index.go +++ b/internal/specs_format/index.go @@ -32,7 +32,7 @@ type TestCase struct { ExpectPostings []interpreter.Posting `json:"expect.postings,omitempty"` ExpectTxMeta map[string]string `json:"expect.txMetadata,omitempty"` ExpectAccountsMeta interpreter.AccountsMetadata `json:"expect.metadata,omitempty"` - ExpectVolumes interpreter.Balances `json:"expect.volumes,omitempty"` + ExpectEndBalances interpreter.Balances `json:"expect.endBalances,omitempty"` ExpectMovements Movements `json:"expect.movements,omitempty"` } @@ -166,10 +166,10 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp ) } - if testCase.ExpectVolumes != nil { + if testCase.ExpectEndBalances != nil { failedAssertions = runAssertion(failedAssertions, - "expect.volumes", - testCase.ExpectVolumes, + "expect.endBalances", + testCase.ExpectEndBalances, getVolumes(result.Postings, balances), interpreter.CompareBalances, ) diff --git a/internal/specs_format/runner_test.go b/internal/specs_format/runner_test.go index cd6f49c5..5ba7b397 100644 --- a/internal/specs_format/runner_test.go +++ b/internal/specs_format/runner_test.go @@ -92,7 +92,7 @@ func TestComplexAssertions(t *testing.T) { "balances": { "alice": { "USD/2": 9999 } }, - "expect.volumes": { + "expect.endBalances": { "alice": { "USD/2": -100 }, "dest": { "USD/2": 1 } }, diff --git a/specs.schema.json b/specs.schema.json index d1e02784..f32c813c 100644 --- a/specs.schema.json +++ b/specs.schema.json @@ -48,7 +48,7 @@ "items": { "$ref": "#/definitions/Posting" } }, - "expect.volumes": { + "expect.endBalances": { "$ref": "#/definitions/Balances" }, From da1f3d2a6114b680f6a104113902b218e1445831 Mon Sep 17 00:00:00 2001 From: ascandone Date: Tue, 9 Sep 2025 13:08:25 +0200 Subject: [PATCH 04/10] breaking: renamed assertions --- .../mixed-source-prefer-single-source.num.specs.json | 2 +- .../experimental/transfer-example.num.specs.json | 4 ++-- .../numscript-cookbook/top-up-max.num.specs.json | 2 +- ...estrict-balance-when-missing-funds.num.specs.json | 2 +- .../oneof/oneof-all-failing.num.specs.json | 2 +- .../script-tests/insufficient-funds.num.specs.json | 2 +- .../overdraft-when-not-enough-funds.num.specs.json | 2 +- ...-from-account__save-causes-failure.num.specs.json | 2 +- .../source-allotment-invalid-amt.num.specs.json | 2 +- .../script-tests/track-balances2.num.specs.json | 2 +- internal/specs_format/__snapshots__/runner_test.snap | 2 +- internal/specs_format/index.go | 12 ++++++------ internal/specs_format/run_test.go | 8 ++++---- internal/specs_format/runner_test.go | 6 +++--- specs.schema.json | 2 +- 15 files changed, 26 insertions(+), 26 deletions(-) diff --git a/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json index 5cae81f7..c54424ab 100644 --- a/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json @@ -75,7 +75,7 @@ "USD": 4 } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json index 974638e5..c0e92c9a 100644 --- a/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json @@ -31,7 +31,7 @@ "wallet": { "EUR": 50 }, "bank_account": { "EUR": -100 } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true }, { "it": "should not authorize transfer if bank account does not display enough balance and wallet does", @@ -39,7 +39,7 @@ "wallet": { "EUR": 100 }, "bank_account": { "EUR": -50 } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json index 2700c7c2..00098115 100644 --- a/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json @@ -49,7 +49,7 @@ "EUR/2": 51 } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json b/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json index beaa0a9c..f71564aa 100644 --- a/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json +++ b/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json @@ -9,7 +9,7 @@ "COIN_RED": 1 } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json b/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json index 6726fe80..a506b5eb 100644 --- a/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json +++ b/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json @@ -8,7 +8,7 @@ "empty2": {}, "empty3": {} }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json b/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json index f7c8a455..3a98515e 100644 --- a/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json +++ b/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json @@ -15,7 +15,7 @@ "payment": "payments:001", "seller": "users:002" }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json b/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json index 3a4be108..540a5447 100644 --- a/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json +++ b/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json @@ -7,7 +7,7 @@ "COIN": 1 } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json b/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json index d1dbf52c..b714008c 100644 --- a/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json +++ b/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json @@ -7,7 +7,7 @@ "USD/2": 30 } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json b/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json index 3bfd8350..15efe3cb 100644 --- a/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json +++ b/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json @@ -7,7 +7,7 @@ "COIN": 1 } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json b/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json index b97d525f..fe4851a6 100644 --- a/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json +++ b/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json @@ -7,7 +7,7 @@ "COIN": 60 } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/specs_format/__snapshots__/runner_test.snap b/internal/specs_format/__snapshots__/runner_test.snap index abaf0dfa..2568138b 100755 --- a/internal/specs_format/__snapshots__/runner_test.snap +++ b/internal/specs_format/__snapshots__/runner_test.snap @@ -52,7 +52,7 @@ GIVEN: - Expected + Received  -expect.errMissingFunds +expect.error.missingFunds - true + false diff --git a/internal/specs_format/index.go b/internal/specs_format/index.go index 048b6dad..bc1549ec 100644 --- a/internal/specs_format/index.go +++ b/internal/specs_format/index.go @@ -26,8 +26,8 @@ type TestCase struct { Meta interpreter.AccountsMetadata `json:"metadata,omitempty"` // Expectations - ExpectMissingFunds bool `json:"expect.errMissingFunds,omitempty"` - ExpectNegativeAmount bool `json:"expect.errNegativeAmount,omitempty"` + ExpectMissingFunds bool `json:"expect.error.missingFunds,omitempty"` + ExpectNegativeAmount bool `json:"expect.error.negativeAmount,omitempty"` ExpectPostings []interpreter.Posting `json:"expect.postings,omitempty"` ExpectTxMeta map[string]string `json:"expect.txMetadata,omitempty"` @@ -101,7 +101,7 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp case interpreter.MissingFundsErr: if !testCase.ExpectMissingFunds { failedAssertions = append(failedAssertions, AssertionMismatch[any]{ - Assertion: "expect.errMissingFunds", + Assertion: "expect.error.missingFunds", Expected: false, Got: true, }) @@ -109,7 +109,7 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp case interpreter.NegativeAmountErr: if !testCase.ExpectNegativeAmount { failedAssertions = append(failedAssertions, AssertionMismatch[any]{ - Assertion: "expect.errNegativeAmount", + Assertion: "expect.error.negativeAmount", Expected: false, Got: true, }) @@ -121,7 +121,7 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp if testCase.ExpectMissingFunds { failedAssertions = append(failedAssertions, AssertionMismatch[any]{ - Assertion: "expect.errMissingFunds", + Assertion: "expect.error.missingFunds", Expected: true, Got: false, }) @@ -129,7 +129,7 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp if testCase.ExpectNegativeAmount { failedAssertions = append(failedAssertions, AssertionMismatch[any]{ - Assertion: "expect.errNegativeAmount", + Assertion: "expect.error.negativeAmount", Expected: true, Got: false, }) diff --git a/internal/specs_format/run_test.go b/internal/specs_format/run_test.go index 8fdf947e..08d7deeb 100644 --- a/internal/specs_format/run_test.go +++ b/internal/specs_format/run_test.go @@ -163,7 +163,7 @@ func TestRunWithMissingBalance(t *testing.T) { "it": "t1", "variables": { "source": "src", "amount": "42" }, "balances": { "src": { "USD": 1 } }, - "expect.errMissingFunds": false, + "expect.error.missingFunds": false, "expect.postings": null } ] @@ -196,7 +196,7 @@ func TestRunWithMissingBalance(t *testing.T) { Meta: interpreter.AccountsMetadata{}, FailedAssertions: []specs_format.AssertionMismatch[any]{ { - Assertion: "expect.errMissingFunds", + Assertion: "expect.error.missingFunds", Expected: false, Got: true, }, @@ -250,7 +250,7 @@ func TestRunWithMissingBalanceWhenExpectedPostings(t *testing.T) { Meta: interpreter.AccountsMetadata{}, FailedAssertions: []specs_format.AssertionMismatch[any]{ { - Assertion: "expect.errMissingFunds", + Assertion: "expect.error.missingFunds", Got: true, Expected: false, }, @@ -321,7 +321,7 @@ func TestNegativeAmt(t *testing.T) { { "it": "t1", "variables": { "amt": "-100" }, - "expect.errNegativeAmount": true + "expect.error.negativeAmount": true } ] }` diff --git a/internal/specs_format/runner_test.go b/internal/specs_format/runner_test.go index 5ba7b397..9a75c242 100644 --- a/internal/specs_format/runner_test.go +++ b/internal/specs_format/runner_test.go @@ -101,14 +101,14 @@ func TestComplexAssertions(t *testing.T) { "dest": { "EUR": 100 } } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true }, { "it": "tpassing", "balances": { "alice": { "USD/2": 0 } }, - "expect.errMissingFunds": true + "expect.error.missingFunds": true } ] } @@ -194,7 +194,7 @@ func TestRuntimeErr(t *testing.T) { "testCases": [ { "it": "runs", - "expect.errMissingFunds": false + "expect.error.missingFunds": false } ] } diff --git a/specs.schema.json b/specs.schema.json index f32c813c..6cb05c77 100644 --- a/specs.schema.json +++ b/specs.schema.json @@ -64,7 +64,7 @@ "$ref": "#/definitions/AccountsMetadata" }, - "expect.errMissingFunds": { + "expect.error.missingFunds": { "type": "boolean" } } From f7b1f316aa245735502e19792e636487258c902e Mon Sep 17 00:00:00 2001 From: ascandone Date: Tue, 9 Sep 2025 14:44:46 +0200 Subject: [PATCH 05/10] feat: added expect.endBalances.include assertion --- internal/interpreter/balances.go | 9 ++++ internal/interpreter/balances_test.go | 64 +++++++++++++++++++++++++++ internal/specs_format/index.go | 24 +++++++--- internal/utils/utils.go | 15 +++++-- specs.schema.json | 3 ++ 5 files changed, 104 insertions(+), 11 deletions(-) diff --git a/internal/interpreter/balances.go b/internal/interpreter/balances.go index 222eb865..b1f31881 100644 --- a/internal/interpreter/balances.go +++ b/internal/interpreter/balances.go @@ -100,3 +100,12 @@ func CompareBalances(b1 Balances, b2 Balances) bool { return ab1.Cmp(ab2) == 0 }) } + +// Returns whether the second value is a subset of the first one +func CompareBalancesIncluding(b1 Balances, b2 Balances) bool { + return utils.MapIncludes(b1, b2, func(a1 AccountBalance, a2 AccountBalance) bool { + return utils.MapIncludes(a1, a2, func(i1 *big.Int, i2 *big.Int) bool { + return i1.Cmp(i2) == 0 + }) + }) +} diff --git a/internal/interpreter/balances_test.go b/internal/interpreter/balances_test.go index f41e24c9..8f5268af 100644 --- a/internal/interpreter/balances_test.go +++ b/internal/interpreter/balances_test.go @@ -79,3 +79,67 @@ func TestCmpMaps(t *testing.T) { require.Equal(t, false, CompareBalances(b1, b2)) } + +func TestCmpMapsIncluding(t *testing.T) { + + t.Run("including (subset)", func(t *testing.T) { + b1 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(100), + }, + "bob": AccountBalance{ + "EUR": big.NewInt(100), + }, + } + + b2 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(100), + }, + } + + require.Equal(t, true, CompareBalancesIncluding(b1, b2)) + }) + + t.Run("different value", func(t *testing.T) { + b1 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(100), + }, + "bob": AccountBalance{ + "EUR": big.NewInt(100), + }, + } + + b2 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(0), + }, + } + + require.Equal(t, false, CompareBalancesIncluding(b1, b2)) + }) + + t.Run("extra value", func(t *testing.T) { + b1 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(100), + }, + "bob": AccountBalance{ + "EUR": big.NewInt(100), + }, + } + + b2 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(100), + }, + + "extra-value": AccountBalance{ + "EUR": big.NewInt(100), + }, + } + + require.Equal(t, false, CompareBalancesIncluding(b1, b2)) + }) +} diff --git a/internal/specs_format/index.go b/internal/specs_format/index.go index bc1549ec..6bce287b 100644 --- a/internal/specs_format/index.go +++ b/internal/specs_format/index.go @@ -29,11 +29,12 @@ type TestCase struct { ExpectMissingFunds bool `json:"expect.error.missingFunds,omitempty"` ExpectNegativeAmount bool `json:"expect.error.negativeAmount,omitempty"` - ExpectPostings []interpreter.Posting `json:"expect.postings,omitempty"` - ExpectTxMeta map[string]string `json:"expect.txMetadata,omitempty"` - ExpectAccountsMeta interpreter.AccountsMetadata `json:"expect.metadata,omitempty"` - ExpectEndBalances interpreter.Balances `json:"expect.endBalances,omitempty"` - ExpectMovements Movements `json:"expect.movements,omitempty"` + ExpectPostings []interpreter.Posting `json:"expect.postings,omitempty"` + ExpectTxMeta map[string]string `json:"expect.txMetadata,omitempty"` + ExpectAccountsMeta interpreter.AccountsMetadata `json:"expect.metadata,omitempty"` + ExpectEndBalances interpreter.Balances `json:"expect.endBalances,omitempty"` + ExpectEndBalancesInclude interpreter.Balances `json:"expect.endBalances.include,omitempty"` + ExpectMovements Movements `json:"expect.movements,omitempty"` } type TestCaseResult struct { @@ -170,11 +171,20 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp failedAssertions = runAssertion(failedAssertions, "expect.endBalances", testCase.ExpectEndBalances, - getVolumes(result.Postings, balances), + getBalances(result.Postings, balances), interpreter.CompareBalances, ) } + if testCase.ExpectEndBalancesInclude != nil { + failedAssertions = runAssertion(failedAssertions, + "expect.endBalances.include", + testCase.ExpectEndBalancesInclude, + getBalances(result.Postings, balances), + interpreter.CompareBalancesIncluding, + ) + } + if testCase.ExpectMovements != nil { failedAssertions = runAssertion[any](failedAssertions, "expect.movements", @@ -256,7 +266,7 @@ func getMovements(postings []interpreter.Posting) Movements { return m } -func getVolumes(postings []interpreter.Posting, initialBalances interpreter.Balances) interpreter.Balances { +func getBalances(postings []interpreter.Posting, initialBalances interpreter.Balances) interpreter.Balances { balances := initialBalances.DeepClone() for _, posting := range postings { sourceBalance := utils.NestedMapGetOrPutDefault(balances, posting.Source, posting.Asset, func() *big.Int { diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 018cddc5..02aa569d 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -65,14 +65,21 @@ func NestedMapGetOrPutDefault[T any](m map[string]map[string]T, key1 string, key return MapGetOrPutDefault(m1, key2, getDefault) } +// Returns whether m1 is equal to m2 (according to the "cmp" equality) func MapCmp[T any](m1, m2 map[string]T, cmp func(x1 T, x2 T) bool) bool { - if len(m1) != len(m2) { + // motivation: if m2 is subset of m1, and they have the same cardinality, then m1==m2 + return len(m1) == len(m2) && MapIncludes(m1, m2, cmp) +} + +// Returns whether m1 is a superset of m2 +func MapIncludes[T any](m1, m2 map[string]T, includes func(x1 T, x2 T) bool) bool { + if len(m1) < len(m2) { return false } - for k1, v1 := range m1 { - v2, ok := m2[k1] - if !ok || !cmp(v1, v2) { + for k1, v1 := range m2 { + v2, ok := m1[k1] + if !ok || !includes(v1, v2) { return false } } diff --git a/specs.schema.json b/specs.schema.json index 6cb05c77..f9979821 100644 --- a/specs.schema.json +++ b/specs.schema.json @@ -51,6 +51,9 @@ "expect.endBalances": { "$ref": "#/definitions/Balances" }, + "expect.endBalances.include": { + "$ref": "#/definitions/Balances" + }, "expect.movements": { "$ref": "#/definitions/Movements" From e645cf8c245f071738417a68eef10b123ebee4ed Mon Sep 17 00:00:00 2001 From: ascandone Date: Tue, 9 Sep 2025 15:00:08 +0200 Subject: [PATCH 06/10] feat: better test output --- internal/specs_format/index.go | 4 ++++ internal/specs_format/runner.go | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/internal/specs_format/index.go b/internal/specs_format/index.go index 6bce287b..a241fe41 100644 --- a/internal/specs_format/index.go +++ b/internal/specs_format/index.go @@ -44,6 +44,9 @@ type TestCaseResult struct { Vars interpreter.VariablesMap `json:"variables"` Meta interpreter.AccountsMetadata `json:"metadata"` + // Output: + Postings []interpreter.Posting `json:"postings"` + // Assertions FailedAssertions []AssertionMismatch[any] `json:"failedAssertions"` } @@ -210,6 +213,7 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp Balances: balances, Vars: vars, FailedAssertions: failedAssertions, + Postings: result.Postings, }) } diff --git a/internal/specs_format/runner.go b/internal/specs_format/runner.go index d7537fbd..f1949b4c 100644 --- a/internal/specs_format/runner.go +++ b/internal/specs_format/runner.go @@ -224,6 +224,7 @@ func showFailingTestCase(w io.Writer, testResult TestResult) { _, _ = fmt.Fprint(w, failColor(" FAIL ")) _, _ = fmt.Fprintln(w, ansi.ColorRed(" "+specsFilePath+" > "+result.It)) + // --- Preconditions showGiven := len(result.Balances) != 0 || len(result.Meta) != 0 || len(result.Vars) != 0 if showGiven { _, _ = fmt.Fprintln(w, ansi.Underline("\nGIVEN:")) @@ -247,6 +248,19 @@ func showFailingTestCase(w io.Writer, testResult TestResult) { _, _ = fmt.Fprintln(w) } + // --- Outputs + _, _ = fmt.Fprintln(w, ansi.Underline("\nGOT:")) + if len(result.Postings) != 0 { + _, _ = fmt.Fprintln(w) + _, _ = fmt.Fprintln(w, interpreter.PrettyPrintPostings(result.Postings)) + _, _ = fmt.Fprintln(w) + } else { + _, _ = fmt.Fprintln(w) + + _, _ = fmt.Fprintln(w, ansi.ColorBrightBlack("")) + _, _ = fmt.Fprintln(w) + } + _, _ = fmt.Fprintln(w) _, _ = fmt.Fprintln(w, ansi.ColorGreen("- Expected")) _, _ = fmt.Fprintln(w, ansi.ColorRed("+ Received\n")) From a300803f3cedb3336989ff5f3f182d7e81e068ef Mon Sep 17 00:00:00 2001 From: ascandone Date: Tue, 9 Sep 2025 15:31:11 +0200 Subject: [PATCH 07/10] fix --- .../__snapshots__/runner_test.snap | 12 +++++ internal/specs_format/index.go | 7 ++- internal/specs_format/run_test.go | 50 +++++++------------ 3 files changed, 36 insertions(+), 33 deletions(-) diff --git a/internal/specs_format/__snapshots__/runner_test.snap b/internal/specs_format/__snapshots__/runner_test.snap index 2568138b..8215fcb7 100755 --- a/internal/specs_format/__snapshots__/runner_test.snap +++ b/internal/specs_format/__snapshots__/runner_test.snap @@ -14,6 +14,12 @@  FAIL  example.num.specs.json > tfailing + +GOT: + +| Source | Destination | Asset | Amount | +| world | dest | USD/2 | 100 | + - Expected + Received @@ -48,6 +54,12 @@ GIVEN: | Account | Asset | Balance | | alice | USD/2 | 9999 | + +GOT: + +| Source | Destination | Asset | Amount | +| alice | dest | USD/2 | 100 | + - Expected + Received diff --git a/internal/specs_format/index.go b/internal/specs_format/index.go index a241fe41..6f45ae88 100644 --- a/internal/specs_format/index.go +++ b/internal/specs_format/index.go @@ -206,6 +206,11 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp specsResult.Failing += 1 } + var postings []interpreter.Posting + if result != nil { + postings = result.Postings + } + specsResult.Cases = append(specsResult.Cases, TestCaseResult{ It: testCase.It, Pass: pass, @@ -213,7 +218,7 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp Balances: balances, Vars: vars, FailedAssertions: failedAssertions, - Postings: result.Postings, + Postings: postings, }) } diff --git a/internal/specs_format/run_test.go b/internal/specs_format/run_test.go index 08d7deeb..a071fb25 100644 --- a/internal/specs_format/run_test.go +++ b/internal/specs_format/run_test.go @@ -63,22 +63,15 @@ func TestRunSpecsSimple(t *testing.T) { }, Meta: interpreter.AccountsMetadata{}, FailedAssertions: nil, - // ExpectedPostings: []interpreter.Posting{ - // { - // Source: "src", - // Destination: "dest", - // Asset: "USD", - // Amount: big.NewInt(42), - // }, - // }, - // ActualPostings: []interpreter.Posting{ - // { - // Source: "src", - // Destination: "dest", - // Asset: "USD", - // Amount: big.NewInt(42), - // }, - // }, + + Postings: []interpreter.Posting{ + { + Source: "src", + Destination: "dest", + Asset: "USD", + Amount: big.NewInt(42), + }, + }, }, }, }, out) @@ -134,22 +127,14 @@ func TestRunSpecsMergeOuter(t *testing.T) { }, }, FailedAssertions: nil, - // ExpectedPostings: []interpreter.Posting{ - // { - // Source: "src", - // Destination: "dest", - // Asset: "USD", - // Amount: big.NewInt(1), - // }, - // }, - // ActualPostings: []interpreter.Posting{ - // { - // Source: "src", - // Destination: "dest", - // Asset: "USD", - // Amount: big.NewInt(1), - // }, - // }, + Postings: []interpreter.Posting{ + { + Source: "src", + Destination: "dest", + Asset: "USD", + Amount: big.NewInt(1), + }, + }, }, }, }, out) @@ -301,6 +286,7 @@ func TestNullPostingsIsNoop(t *testing.T) { }, Meta: interpreter.AccountsMetadata{}, FailedAssertions: nil, + Postings: []interpreter.Posting{}, }, }, }, out) From 490b887abdc75a2c07e43f6f03f1feeb37f388a4 Mon Sep 17 00:00:00 2001 From: ascandone Date: Tue, 9 Sep 2025 15:44:55 +0200 Subject: [PATCH 08/10] feat: added focus and skip --- internal/specs_format/index.go | 28 +++++++- internal/specs_format/run_test.go | 103 ++++++++++++++++++++++++++++++ specs.schema.json | 9 +++ 3 files changed, 137 insertions(+), 3 deletions(-) diff --git a/internal/specs_format/index.go b/internal/specs_format/index.go index 6f45ae88..5d4ffb7d 100644 --- a/internal/specs_format/index.go +++ b/internal/specs_format/index.go @@ -4,6 +4,7 @@ import ( "context" "math/big" "reflect" + "slices" "github.com/formancehq/numscript/internal/interpreter" "github.com/formancehq/numscript/internal/parser" @@ -20,11 +21,17 @@ type Specs struct { } type TestCase struct { - It string `json:"it"` + It string `json:"it"` + + // Preconditions Balances interpreter.Balances `json:"balances,omitempty"` Vars interpreter.VariablesMap `json:"variables,omitempty"` Meta interpreter.AccountsMetadata `json:"metadata,omitempty"` + // Select tests + Focus bool `json:"focus,omitempty"` + Skip bool `json:"skip,omitempty"` + // Expectations ExpectMissingFunds bool `json:"expect.error.missingFunds,omitempty"` ExpectNegativeAmount bool `json:"expect.error.negativeAmount,omitempty"` @@ -38,6 +45,7 @@ type TestCase struct { } type TestCaseResult struct { + Skipped bool `json:"skipped"` It string `json:"it"` Pass bool `json:"pass"` Balances interpreter.Balances `json:"balances"` @@ -56,6 +64,7 @@ type SpecsResult struct { Total uint `json:"total"` Passing uint `json:"passing"` Failing uint `json:"failing"` + Skipped uint `json:"skipped"` Cases []TestCaseResult } @@ -74,14 +83,26 @@ func runAssertion[T any](failedAssertions []AssertionMismatch[any], assertion st func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.InterpreterError) { specsResult := SpecsResult{} + // we need a first pass to know whether there is at least on test in focus mode + hasFocusedTest := slices.ContainsFunc(specs.TestCases, func(t TestCase) bool { + return t.Focus + }) for _, testCase := range specs.TestCases { + shouldSkip := testCase.Skip || (hasFocusedTest && !testCase.Focus) + if shouldSkip { + specsResult.Skipped += 1 + specsResult.Cases = append(specsResult.Cases, TestCaseResult{ + It: testCase.It, + Skipped: true, + }) + continue + } + meta := mergeAccountsMeta(specs.Meta, testCase.Meta) balances := mergeBalances(specs.Balances, testCase.Balances) vars := mergeVars(specs.Vars, testCase.Vars) - specsResult.Total += 1 - featureFlags := make(map[string]struct{}) for _, flag := range specs.FeatureFlags { featureFlags[flag] = struct{}{} @@ -222,6 +243,7 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp }) } + specsResult.Total = uint(len(specs.TestCases)) return specsResult, nil } diff --git a/internal/specs_format/run_test.go b/internal/specs_format/run_test.go index a071fb25..dc7be8bd 100644 --- a/internal/specs_format/run_test.go +++ b/internal/specs_format/run_test.go @@ -338,3 +338,106 @@ func TestNegativeAmt(t *testing.T) { }, out) } + +func TestSkip(t *testing.T) { + j := `{ + "testCases": [ + { + "it": "t1", + "skip": true + } + ] + }` + + var specs specs_format.Specs + err := json.Unmarshal([]byte(j), &specs) + require.Nil(t, err) + + out, err := specs_format.Check(exampleProgram.Value, specs) + require.Nil(t, err) + + require.Equal(t, specs_format.SpecsResult{ + Total: 1, + Failing: 0, + Passing: 0, + Skipped: 1, + Cases: []specs_format.TestCaseResult{ + { + It: "t1", + Pass: false, + Skipped: true, + }, + }, + }, out) + +} + +func TestFocus(t *testing.T) { + j := `{ + "testCases": [ + { + "it": "t1", + "variables": { "source": "src", "amount": "10" }, + "balances": { "src": { "USD": 9999 } }, + "expect.postings": [ + { "source": "src", "destination": "dest", "asset": "USD", "amount": 42 } + ] + }, + { + "it": "t2", + "focus": true, + "variables": { "source": "src", "amount": "42" }, + "balances": { "src": { "USD": 9999 } }, + "expect.postings": [ + { "source": "src", "destination": "dest", "asset": "USD", "amount": 42 } + ] + } + ] + }` + + var specs specs_format.Specs + err := json.Unmarshal([]byte(j), &specs) + require.Nil(t, err) + + out, err := specs_format.Check(exampleProgram.Value, specs) + require.Nil(t, err) + + require.Equal(t, specs_format.SpecsResult{ + Total: 2, + Failing: 0, + Passing: 1, + Skipped: 1, + Cases: []specs_format.TestCaseResult{ + { + It: "t1", + Skipped: true, + }, + + { + It: "t2", + Pass: true, + Vars: interpreter.VariablesMap{ + "source": "src", + "amount": "42", + }, + Balances: interpreter.Balances{ + "src": interpreter.AccountBalance{ + "USD": big.NewInt(9999), + }, + }, + Meta: interpreter.AccountsMetadata{}, + FailedAssertions: nil, + + Postings: []interpreter.Posting{ + { + Source: "src", + Destination: "dest", + Asset: "USD", + Amount: big.NewInt(42), + }, + }, + }, + }, + }, out) + +} diff --git a/specs.schema.json b/specs.schema.json index f9979821..a3cd1680 100644 --- a/specs.schema.json +++ b/specs.schema.json @@ -34,6 +34,15 @@ "type": "string", "description": "Test case description" }, + "focus": { + "type": "boolean", + "description": "Skip all the other tests. Just for debugging: it will result in an error exit code" + }, + "skip": { + "type": "boolean", + "description": "Skip this test. Just for debugging: it will result in an error exit code" + }, + "balances": { "$ref": "#/definitions/Balances" }, From a0b59ff0dfc4b235949604426350f2d6892bae98 Mon Sep 17 00:00:00 2001 From: ascandone Date: Tue, 9 Sep 2025 16:30:28 +0200 Subject: [PATCH 09/10] improve UI for focused/skipped tests --- .../__snapshots__/runner_test.snap | 12 ++++- internal/specs_format/index.go | 3 +- internal/specs_format/runner.go | 51 ++++++++++++++----- internal/specs_format/runner_test.go | 45 ++++++++++++++++ 4 files changed, 96 insertions(+), 15 deletions(-) diff --git a/internal/specs_format/__snapshots__/runner_test.snap b/internal/specs_format/__snapshots__/runner_test.snap index 8215fcb7..345a0c4a 100755 --- a/internal/specs_format/__snapshots__/runner_test.snap +++ b/internal/specs_format/__snapshots__/runner_test.snap @@ -9,7 +9,7 @@ --- [TestSingleTest - 1] -❯ example.num (2 tests | 1 failed) +❯ example.num (2 tests | 1 failed) × tfailing @@ -43,7 +43,7 @@ GOT: --- [TestComplexAssertions - 1] -❯ example.num (2 tests | 1 failed) +❯ example.num (2 tests | 1 failed) × send when there are enough funds @@ -148,3 +148,11 @@ Error: example.num:1:29 | ~~~~~~ --- + +[TestFocusUi - 1] +✓ example.num (2 tests | 1 skipped) + + Test files  1 passed (1) + Tests  1 passed | 1 skipped (2) + +--- diff --git a/internal/specs_format/index.go b/internal/specs_format/index.go index 5d4ffb7d..946f6b69 100644 --- a/internal/specs_format/index.go +++ b/internal/specs_format/index.go @@ -243,7 +243,8 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp }) } - specsResult.Total = uint(len(specs.TestCases)) + specsResult.Total = specsResult.Failing + specsResult.Passing + specsResult.Skipped + return specsResult, nil } diff --git a/internal/specs_format/runner.go b/internal/specs_format/runner.go index f1949b4c..8ea4f17f 100644 --- a/internal/specs_format/runner.go +++ b/internal/specs_format/runner.go @@ -161,22 +161,37 @@ func runRawSpec(stdout io.Writer, stderr io.Writer, rawSpec RawSpec) (Specs, Spe if out.Total == 0 { _, _ = fmt.Fprintln(stdout, ansi.ColorRed("Empty test suite: "+rawSpec.SpecsPath)) return Specs{}, SpecsResult{}, false - } else if out.Failing == 0 { - testsCount := ansi.ColorBrightBlack(fmt.Sprintf("(%d tests)", out.Total)) - _, _ = fmt.Fprintf(stdout, "%s %s %s\n", ansi.ColorGreen("✓"), rawSpec.NumscriptPath, testsCount) - } else { - failedTestsCount := ansi.ColorRed(fmt.Sprintf("%d failed", out.Failing)) + } - testsCount := ansi.ColorBrightBlack(fmt.Sprintf("(%d tests | %s)", out.Total, failedTestsCount)) - _, _ = fmt.Fprintf(stdout, "%s %s %s\n", ansi.ColorRed("❯"), rawSpec.NumscriptPath, testsCount) + parts := []string{ + ansi.ColorBrightBlack(fmt.Sprintf("%d tests", out.Total)), + } + + if out.Failing != 0 { + parts = append(parts, + ansi.ColorRed(fmt.Sprintf("%d failed", out.Failing)), + ) + } + + if out.Skipped != 0 { + parts = append(parts, + ansi.ColorYellow(fmt.Sprintf("%d skipped", out.Skipped)), + ) + } + testsCount := ansi.ColorBrightBlack("(") + strings.Join(parts, ansi.ColorBrightBlack(" | ")) + ansi.ColorBrightBlack(")") + + if out.Failing != 0 { + _, _ = fmt.Fprintf(stdout, "%s %s %s\n", ansi.ColorRed("❯"), rawSpec.NumscriptPath, testsCount) for _, result := range out.Cases { - if result.Pass { + if result.Pass || result.Skipped { continue } _, _ = fmt.Fprintf(stdout, " %s %s\n", ansi.ColorRed("×"), result.It) } + } else { + _, _ = fmt.Fprintf(stdout, "%s %s %s\n", ansi.ColorGreen("✓"), rawSpec.NumscriptPath, testsCount) } return specs, out, true @@ -211,7 +226,7 @@ func ShowDiff(w io.Writer, expected_ any, got_ any) { } func showFailingTestCase(w io.Writer, testResult TestResult) { - if testResult.Result.Pass { + if testResult.Result.Pass || testResult.Result.Skipped { return } @@ -287,8 +302,14 @@ func showErr(stderr io.Writer, filename string, script string, err interpreter.I } func printFilesStats(w io.Writer, allTests []TestResult) bool { + passedTests := utils.Filter(allTests, func(t TestResult) bool { + return t.Result.Pass + }) + skippedTests := utils.Filter(allTests, func(t TestResult) bool { + return t.Result.Skipped + }) failedTests := utils.Filter(allTests, func(t TestResult) bool { - return !t.Result.Pass + return !t.Result.Pass && !t.Result.Skipped }) testFilesLabel := "Test files" @@ -334,7 +355,8 @@ func printFilesStats(w io.Writer, allTests []TestResult) bool { testsCount := len(allTests) failedTestsCount := len(failedTests) - passedTestsCount := testsCount - failedTestsCount + skippedTestsCount := len(skippedTests) + passedTestsCount := len(passedTests) var testUIParts []string if failedTestsCount != 0 { @@ -347,13 +369,18 @@ func printFilesStats(w io.Writer, allTests []TestResult) bool { ansi.Compose(ansi.ColorBrightGreen, ansi.Bold)(fmt.Sprintf("%d passed", passedTestsCount)), ) } + if skippedTestsCount != 0 { + testUIParts = append(testUIParts, + ansi.Compose(ansi.ColorBrightYellow, ansi.Bold)(fmt.Sprintf("%d skipped", skippedTestsCount)), + ) + } testsUI := strings.Join(testUIParts, ansi.ColorBrightBlack(" | ")) totalTestsUI := ansi.ColorBrightBlack(fmt.Sprintf("(%d)", testsCount)) _, _ = fmt.Fprintln(w, paddedLabel(testsLabel)+" "+testsUI+" "+totalTestsUI) - return failedTestsCount == 0 + return failedTestsCount+skippedTestsCount == 0 } } diff --git a/internal/specs_format/runner_test.go b/internal/specs_format/runner_test.go index 9a75c242..cb543b02 100644 --- a/internal/specs_format/runner_test.go +++ b/internal/specs_format/runner_test.go @@ -211,3 +211,48 @@ func TestRuntimeErr(t *testing.T) { require.False(t, success) snaps.MatchSnapshot(t, out.String()) } + +func TestFocusUi(t *testing.T) { + + specs := `{ + "testCases": [ + { + "it": "t1", + "variables": { "source": "src", "amount": "10" }, + "balances": { "src": { "USD": 9999 } }, + "expect.postings": [ + { "source": "src", "destination": "dest", "asset": "USD", "amount": 42 } + ] + }, + { + "it": "t2", + "focus": true, + "variables": { "source": "src", "amount": "42" }, + "balances": { "src": { "USD": 9999 } }, + "expect.postings": [ + { "source": "src", "destination": "dest", "asset": "USD", "amount": 42 } + ] + } + ] + }` + + var out bytes.Buffer + success := specs_format.RunSpecs(&out, &out, []specs_format.RawSpec{ + { + NumscriptPath: "example.num", + SpecsPath: "example.num.specs.json", + NumscriptContent: ` vars { + account $source + number $amount + } + + send [USD $amount] ( + source = $source + destination = @dest + )`, + SpecsFileContent: []byte(specs), + }, + }) + require.False(t, success) + snaps.MatchSnapshot(t, out.String()) +} From dfb55a462f50a9b6fc7834b239e483e686dc61f9 Mon Sep 17 00:00:00 2001 From: ascandone Date: Wed, 10 Sep 2025 11:29:51 +0200 Subject: [PATCH 10/10] improve test ui --- .../specs_format/__snapshots__/runner_test.snap | 2 ++ internal/specs_format/runner.go | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/internal/specs_format/__snapshots__/runner_test.snap b/internal/specs_format/__snapshots__/runner_test.snap index 345a0c4a..8eab7c2a 100755 --- a/internal/specs_format/__snapshots__/runner_test.snap +++ b/internal/specs_format/__snapshots__/runner_test.snap @@ -11,6 +11,7 @@ [TestSingleTest - 1] ❯ example.num (2 tests | 1 failed) × tfailing + ✓ tpassing  FAIL  example.num.specs.json > tfailing @@ -45,6 +46,7 @@ GOT: [TestComplexAssertions - 1] ❯ example.num (2 tests | 1 failed) × send when there are enough funds + ✓ tpassing  FAIL  example.num.specs.json > send when there are enough funds diff --git a/internal/specs_format/runner.go b/internal/specs_format/runner.go index 8ea4f17f..afa00634 100644 --- a/internal/specs_format/runner.go +++ b/internal/specs_format/runner.go @@ -134,6 +134,10 @@ func RunSpecs(stdout io.Writer, stderr io.Writer, rawSpecs []RawSpec) bool { } +var passingTestIcon = ansi.ColorGreen("✓") +var skippedTestIcon = ansi.ColorBrightBlack("↓") +var failingTestIcon = ansi.ColorRed("×") + func runRawSpec(stdout io.Writer, stderr io.Writer, rawSpec RawSpec) (Specs, SpecsResult, bool) { parseResult := parser.Parse(rawSpec.NumscriptContent) if len(parseResult.Errors) != 0 { @@ -184,14 +188,17 @@ func runRawSpec(stdout io.Writer, stderr io.Writer, rawSpec RawSpec) (Specs, Spe if out.Failing != 0 { _, _ = fmt.Fprintf(stdout, "%s %s %s\n", ansi.ColorRed("❯"), rawSpec.NumscriptPath, testsCount) for _, result := range out.Cases { - if result.Pass || result.Skipped { - continue + if result.Pass { + _, _ = fmt.Fprintf(stdout, " %s %s\n", passingTestIcon, result.It) + } else if result.Skipped { + _, _ = fmt.Fprintf(stdout, " %s %s\n", skippedTestIcon, result.It) + } else { + _, _ = fmt.Fprintf(stdout, " %s %s\n", failingTestIcon, result.It) } - _, _ = fmt.Fprintf(stdout, " %s %s\n", ansi.ColorRed("×"), result.It) } } else { - _, _ = fmt.Fprintf(stdout, "%s %s %s\n", ansi.ColorGreen("✓"), rawSpec.NumscriptPath, testsCount) + _, _ = fmt.Fprintf(stdout, "%s %s %s\n", passingTestIcon, rawSpec.NumscriptPath, testsCount) } return specs, out, true