diff --git a/internal/interpreter/balances.go b/internal/interpreter/balances.go index 222eb865..b1f31881 100644 --- a/internal/interpreter/balances.go +++ b/internal/interpreter/balances.go @@ -100,3 +100,12 @@ func CompareBalances(b1 Balances, b2 Balances) bool { return ab1.Cmp(ab2) == 0 }) } + +// Returns whether the second value is a subset of the first one +func CompareBalancesIncluding(b1 Balances, b2 Balances) bool { + return utils.MapIncludes(b1, b2, func(a1 AccountBalance, a2 AccountBalance) bool { + return utils.MapIncludes(a1, a2, func(i1 *big.Int, i2 *big.Int) bool { + return i1.Cmp(i2) == 0 + }) + }) +} diff --git a/internal/interpreter/balances_test.go b/internal/interpreter/balances_test.go index f41e24c9..8f5268af 100644 --- a/internal/interpreter/balances_test.go +++ b/internal/interpreter/balances_test.go @@ -79,3 +79,67 @@ func TestCmpMaps(t *testing.T) { require.Equal(t, false, CompareBalances(b1, b2)) } + +func TestCmpMapsIncluding(t *testing.T) { + + t.Run("including (subset)", func(t *testing.T) { + b1 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(100), + }, + "bob": AccountBalance{ + "EUR": big.NewInt(100), + }, + } + + b2 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(100), + }, + } + + require.Equal(t, true, CompareBalancesIncluding(b1, b2)) + }) + + t.Run("different value", func(t *testing.T) { + b1 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(100), + }, + "bob": AccountBalance{ + "EUR": big.NewInt(100), + }, + } + + b2 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(0), + }, + } + + require.Equal(t, false, CompareBalancesIncluding(b1, b2)) + }) + + t.Run("extra value", func(t *testing.T) { + b1 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(100), + }, + "bob": AccountBalance{ + "EUR": big.NewInt(100), + }, + } + + b2 := Balances{ + "alice": AccountBalance{ + "EUR": big.NewInt(100), + }, + + "extra-value": AccountBalance{ + "EUR": big.NewInt(100), + }, + } + + require.Equal(t, false, CompareBalancesIncluding(b1, b2)) + }) +} diff --git a/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json index 431cca04..c54424ab 100644 --- a/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/experimental/mixed-source-prefer-single-source.num.specs.json @@ -75,7 +75,7 @@ "USD": 4 } }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/numscript-cookbook/experimental/top-up.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/experimental/top-up.num.specs.json index fbaddc23..05fcf681 100644 --- a/internal/interpreter/testdata/numscript-cookbook/experimental/top-up.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/experimental/top-up.num.specs.json @@ -10,7 +10,7 @@ { "it": "should send the missing amount to an overdraft account", "balances": { "alice": { "EUR": -100 } }, - "expect.volumes": { + "expect.endBalances": { "alice": { "EUR": 0 }, "world": { "EUR": -100 } }, diff --git a/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json index fd4844de..c0e92c9a 100644 --- a/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/experimental/transfer-example.num.specs.json @@ -31,7 +31,7 @@ "wallet": { "EUR": 50 }, "bank_account": { "EUR": -100 } }, - "expect.missingFunds": true + "expect.error.missingFunds": true }, { "it": "should not authorize transfer if bank account does not display enough balance and wallet does", @@ -39,7 +39,7 @@ "wallet": { "EUR": 100 }, "bank_account": { "EUR": -50 } }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json b/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json index 0ef59acf..00098115 100644 --- a/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json +++ b/internal/interpreter/testdata/numscript-cookbook/top-up-max.num.specs.json @@ -49,7 +49,7 @@ "EUR/2": 51 } }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json b/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json index ad2f94d1..f71564aa 100644 --- a/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json +++ b/internal/interpreter/testdata/script-tests/experimental/asset-colors/color-restrict-balance-when-missing-funds.num.specs.json @@ -1,7 +1,5 @@ { - "featureFlags": [ - "experimental-asset-colors" - ], + "featureFlags": ["experimental-asset-colors"], "testCases": [ { "it": "-", @@ -11,7 +9,7 @@ "COIN_RED": 1 } }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json b/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json index 6e799080..a506b5eb 100644 --- a/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json +++ b/internal/interpreter/testdata/script-tests/experimental/oneof/oneof-all-failing.num.specs.json @@ -1,7 +1,5 @@ { - "featureFlags": [ - "experimental-oneof" - ], + "featureFlags": ["experimental-oneof"], "testCases": [ { "it": "-", @@ -10,7 +8,7 @@ "empty2": {}, "empty3": {} }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json b/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json index 53642e8b..3a98515e 100644 --- a/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json +++ b/internal/interpreter/testdata/script-tests/insufficient-funds.num.specs.json @@ -15,7 +15,7 @@ "payment": "payments:001", "seller": "users:002" }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json b/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json index d6ff9698..540a5447 100644 --- a/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json +++ b/internal/interpreter/testdata/script-tests/overdraft-when-not-enough-funds.num.specs.json @@ -7,7 +7,7 @@ "COIN": 1 } }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json b/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json index 00f4e4ab..b714008c 100644 --- a/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json +++ b/internal/interpreter/testdata/script-tests/save/save-from-account__save-causes-failure.num.specs.json @@ -7,7 +7,7 @@ "USD/2": 30 } }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json b/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json index 42603ca8..15efe3cb 100644 --- a/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json +++ b/internal/interpreter/testdata/script-tests/source-allotment-invalid-amt.num.specs.json @@ -7,7 +7,7 @@ "COIN": 1 } }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json b/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json index 9a740b96..fe4851a6 100644 --- a/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json +++ b/internal/interpreter/testdata/script-tests/track-balances2.num.specs.json @@ -7,7 +7,7 @@ "COIN": 60 } }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } diff --git a/internal/specs_format/__snapshots__/runner_test.snap b/internal/specs_format/__snapshots__/runner_test.snap index 0cfee566..8eab7c2a 100755 --- a/internal/specs_format/__snapshots__/runner_test.snap +++ b/internal/specs_format/__snapshots__/runner_test.snap @@ -9,11 +9,18 @@ --- [TestSingleTest - 1] -❯ example.num (2 tests | 1 failed) +❯ example.num (2 tests | 1 failed) × tfailing + ✓ tpassing  FAIL  example.num.specs.json > tfailing + +GOT: + +| Source | Destination | Asset | Amount | +| world | dest | USD/2 | 100 | + - Expected + Received @@ -37,8 +44,9 @@ --- [TestComplexAssertions - 1] -❯ example.num (2 tests | 1 failed) +❯ example.num (2 tests | 1 failed) × send when there are enough funds + ✓ tpassing  FAIL  example.num.specs.json > send when there are enough funds @@ -48,16 +56,22 @@ GIVEN: | Account | Asset | Balance | | alice | USD/2 | 9999 | + +GOT: + +| Source | Destination | Asset | Amount | +| alice | dest | USD/2 | 100 | + - Expected + Received  -expect.missingFunds +expect.error.missingFunds - true + false -expect.volumes +expect.endBalances  {  "alice": { @@ -136,3 +150,11 @@ Error: example.num:1:29 | ~~~~~~ --- + +[TestFocusUi - 1] +✓ example.num (2 tests | 1 skipped) + + Test files  1 passed (1) + Tests  1 passed | 1 skipped (2) + +--- diff --git a/internal/specs_format/index.go b/internal/specs_format/index.go index f4f27d6a..946f6b69 100644 --- a/internal/specs_format/index.go +++ b/internal/specs_format/index.go @@ -4,6 +4,7 @@ import ( "context" "math/big" "reflect" + "slices" "github.com/formancehq/numscript/internal/interpreter" "github.com/formancehq/numscript/internal/parser" @@ -20,27 +21,40 @@ type Specs struct { } type TestCase struct { - It string `json:"it"` + It string `json:"it"` + + // Preconditions Balances interpreter.Balances `json:"balances,omitempty"` Vars interpreter.VariablesMap `json:"variables,omitempty"` Meta interpreter.AccountsMetadata `json:"metadata,omitempty"` + // Select tests + Focus bool `json:"focus,omitempty"` + Skip bool `json:"skip,omitempty"` + // Expectations - ExpectMissingFunds bool `json:"expect.missingFunds,omitempty"` - ExpectPostings []interpreter.Posting `json:"expect.postings,omitempty"` - ExpectTxMeta map[string]string `json:"expect.txMetadata,omitempty"` - ExpectAccountsMeta interpreter.AccountsMetadata `json:"expect.metadata,omitempty"` - ExpectVolumes interpreter.Balances `json:"expect.volumes,omitempty"` - ExpectMovements Movements `json:"expect.movements,omitempty"` + ExpectMissingFunds bool `json:"expect.error.missingFunds,omitempty"` + ExpectNegativeAmount bool `json:"expect.error.negativeAmount,omitempty"` + + ExpectPostings []interpreter.Posting `json:"expect.postings,omitempty"` + ExpectTxMeta map[string]string `json:"expect.txMetadata,omitempty"` + ExpectAccountsMeta interpreter.AccountsMetadata `json:"expect.metadata,omitempty"` + ExpectEndBalances interpreter.Balances `json:"expect.endBalances,omitempty"` + ExpectEndBalancesInclude interpreter.Balances `json:"expect.endBalances.include,omitempty"` + ExpectMovements Movements `json:"expect.movements,omitempty"` } type TestCaseResult struct { + Skipped bool `json:"skipped"` It string `json:"it"` Pass bool `json:"pass"` Balances interpreter.Balances `json:"balances"` Vars interpreter.VariablesMap `json:"variables"` Meta interpreter.AccountsMetadata `json:"metadata"` + // Output: + Postings []interpreter.Posting `json:"postings"` + // Assertions FailedAssertions []AssertionMismatch[any] `json:"failedAssertions"` } @@ -50,6 +64,7 @@ type SpecsResult struct { Total uint `json:"total"` Passing uint `json:"passing"` Failing uint `json:"failing"` + Skipped uint `json:"skipped"` Cases []TestCaseResult } @@ -68,14 +83,26 @@ func runAssertion[T any](failedAssertions []AssertionMismatch[any], assertion st func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.InterpreterError) { specsResult := SpecsResult{} + // we need a first pass to know whether there is at least on test in focus mode + hasFocusedTest := slices.ContainsFunc(specs.TestCases, func(t TestCase) bool { + return t.Focus + }) for _, testCase := range specs.TestCases { + shouldSkip := testCase.Skip || (hasFocusedTest && !testCase.Focus) + if shouldSkip { + specsResult.Skipped += 1 + specsResult.Cases = append(specsResult.Cases, TestCaseResult{ + It: testCase.It, + Skipped: true, + }) + continue + } + meta := mergeAccountsMeta(specs.Meta, testCase.Meta) balances := mergeBalances(specs.Balances, testCase.Balances) vars := mergeVars(specs.Vars, testCase.Vars) - specsResult.Total += 1 - featureFlags := make(map[string]struct{}) for _, flag := range specs.FeatureFlags { featureFlags[flag] = struct{}{} @@ -95,24 +122,39 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp var failedAssertions []AssertionMismatch[any] if err != nil { - _, ok := err.(interpreter.MissingFundsErr) - if !ok { + switch err.(type) { + case interpreter.MissingFundsErr: + if !testCase.ExpectMissingFunds { + failedAssertions = append(failedAssertions, AssertionMismatch[any]{ + Assertion: "expect.error.missingFunds", + Expected: false, + Got: true, + }) + } + case interpreter.NegativeAmountErr: + if !testCase.ExpectNegativeAmount { + failedAssertions = append(failedAssertions, AssertionMismatch[any]{ + Assertion: "expect.error.negativeAmount", + Expected: false, + Got: true, + }) + } + default: return SpecsResult{}, err } + } else { - if !testCase.ExpectMissingFunds { + if testCase.ExpectMissingFunds { failedAssertions = append(failedAssertions, AssertionMismatch[any]{ - Assertion: "expect.missingFunds", - Expected: false, - Got: true, + Assertion: "expect.error.missingFunds", + Expected: true, + Got: false, }) } - } else { - - if testCase.ExpectMissingFunds { + if testCase.ExpectNegativeAmount { failedAssertions = append(failedAssertions, AssertionMismatch[any]{ - Assertion: "expect.missingFunds", + Assertion: "expect.error.negativeAmount", Expected: true, Got: false, }) @@ -149,15 +191,24 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp ) } - if testCase.ExpectVolumes != nil { + if testCase.ExpectEndBalances != nil { failedAssertions = runAssertion(failedAssertions, - "expect.volumes", - testCase.ExpectVolumes, - getVolumes(result.Postings, balances), + "expect.endBalances", + testCase.ExpectEndBalances, + getBalances(result.Postings, balances), interpreter.CompareBalances, ) } + if testCase.ExpectEndBalancesInclude != nil { + failedAssertions = runAssertion(failedAssertions, + "expect.endBalances.include", + testCase.ExpectEndBalancesInclude, + getBalances(result.Postings, balances), + interpreter.CompareBalancesIncluding, + ) + } + if testCase.ExpectMovements != nil { failedAssertions = runAssertion[any](failedAssertions, "expect.movements", @@ -176,6 +227,11 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp specsResult.Failing += 1 } + var postings []interpreter.Posting + if result != nil { + postings = result.Postings + } + specsResult.Cases = append(specsResult.Cases, TestCaseResult{ It: testCase.It, Pass: pass, @@ -183,9 +239,12 @@ func Check(program parser.Program, specs Specs) (SpecsResult, interpreter.Interp Balances: balances, Vars: vars, FailedAssertions: failedAssertions, + Postings: postings, }) } + specsResult.Total = specsResult.Failing + specsResult.Passing + specsResult.Skipped + return specsResult, nil } @@ -239,7 +298,7 @@ func getMovements(postings []interpreter.Posting) Movements { return m } -func getVolumes(postings []interpreter.Posting, initialBalances interpreter.Balances) interpreter.Balances { +func getBalances(postings []interpreter.Posting, initialBalances interpreter.Balances) interpreter.Balances { balances := initialBalances.DeepClone() for _, posting := range postings { sourceBalance := utils.NestedMapGetOrPutDefault(balances, posting.Source, posting.Asset, func() *big.Int { diff --git a/internal/specs_format/run_test.go b/internal/specs_format/run_test.go index 16582ca0..dc7be8bd 100644 --- a/internal/specs_format/run_test.go +++ b/internal/specs_format/run_test.go @@ -63,22 +63,15 @@ func TestRunSpecsSimple(t *testing.T) { }, Meta: interpreter.AccountsMetadata{}, FailedAssertions: nil, - // ExpectedPostings: []interpreter.Posting{ - // { - // Source: "src", - // Destination: "dest", - // Asset: "USD", - // Amount: big.NewInt(42), - // }, - // }, - // ActualPostings: []interpreter.Posting{ - // { - // Source: "src", - // Destination: "dest", - // Asset: "USD", - // Amount: big.NewInt(42), - // }, - // }, + + Postings: []interpreter.Posting{ + { + Source: "src", + Destination: "dest", + Asset: "USD", + Amount: big.NewInt(42), + }, + }, }, }, }, out) @@ -134,22 +127,14 @@ func TestRunSpecsMergeOuter(t *testing.T) { }, }, FailedAssertions: nil, - // ExpectedPostings: []interpreter.Posting{ - // { - // Source: "src", - // Destination: "dest", - // Asset: "USD", - // Amount: big.NewInt(1), - // }, - // }, - // ActualPostings: []interpreter.Posting{ - // { - // Source: "src", - // Destination: "dest", - // Asset: "USD", - // Amount: big.NewInt(1), - // }, - // }, + Postings: []interpreter.Posting{ + { + Source: "src", + Destination: "dest", + Asset: "USD", + Amount: big.NewInt(1), + }, + }, }, }, }, out) @@ -163,7 +148,7 @@ func TestRunWithMissingBalance(t *testing.T) { "it": "t1", "variables": { "source": "src", "amount": "42" }, "balances": { "src": { "USD": 1 } }, - "expect.missingFunds": false, + "expect.error.missingFunds": false, "expect.postings": null } ] @@ -196,7 +181,7 @@ func TestRunWithMissingBalance(t *testing.T) { Meta: interpreter.AccountsMetadata{}, FailedAssertions: []specs_format.AssertionMismatch[any]{ { - Assertion: "expect.missingFunds", + Assertion: "expect.error.missingFunds", Expected: false, Got: true, }, @@ -250,7 +235,7 @@ func TestRunWithMissingBalanceWhenExpectedPostings(t *testing.T) { Meta: interpreter.AccountsMetadata{}, FailedAssertions: []specs_format.AssertionMismatch[any]{ { - Assertion: "expect.missingFunds", + Assertion: "expect.error.missingFunds", Got: true, Expected: false, }, @@ -301,6 +286,156 @@ func TestNullPostingsIsNoop(t *testing.T) { }, Meta: interpreter.AccountsMetadata{}, FailedAssertions: nil, + Postings: []interpreter.Posting{}, + }, + }, + }, out) + +} + +func TestNegativeAmt(t *testing.T) { + exampleProgram := parser.Parse(` + vars { number $amt } + send [USD $amt] ( + source = @world + destination = @dest + ) + `) + + j := `{ + "testCases": [ + { + "it": "t1", + "variables": { "amt": "-100" }, + "expect.error.negativeAmount": true + } + ] + }` + + var specs specs_format.Specs + err := json.Unmarshal([]byte(j), &specs) + require.Nil(t, err) + + out, err := specs_format.Check(exampleProgram.Value, specs) + require.Nil(t, err) + + require.Equal(t, specs_format.SpecsResult{ + Total: 1, + Failing: 0, + Passing: 1, + Cases: []specs_format.TestCaseResult{ + { + It: "t1", + Pass: true, + Vars: interpreter.VariablesMap{ + "amt": "-100", + }, + Balances: interpreter.Balances{}, + Meta: interpreter.AccountsMetadata{}, + FailedAssertions: nil, + }, + }, + }, out) + +} + +func TestSkip(t *testing.T) { + j := `{ + "testCases": [ + { + "it": "t1", + "skip": true + } + ] + }` + + var specs specs_format.Specs + err := json.Unmarshal([]byte(j), &specs) + require.Nil(t, err) + + out, err := specs_format.Check(exampleProgram.Value, specs) + require.Nil(t, err) + + require.Equal(t, specs_format.SpecsResult{ + Total: 1, + Failing: 0, + Passing: 0, + Skipped: 1, + Cases: []specs_format.TestCaseResult{ + { + It: "t1", + Pass: false, + Skipped: true, + }, + }, + }, out) + +} + +func TestFocus(t *testing.T) { + j := `{ + "testCases": [ + { + "it": "t1", + "variables": { "source": "src", "amount": "10" }, + "balances": { "src": { "USD": 9999 } }, + "expect.postings": [ + { "source": "src", "destination": "dest", "asset": "USD", "amount": 42 } + ] + }, + { + "it": "t2", + "focus": true, + "variables": { "source": "src", "amount": "42" }, + "balances": { "src": { "USD": 9999 } }, + "expect.postings": [ + { "source": "src", "destination": "dest", "asset": "USD", "amount": 42 } + ] + } + ] + }` + + var specs specs_format.Specs + err := json.Unmarshal([]byte(j), &specs) + require.Nil(t, err) + + out, err := specs_format.Check(exampleProgram.Value, specs) + require.Nil(t, err) + + require.Equal(t, specs_format.SpecsResult{ + Total: 2, + Failing: 0, + Passing: 1, + Skipped: 1, + Cases: []specs_format.TestCaseResult{ + { + It: "t1", + Skipped: true, + }, + + { + It: "t2", + Pass: true, + Vars: interpreter.VariablesMap{ + "source": "src", + "amount": "42", + }, + Balances: interpreter.Balances{ + "src": interpreter.AccountBalance{ + "USD": big.NewInt(9999), + }, + }, + Meta: interpreter.AccountsMetadata{}, + FailedAssertions: nil, + + Postings: []interpreter.Posting{ + { + Source: "src", + Destination: "dest", + Asset: "USD", + Amount: big.NewInt(42), + }, + }, }, }, }, out) diff --git a/internal/specs_format/runner.go b/internal/specs_format/runner.go index d7537fbd..afa00634 100644 --- a/internal/specs_format/runner.go +++ b/internal/specs_format/runner.go @@ -134,6 +134,10 @@ func RunSpecs(stdout io.Writer, stderr io.Writer, rawSpecs []RawSpec) bool { } +var passingTestIcon = ansi.ColorGreen("✓") +var skippedTestIcon = ansi.ColorBrightBlack("↓") +var failingTestIcon = ansi.ColorRed("×") + func runRawSpec(stdout io.Writer, stderr io.Writer, rawSpec RawSpec) (Specs, SpecsResult, bool) { parseResult := parser.Parse(rawSpec.NumscriptContent) if len(parseResult.Errors) != 0 { @@ -161,22 +165,40 @@ func runRawSpec(stdout io.Writer, stderr io.Writer, rawSpec RawSpec) (Specs, Spe if out.Total == 0 { _, _ = fmt.Fprintln(stdout, ansi.ColorRed("Empty test suite: "+rawSpec.SpecsPath)) return Specs{}, SpecsResult{}, false - } else if out.Failing == 0 { - testsCount := ansi.ColorBrightBlack(fmt.Sprintf("(%d tests)", out.Total)) - _, _ = fmt.Fprintf(stdout, "%s %s %s\n", ansi.ColorGreen("✓"), rawSpec.NumscriptPath, testsCount) - } else { - failedTestsCount := ansi.ColorRed(fmt.Sprintf("%d failed", out.Failing)) + } - testsCount := ansi.ColorBrightBlack(fmt.Sprintf("(%d tests | %s)", out.Total, failedTestsCount)) - _, _ = fmt.Fprintf(stdout, "%s %s %s\n", ansi.ColorRed("❯"), rawSpec.NumscriptPath, testsCount) + parts := []string{ + ansi.ColorBrightBlack(fmt.Sprintf("%d tests", out.Total)), + } + + if out.Failing != 0 { + parts = append(parts, + ansi.ColorRed(fmt.Sprintf("%d failed", out.Failing)), + ) + } + if out.Skipped != 0 { + parts = append(parts, + ansi.ColorYellow(fmt.Sprintf("%d skipped", out.Skipped)), + ) + } + + testsCount := ansi.ColorBrightBlack("(") + strings.Join(parts, ansi.ColorBrightBlack(" | ")) + ansi.ColorBrightBlack(")") + + if out.Failing != 0 { + _, _ = fmt.Fprintf(stdout, "%s %s %s\n", ansi.ColorRed("❯"), rawSpec.NumscriptPath, testsCount) for _, result := range out.Cases { if result.Pass { - continue + _, _ = fmt.Fprintf(stdout, " %s %s\n", passingTestIcon, result.It) + } else if result.Skipped { + _, _ = fmt.Fprintf(stdout, " %s %s\n", skippedTestIcon, result.It) + } else { + _, _ = fmt.Fprintf(stdout, " %s %s\n", failingTestIcon, result.It) } - _, _ = fmt.Fprintf(stdout, " %s %s\n", ansi.ColorRed("×"), result.It) } + } else { + _, _ = fmt.Fprintf(stdout, "%s %s %s\n", passingTestIcon, rawSpec.NumscriptPath, testsCount) } return specs, out, true @@ -211,7 +233,7 @@ func ShowDiff(w io.Writer, expected_ any, got_ any) { } func showFailingTestCase(w io.Writer, testResult TestResult) { - if testResult.Result.Pass { + if testResult.Result.Pass || testResult.Result.Skipped { return } @@ -224,6 +246,7 @@ func showFailingTestCase(w io.Writer, testResult TestResult) { _, _ = fmt.Fprint(w, failColor(" FAIL ")) _, _ = fmt.Fprintln(w, ansi.ColorRed(" "+specsFilePath+" > "+result.It)) + // --- Preconditions showGiven := len(result.Balances) != 0 || len(result.Meta) != 0 || len(result.Vars) != 0 if showGiven { _, _ = fmt.Fprintln(w, ansi.Underline("\nGIVEN:")) @@ -247,6 +270,19 @@ func showFailingTestCase(w io.Writer, testResult TestResult) { _, _ = fmt.Fprintln(w) } + // --- Outputs + _, _ = fmt.Fprintln(w, ansi.Underline("\nGOT:")) + if len(result.Postings) != 0 { + _, _ = fmt.Fprintln(w) + _, _ = fmt.Fprintln(w, interpreter.PrettyPrintPostings(result.Postings)) + _, _ = fmt.Fprintln(w) + } else { + _, _ = fmt.Fprintln(w) + + _, _ = fmt.Fprintln(w, ansi.ColorBrightBlack("")) + _, _ = fmt.Fprintln(w) + } + _, _ = fmt.Fprintln(w) _, _ = fmt.Fprintln(w, ansi.ColorGreen("- Expected")) _, _ = fmt.Fprintln(w, ansi.ColorRed("+ Received\n")) @@ -273,8 +309,14 @@ func showErr(stderr io.Writer, filename string, script string, err interpreter.I } func printFilesStats(w io.Writer, allTests []TestResult) bool { + passedTests := utils.Filter(allTests, func(t TestResult) bool { + return t.Result.Pass + }) + skippedTests := utils.Filter(allTests, func(t TestResult) bool { + return t.Result.Skipped + }) failedTests := utils.Filter(allTests, func(t TestResult) bool { - return !t.Result.Pass + return !t.Result.Pass && !t.Result.Skipped }) testFilesLabel := "Test files" @@ -320,7 +362,8 @@ func printFilesStats(w io.Writer, allTests []TestResult) bool { testsCount := len(allTests) failedTestsCount := len(failedTests) - passedTestsCount := testsCount - failedTestsCount + skippedTestsCount := len(skippedTests) + passedTestsCount := len(passedTests) var testUIParts []string if failedTestsCount != 0 { @@ -333,13 +376,18 @@ func printFilesStats(w io.Writer, allTests []TestResult) bool { ansi.Compose(ansi.ColorBrightGreen, ansi.Bold)(fmt.Sprintf("%d passed", passedTestsCount)), ) } + if skippedTestsCount != 0 { + testUIParts = append(testUIParts, + ansi.Compose(ansi.ColorBrightYellow, ansi.Bold)(fmt.Sprintf("%d skipped", skippedTestsCount)), + ) + } testsUI := strings.Join(testUIParts, ansi.ColorBrightBlack(" | ")) totalTestsUI := ansi.ColorBrightBlack(fmt.Sprintf("(%d)", testsCount)) _, _ = fmt.Fprintln(w, paddedLabel(testsLabel)+" "+testsUI+" "+totalTestsUI) - return failedTestsCount == 0 + return failedTestsCount+skippedTestsCount == 0 } } diff --git a/internal/specs_format/runner_test.go b/internal/specs_format/runner_test.go index 21973bb0..cb543b02 100644 --- a/internal/specs_format/runner_test.go +++ b/internal/specs_format/runner_test.go @@ -92,7 +92,7 @@ func TestComplexAssertions(t *testing.T) { "balances": { "alice": { "USD/2": 9999 } }, - "expect.volumes": { + "expect.endBalances": { "alice": { "USD/2": -100 }, "dest": { "USD/2": 1 } }, @@ -101,14 +101,14 @@ func TestComplexAssertions(t *testing.T) { "dest": { "EUR": 100 } } }, - "expect.missingFunds": true + "expect.error.missingFunds": true }, { "it": "tpassing", "balances": { "alice": { "USD/2": 0 } }, - "expect.missingFunds": true + "expect.error.missingFunds": true } ] } @@ -194,7 +194,7 @@ func TestRuntimeErr(t *testing.T) { "testCases": [ { "it": "runs", - "expect.missingFunds": false + "expect.error.missingFunds": false } ] } @@ -211,3 +211,48 @@ func TestRuntimeErr(t *testing.T) { require.False(t, success) snaps.MatchSnapshot(t, out.String()) } + +func TestFocusUi(t *testing.T) { + + specs := `{ + "testCases": [ + { + "it": "t1", + "variables": { "source": "src", "amount": "10" }, + "balances": { "src": { "USD": 9999 } }, + "expect.postings": [ + { "source": "src", "destination": "dest", "asset": "USD", "amount": 42 } + ] + }, + { + "it": "t2", + "focus": true, + "variables": { "source": "src", "amount": "42" }, + "balances": { "src": { "USD": 9999 } }, + "expect.postings": [ + { "source": "src", "destination": "dest", "asset": "USD", "amount": 42 } + ] + } + ] + }` + + var out bytes.Buffer + success := specs_format.RunSpecs(&out, &out, []specs_format.RawSpec{ + { + NumscriptPath: "example.num", + SpecsPath: "example.num.specs.json", + NumscriptContent: ` vars { + account $source + number $amount + } + + send [USD $amount] ( + source = $source + destination = @dest + )`, + SpecsFileContent: []byte(specs), + }, + }) + require.False(t, success) + snaps.MatchSnapshot(t, out.String()) +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 018cddc5..02aa569d 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -65,14 +65,21 @@ func NestedMapGetOrPutDefault[T any](m map[string]map[string]T, key1 string, key return MapGetOrPutDefault(m1, key2, getDefault) } +// Returns whether m1 is equal to m2 (according to the "cmp" equality) func MapCmp[T any](m1, m2 map[string]T, cmp func(x1 T, x2 T) bool) bool { - if len(m1) != len(m2) { + // motivation: if m2 is subset of m1, and they have the same cardinality, then m1==m2 + return len(m1) == len(m2) && MapIncludes(m1, m2, cmp) +} + +// Returns whether m1 is a superset of m2 +func MapIncludes[T any](m1, m2 map[string]T, includes func(x1 T, x2 T) bool) bool { + if len(m1) < len(m2) { return false } - for k1, v1 := range m1 { - v2, ok := m2[k1] - if !ok || !cmp(v1, v2) { + for k1, v1 := range m2 { + v2, ok := m1[k1] + if !ok || !includes(v1, v2) { return false } } diff --git a/specs.schema.json b/specs.schema.json index 79a2aaf8..a3cd1680 100644 --- a/specs.schema.json +++ b/specs.schema.json @@ -34,6 +34,15 @@ "type": "string", "description": "Test case description" }, + "focus": { + "type": "boolean", + "description": "Skip all the other tests. Just for debugging: it will result in an error exit code" + }, + "skip": { + "type": "boolean", + "description": "Skip this test. Just for debugging: it will result in an error exit code" + }, + "balances": { "$ref": "#/definitions/Balances" }, @@ -48,7 +57,10 @@ "items": { "$ref": "#/definitions/Posting" } }, - "expect.volumes": { + "expect.endBalances": { + "$ref": "#/definitions/Balances" + }, + "expect.endBalances.include": { "$ref": "#/definitions/Balances" }, @@ -64,7 +76,7 @@ "$ref": "#/definitions/AccountsMetadata" }, - "expect.missingFunds": { + "expect.error.missingFunds": { "type": "boolean" } }