diff --git a/test_suite/starlark_test.py b/test_suite/starlark_test.py index e910def..caeef41 100644 --- a/test_suite/starlark_test.py +++ b/test_suite/starlark_test.py @@ -1,4 +1,5 @@ import sys +import typing import unittest import tempfile import subprocess @@ -8,11 +9,14 @@ import testenv +def indent(s: str) -> str: + return "".join(" " + line.rstrip("\n") + "\n" for line in s.splitlines()) + class StarlarkTest(unittest.TestCase): CHUNK_SEP = "---" seen_error = False - def chunks(self, path): + def chunks(self, path) -> typing.Iterable[typing.Tuple[typing.List[str], typing.List[str], int]]: code = [] expected_errors = [] # Current line no @@ -22,20 +26,23 @@ def chunks(self, path): with open(path, mode="rb") as f: for line in f: line_no += 1 - line = line.decode("utf-8") - if line.strip() == self.CHUNK_SEP: + line = line.decode("utf-8").rstrip() + if line == self.CHUNK_SEP: yield code, expected_errors, test_line_no expected_errors = [] code = [] test_line_no = line_no + 1 else: - m = re.search("### *((go|java|rust):)? *(.*)", line) + m = re.fullmatch("(.*?) *### *((go|java|rust):)? *(.*)", line) if m: - error_impl = m.group(2) + error_impl = m.group(3) assert error_impl is None or error_impl in ["go", "java", "rust"] if (not error_impl) or error_impl == impl: - expected_errors.append(m.group(3)) - code.append(line) + expected_errors.append(m.group(4)) + code.append(m.group(1)) + else: + code.append(line) + code.append("\n") assert len(expected_errors) <= 1 yield code, expected_errors, test_line_no @@ -49,13 +56,18 @@ def evaluate(self, f): else: return stdout + stderr + def mark_error(self): + if self.seen_error: + print() + self.seen_error = True + def check_output(self, output, expected, line_no): if expected and not output: - self.seen_error = True + self.mark_error() print("Test L{}: expected error: {}".format(line_no, expected)) if output and not expected: - self.seen_error = True + self.mark_error() print("Test L{}: unexpected error: {}".format(line_no, output)) output_ = output.lower() @@ -66,9 +78,10 @@ def check_output(self, output, expected, line_no): # because error message contains source snippet. # Fix it by removing `###` before evaluating. if exp not in output_ and not re.search(exp, output_): - self.seen_error = True + self.mark_error() print("Test L{}: error not found: `{}`".format(line_no, exp.encode('utf-8'))) - print("Got: `{}`".format(output.encode('utf-8'))) + print("Got:") + print(indent(output), end="") PRELUDE = """ def assert_eq(x, y): diff --git a/test_suite/testdata/go/builtins.star b/test_suite/testdata/go/builtins.star index aab0702..55dbc26 100644 --- a/test_suite/testdata/go/builtins.star +++ b/test_suite/testdata/go/builtins.star @@ -61,11 +61,20 @@ assert_eq(sorted(["two", "three", "four"], key=len, reverse=True), ["three", "four", "two"]) --- -sorted(1) ### (for parameter iterable: got int, want iterable|not a collection|not iterable|operation.*not supported) ---- -sorted([1, 2, None, 3]) ### (not implemented|cannot compare|operation.*not supported) ---- -sorted([1, "one"]) ### (string < int not implemented|cannot compare|operation.*not supported) +### java: Error in sorted +### rust: not supported +### go: Error in sorted +sorted(1) +--- +### java: unsupported +### rust: not supported +### go: Error in sorted +sorted([1, 2, None, 3]) +--- +### java: unsupported +### rust: not supported +### go: Error in sorted +sorted([1, "one"]) --- # _inconsistency_: java accepts key to be None # sorted([1, 2, 3], key=None) ## (want callable|not supported) diff --git a/test_suite/testdata/go/dict.star b/test_suite/testdata/go/dict.star index b640ecc..cf1290c 100644 --- a/test_suite/testdata/go/dict.star +++ b/test_suite/testdata/go/dict.star @@ -88,7 +88,7 @@ setIndex(x9, [], 2) ### (unhashable|not hashable) --- x9a = {} -x9a[1, 2] = 3 ### rust: TODO(stepancheg): checked incorrectly on rust because error message includes snippet +x9a[1, 2] = 3 ### rust: left-hand-side of assignment must take assert_eq(x9a.keys()[0], (1, 2)) --- diff --git a/test_suite/testdata/go/function.star b/test_suite/testdata/go/function.star index fe8d189..6cfa4ed 100644 --- a/test_suite/testdata/go/function.star +++ b/test_suite/testdata/go/function.star @@ -174,6 +174,10 @@ def f(a, b, c, d, e, f, g, h, mm): pass +### rust: more than once +### java: multiple values +### go: multiple values + f( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, @@ -183,7 +187,7 @@ f( 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, - mm = 100) ### (multiple values|argument 'mm' passed both by position and by name|Extraneous parameter|occurs both explicitly and in \*\*kwargs) + mm = 100) --- # Regression test for github.com/google/starlark-go/issues/21, diff --git a/test_suite/testdata/go/int.star b/test_suite/testdata/go/int.star index e48cb33..1ed6c0a 100644 --- a/test_suite/testdata/go/int.star +++ b/test_suite/testdata/go/int.star @@ -164,15 +164,15 @@ assert_eq(int("0b00000", 0), 0) assert_eq(11111 * 11111, 123454321) --- -int("0x123", 8) ### (invalid literal.*base 8|not a base 8|not a valid number in base 8) +int("0x123", 8) ### base --- -int("-0x123", 8) ### (invalid literal.*base 8|not a base 8|not a valid number in base 8) +int("-0x123", 8) ### base --- -int("0o123", 16) ### (invalid literal.*base 16|not a base 16|not a valid number in base 16) +int("0o123", 16) ### base --- -int("-0o123", 16) ### (invalid literal.*base 16|not a base 16|not a valid number in base 16) +int("-0o123", 16) ### base --- -int("0x110", 2) ### (invalid literal.*base 2|not a base 2|not a valid number in base 2) +int("0x110", 2) ### base --- # int from string, auto detect base assert_eq(int("123", 0), 123) @@ -194,7 +194,7 @@ assert_eq(int("-0o123", 0), -83) # github.com/google/starlark-go/issues/108 -int("0Oxa", 8) ### (invalid literal|not a base 8|not a valid number in base 8) +int("0Oxa", 8) ### base --- # _inconsistency_: some implementations allow some of these conversions diff --git a/test_suite/testdata/go/misc.star b/test_suite/testdata/go/misc.star index a91fb55..c1e662e 100644 --- a/test_suite/testdata/go/misc.star +++ b/test_suite/testdata/go/misc.star @@ -1,21 +1,45 @@ # Miscellaneous tests of Starlark evaluation. # Ordered comparisons require values of the same type. -None < False ### (not impl|cannot compare|operation.*not supported) +### java: unsupported +### go: not implemented +### rust: not supported +None < False --- -False < list ### (not impl|cannot compare|operation.*not supported) +### java: unsupported +### go: not implemented +### rust: not supported +False < list --- -list < {} ### (not impl|cannot compare|operation.*not supported) +### java: unsupported +### go: not implemented +### rust: not supported +list < {} --- -{} < None ### (not impl|cannot compare|operation.*not supported) +### java: unsupported +### go: not implemented +### rust: not supported +{} < None --- -None < 0 ### (not impl|cannot compare|operation.*not supported) +### java: unsupported +### go: not implemented +### rust: not supported +None < 0 --- -0 < [] ### (not impl|cannot compare|operation.*not supported) +### java: unsupported +### go: not implemented +### rust: not supported +0 < [] --- -[] < "" ### (not impl|cannot compare|operation.*not supported) +### java: unsupported +### go: not implemented +### rust: not supported +[] < "" --- -"" < () ### (not impl|cannot compare|operation.*not supported) +### java: unsupported +### go: not implemented +### rust: not supported +"" < () --- # _inconsistency_: cyclic data structures not supported in Rust and Java diff --git a/test_suite/testdata/go/string.star b/test_suite/testdata/go/string.star index bfee3f7..1e3f522 100644 --- a/test_suite/testdata/go/string.star +++ b/test_suite/testdata/go/string.star @@ -146,7 +146,10 @@ assert_eq("banana"[4::-2], "nnb") assert_eq("banana"[::-1], "ananab") assert_eq("banana"[None:None:-2], "aaa") --- -"banana"[:"":] ### (got.*want|parameters mismatch) +### rust: not supported +### java: want int +### go: want int +"banana"[:"":] --- "banana"[:"":True] ### (got.*want|parameters mismatch) --- @@ -374,7 +377,10 @@ assert_('abc'.startswith(('a', 'A'))) assert_('ABC'.startswith(('a', 'A'))) assert_(not 'ABC'.startswith(('b', 'B'))) --- -'123'.startswith((1, 2)) ### (got int, for element 0|type of parameter.*doesn't match) +### rust: Type of parameter +### java: want string +### go: want string +'123'.startswith((1, 2)) --- # _inconsistency_: rust startswith allows a list, not just tuple # https://github.com/facebookexperimental/starlark-rust/issues/23 @@ -384,7 +390,10 @@ assert_('abc'.endswith(('c', 'C'))) assert_('ABC'.endswith(('c', 'C'))) assert_(not 'ABC'.endswith(('b', 'B'))) --- -'123'.endswith((1, 2)) ### (got int, for element 0|type of parameter.*doesn't match) +### java: want string +### rust: Type of parameter +### go: want string +'123'.endswith((1, 2)) --- # _inconsistency_: rust endswith allows a lists, not just tuple # https://github.com/facebookexperimental/starlark-rust/issues/23 @@ -464,16 +473,25 @@ def args(*args): return args args(*"abc") ### (must be iterable, not string|not iterable|must be an iterable|operation.*not supported) --- # list(str) -list("abc") ### (got string, want iterable|not iterable|not a collection|operation.*not supported) +### java: got value of type +### rust: not supported +### go: got string, want iterable +list("abc") --- # tuple(str) -tuple("abc") ### (got string, want iterable|not iterable|not a collection|operation.*not supported) +### java: got value of type +### rust: not supported +### go: got string, want iterable +tuple("abc") --- # enumerate enumerate("ab") ### (got string, want iterable|not iterable|expected value of type 'sequence'|operation.*not supported) --- # sorted -sorted("abc") ### (got string, want iterable|not iterable|not a collection|operation.*not supported) +### java: got value of type +### rust: not supported +### go: got string, want iterable +sorted("abc") --- # list.extend [].extend("bc") ### (got string, want iterable|not iterable|expected value of type 'sequence'|operation.*not supported) @@ -552,8 +570,17 @@ assert_eq("¿Por qué?".title(), "¿Por Qué?") --- # method spell check -"".starts_with() ### (no .starts_with field.*did you mean .startswith|not supported|no field or method) ---- -"".StartsWith() ### (no .StartsWith field.*did you mean .startswith|not supported|no field or method) ---- -"".fin() ### (no .fin field.*.did you mean .find|not supported|no field or method) +### rust: has no attribute +### java: has no field or method +### go: field or method +"".starts_with() +--- +### rust: has no attribute +### java: has no field or method +### go: field or method +"".StartsWith() +--- +### rust: has no attribute +### java: has no field or method +### go: field or method +"".fin() diff --git a/test_suite/testdata/go/tuple.star b/test_suite/testdata/go/tuple.star index 244025e..932287d 100644 --- a/test_suite/testdata/go/tuple.star +++ b/test_suite/testdata/go/tuple.star @@ -36,7 +36,10 @@ assert_eq(tuple(["a", "b", "c"]), ("a", "b", "c")) assert_eq(tuple(["a", "b", "c"]), ("a", "b", "c")) assert_eq(tuple([1]), (1,)) --- -tuple(1) ### (got int, want iterable|not iterable|not a collection|operation.*not supported) +### go: want iterable +### java: got value of type +### rust: not supported +tuple(1) --- # tuple * int, int * tuple diff --git a/test_suite/testdata/java/int.star b/test_suite/testdata/java/int.star index 59285be..2cc0d6b 100644 --- a/test_suite/testdata/java/int.star +++ b/test_suite/testdata/java/int.star @@ -63,4 +63,4 @@ compound() --- 1 // 0 ### (division by zero|divide by zero) --- -1 % 0 ### (integer modulo by zero|division by zero|divide by zero) +1 % 0 ### by zero diff --git a/test_suite/testdata/java/int_constructor.star b/test_suite/testdata/java/int_constructor.star index bfd913d..d88cc62 100644 --- a/test_suite/testdata/java/int_constructor.star +++ b/test_suite/testdata/java/int_constructor.star @@ -19,23 +19,23 @@ assert_eq(int('0XFF', 0), 255) assert_eq(int('0xFF', 16), 255) --- -int('1.5') ### (invalid literal|not a base 10|not a valid number in base 10) +int('1.5') ### base --- -int('ab') ### (invalid literal|not a base 10|not a valid number in base 10) +int('ab') ### base --- int(None) ### None --- -int('123', 3) ### (invalid literal|not a base 3|not a valid number in base 3) +int('123', 3) ### base --- -int('FF', 15) ### (invalid literal|not a base 15|not a valid number in base 15) +int('FF', 15) ### base --- -int('123', -1) ### >= 2 (and|&&) <= 36 +int('123', -1) ### base --- -int('123', 1) ### >= 2 (and|&&) <= 36 +int('123', 1) ### base --- -int('123', 37) ### >= 2 (and|&&) <= 36 +int('123', 37) ### base --- -int('0xFF', 8) ### (invalid literal|not a base 8|not a valid number in base 8) +int('0xFF', 8) ### base --- int(True, 2) ### (can't convert non-string with explicit base|non-string) --- diff --git a/test_suite/testdata/java/int_function.star b/test_suite/testdata/java/int_function.star index 1b3a73a..c45e594 100644 --- a/test_suite/testdata/java/int_function.star +++ b/test_suite/testdata/java/int_function.star @@ -37,18 +37,35 @@ assert_eq(int('+42'), 42) --- # int(-2147483649) ## invalid base-10 integer constant: 2147483649 --- -int('') ### (cannot be empty|invalid literal|not a base 10|not a valid number in base 10) ---- -# Surrounding whitespace is not allowed -int(' 42 ') ### (invalid literal|not a base 10|not a valid number in base 10) ---- -int('-') ### (invalid literal|not a base 10|not a valid number in base 10) ---- -int('0x') ### (invalid literal|not a base 16|not a valid number in base 16) ---- -int('1.5') ### (invalid literal|not a base 10|not a valid number in base 10) ---- -int('ab') ### (invalid literal|not a base 10|not a valid number in base 10) +### rust: Cannot parse +### java: Error in int +### go: invalid literal +int('') +--- +### rust: Cannot parse +### java: Error in int +### go: invalid literal +int(' 42 ') +--- +### rust: Cannot parse +### java: Error in int +### go: invalid literal +int('-') +--- +### rust: Cannot parse +### java: Error in int +### go: invalid literal +int('0x') +--- +### rust: Cannot parse +### java: Error in int +### go: invalid literal +int('1.5') +--- +### rust: Cannot parse +### java: Error in int +### go: invalid literal +int('ab') --- assert_eq(int('11', 2), 3) @@ -67,17 +84,35 @@ assert_eq(int('016', 16), 22) # invalid base # int('016', 0) ## base.*016 --- -int('123', 3) ### (invalid literal|not a base 3|not a valid number in base 3) +### rust: Cannot parse +### java: Error in int +### go: invalid literal +int('123', 3) --- -int('FF', 15) ### (invalid literal|not a base 15|not a valid number in base 15) +### rust: Cannot parse +### java: Error in int +### go: invalid literal +int('FF', 15) --- -int('123', -1) ### >= 2 (and|&&) <= 36 +### rust: not a valid base +### java: Error in int +### go: base must be +int('123', -1) --- -int('123', 1) ### >= 2 (and|&&) <= 36 +### rust: not a valid base +### java: Error in int +### go: base must be +int('123', 1) --- -int('123', 37) ### >= 2 (and|&&) <= 36 +### rust: not a valid base +### java: Error in int +### go: base must be +int('123', 37) --- -int('123', 'x') ### (base must be an integer|for base, got string, want int|not supported) +### rust: Type of parameter +### java: Error in int +### go: want int +int('123', 'x') --- # base with prefix @@ -91,10 +126,22 @@ assert_eq(int('0XFF', 0), 255) assert_eq(int('0xFF', 16), 255) --- -int('0xFF', 8) ### (invalid literal|not a base 8|not a valid number in base 8) ---- -int(True, 2) ### (can't convert non-string with explicit base|non-string) ---- -int(1, 2) ### (can't convert non-string with explicit base|non-string) ---- -int(True, 10) ### (can't convert non-string with explicit base|non-string) +### rust: Cannot parse +### java: Error in int +### go: invalid literal +int('0xFF', 8) +--- +### rust: non-string +### java: Error in int +### go: non-string with explicit base +int(True, 2) +--- +### rust: non-string +### java: Error in int +### go: non-string with explicit base +int(1, 2) +--- +### rust: non-string +### java: Error in int +### go: non-string with explicit base +int(True, 10) diff --git a/test_suite/testdata/java/list_mutation.star b/test_suite/testdata/java/list_mutation.star index d374e59..f1151ff 100644 --- a/test_suite/testdata/java/list_mutation.star +++ b/test_suite/testdata/java/list_mutation.star @@ -18,7 +18,10 @@ foo.insert(10, 'g') assert_eq(foo, ['f', 'c', 'd', 'a', 'b', 'e', 'g']) --- -(1, 2).insert(3) ### (no (field or method 'insert|\.insert field)|not supported) +### java: has no field or method +### rust: has no attribute +### go: field or method +(1, 2).insert(3) --- # append @@ -30,7 +33,10 @@ foo.append('d') assert_eq(foo, ['a', 'b', 'c', 'd']) --- -(1, 2).append(3) ### (no (field or method 'append|\.append field)|not supported) +### rust: has no attribute +### go: field or method +### java: has no field or method +(1, 2).append(3) --- # extend @@ -41,7 +47,10 @@ foo.extend(('e', 'f')) assert_eq(foo, ['a', 'b', 'c', 'd', 'e', 'f']) --- -(1, 2).extend([3, 4]) ### (no (field or method 'extend|\.extend field)|not supported) +### rust: has no attribute +### java: has no field or method +### go: field or method +(1, 2).extend([3, 4]) --- [1, 2].extend(3) ### (expected value of type|got int, want iterable|not iterable|operation.*not supported on type) @@ -62,7 +71,10 @@ foo.remove('b') assert_eq(foo, []) --- -(1, 2).remove(3) ### (no (field or method 'remove|\.remove field)|not supported) +### rust: has no attribute +### java: has no field or method +### go: field or method +(1, 2).remove(3) --- [1, 2].remove(3) ### not found --- @@ -87,4 +99,7 @@ assert_eq(li3, [2, 4]) --- [1, 2].pop(3) ### (out of range|out of bound) --- -(1, 2).pop() ### (no (field or method 'pop|\.pop field)|not supported) +### rust: has no attribute +### java: has no field or method +### go: field or method +(1, 2).pop() diff --git a/test_suite/testdata/java/list_slices.star b/test_suite/testdata/java/list_slices.star index 81e84a2..bab3035 100644 --- a/test_suite/testdata/java/list_slices.star +++ b/test_suite/testdata/java/list_slices.star @@ -57,9 +57,15 @@ assert_eq([0, 1, 2][-1], 2) assert_eq([0, 1, 2][0], 0) --- -'123'['a'::] ### (got.*want|invalid start index|parameters mismatch) +### java: want int +### rust: not supported +### go: invalid start index +'123'['a'::] --- -'123'[:'b':] ### (got.*want|invalid end index|parameters mismatch) +### java: want int +### rust: not supported +### go: invalid end index +'123'[:'b':] --- (1, 2, 3)[1::0] ### (slice step cannot be zero|zero is not a valid slice step|out of bound) --- diff --git a/test_suite/testdata/java/string_misc.star b/test_suite/testdata/java/string_misc.star index cadc68e..44f7a7e 100644 --- a/test_suite/testdata/java/string_misc.star +++ b/test_suite/testdata/java/string_misc.star @@ -85,9 +85,15 @@ assert_eq(''.endswith(()), False) # https://github.com/facebookexperimental/starlark-rust/issues/23 # 'a'.endswith(['a']) # # # (Type of parameter.*doesn't match|got list|parameters mismatch) # --- -'1'.endswith((1,)) ### (Type of parameter.*doesn't match|got int|parameters mismatch) +### rust: Type of parameter +### java: want string +### go: want string +'1'.endswith((1,)) --- -'a'.endswith(('1', 1)) ### (Type of parameter.*doesn't match|got int|parameters mismatch) +### rust: Type of parameter +### java: want string +### go: want string +'a'.endswith(('1', 1)) --- # startswith @@ -116,9 +122,15 @@ assert_eq(''.startswith(()), False) # https://github.com/facebookexperimental/starlark-rust/issues/23 # 'a'.startswith(['a']) # # # (Type of parameter.*doesn't match|got list|expected string) # --- -'1'.startswith((1,)) ### (Type of parameter.*doesn't match|got int|expected string) +### rust: Type of parameter +### java: want string +### go: want string +'1'.startswith((1,)) --- -'a'.startswith(('1', 1)) ### (Type of parameter.*doesn't match|got int|expected string) +### rust: Type of parameter +### java: want string +### go: want string +'a'.startswith(('1', 1)) --- # substring diff --git a/test_suite/testdata/java/string_slice_index.star b/test_suite/testdata/java/string_slice_index.star index 32d6b90..a0ce62c 100644 --- a/test_suite/testdata/java/string_slice_index.star +++ b/test_suite/testdata/java/string_slice_index.star @@ -54,7 +54,13 @@ assert_eq('123'[1:3:-1], "") --- '123'[1:3:0] ### (slice step cannot be zero|not a valid slice step|out of bound) --- -'123'['a'::] ### (slice start must be an integer, not 'a'|want int|parameters mismatch) +### java: want int +### go: invalid start index +### rust: not supported +'123'['a'::] --- -'123'[:'b':] ### (slice end must be an integer, not 'b'|want int|parameters mismatch) +### java: want int +### go: invalid end index +### rust: not supported +'123'[:'b':] --- diff --git a/test_suite/testdata/rust/bool.star b/test_suite/testdata/rust/bool.star index dc2e15f..22eacd8 100644 --- a/test_suite/testdata/rust/bool.star +++ b/test_suite/testdata/rust/bool.star @@ -1,3 +1,6 @@ # Boolean tests -True + 1000 ### (parameters mismatch|unknown binary op|not supported for types) +### java: unsupported binary operation +### rust: not supported +### go: unknown binary op +True + 1000