Skip to content

Commit

Permalink
Simplify tests for splits in negative test data (#29)
Browse files Browse the repository at this point in the history 
We propagate the split of negative samples in
[aas-core3.0-testgen 08d3ec37c] into unserializable and invalid negative
examples. This makes the generation and future maintenance of the test
code easier in this SDK as well, since we now do not have to track the
individual negative cases.

In addition, we improve the error messages if the recorded test data is
missing to make it clearer for the programmer what needs to be fixed in
that case.

[aas-core3.0-testgen 08d3ec37c]: aas-core-works/aas-core3.0-testgen@08d3ec37c
  • Loading branch information
@mristin
mristin committed May 31, 2024
1 parent 7e20ecd commit 305d986
Show file tree
Hide file tree
Showing 8,423 changed files with 136,856 additions and 743 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
4 changes: 3 additions & 1 deletion dev_scripts/test_codegen/generate_test_for_descend_and_pass_through_visitor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ class TracingVisitor extends AasTypes.PassThroughVisitor {{
{I}}} else {{
{II}if (!fs.existsSync(expectedPath)) {{
{III}throw new Error(
{IIII}`The file with the recorded trace does not exist: ${{expectedPath}}`
{IIII}`The file with the recorded trace does not exist: ${{expectedPath}}; ` +
{IIII}`you probably want to set the environment ` +
{IIII}`variable ${{TestCommon.RECORD_MODE_ENVIRONMENT_VARIABLE_NAME}}?`
{III});
{II}}}
Expand Down
4 changes: 3 additions & 1 deletion dev_scripts/test_codegen/generate_test_for_descend_once.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,9 @@ def main() -> int:
{I}}} else {{
{II}if (!fs.existsSync(expectedPath)) {{
{III}throw new Error(
{IIII}`The file with the recorded trace does not exist: ${{expectedPath}}`
{IIII}`The file with the recorded trace does not exist: ${{expectedPath}}; ` +
{IIII}`you probably want to set the environment ` +
{IIII}`variable ${{TestCommon.RECORD_MODE_ENVIRONMENT_VARIABLE_NAME}}?`
{III});
{II}}}
Expand Down
4 changes: 3 additions & 1 deletion dev_scripts/test_codegen/generate_test_for_get_X_or_default.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,9 @@ class EnumerationLiteral {{
{I}}} else {{
{II}if (!fs.existsSync(expectedPath)) {{
{III}throw new Error(
{IIII}`The file with the recorded value does not exist: ${{expectedPath}}`
{IIII}`The file with the recorded value does not exist: ${{expectedPath}}; ` +
{IIII}`you probably want to set the environment ` +
{IIII}`variable ${{TestCommon.RECORD_MODE_ENVIRONMENT_VARIABLE_NAME}}?`
{III});
{II}}}
Expand Down
164 changes: 99 additions & 65 deletions dev_scripts/test_codegen/generate_test_for_jsonization_of_concrete_classes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,24 +83,39 @@ def _generate_for_self_contained(
Stripped(
f"""\
test("{cls_name_typescript} deserialization fail", () => {{
{I}for (const cause of CAUSES_FOR_DESERIALIZATION_FAILURE) {{
{II}const baseDir = path.join(
{III}TestCommon.TEST_DATA_DIR,
{III}"Json",
{III}"SelfContained",
{III}"Unexpected",
{III}cause,
{I}for (
{II}const causeDir of
{II}TestCommon.findImmediateSubdirectories(
{III}path.join(
{IIII}TestCommon.TEST_DATA_DIR,
{IIII}"Json",
{IIII}"SelfContained",
{IIII}"Unexpected",
{IIII}"Unserializable"
{III})
{II})
{I}) {{
{II}// NOTE (mristin):
{II}// Unlike other SDKs, we can not be really sure what additional properties
{II}// JavaScript might bring about. Therefore, we leave out the tests with
{II}// the validation of additional properties.
{II}if (path.basename(causeDir) == "UnexpectedAdditionalProperty") {{
{III}continue;
{II}}}
{II}const clsDir = path.join(
{III}causeDir,
{III}{typescript_common.string_literal(cls_name_json)}
{II});
{II}if (!fs.existsSync(baseDir)) {{
{III}// No examples of {cls_name_typescript} exist for the failure cause.
{II}if (!fs.existsSync(clsDir)) {{
{III}// NOTE (mristin):
{III}// Some classes indeed lack the invalid examples.
{III}continue;
{II}}}
{II}}}
{II}const pths = Array.from(
{III}TestCommon.findFilesBySuffixRecursively(
{IIII}baseDir,
{IIII}clsDir,
{IIII}".json"
{III})
{II});
Expand All @@ -127,24 +142,31 @@ def _generate_for_self_contained(
Stripped(
f"""\
test("{cls_name_typescript} verification fail", () => {{
{I}for (const cause of TestCommon.CAUSES_FOR_VERIFICATION_FAILURE) {{
{II}const baseDir = path.join(
{III}TestCommon.TEST_DATA_DIR,
{III}"Json",
{III}"SelfContained",
{III}"Unexpected",
{III}cause,
{I}for (
{II}const causeDir of
{II}TestCommon.findImmediateSubdirectories(
{III}path.join(
{IIII}TestCommon.TEST_DATA_DIR,
{IIII}"Json",
{IIII}"SelfContained",
{IIII}"Unexpected",
{IIII}"Invalid"
{III})
{II})
{I}) {{
{II}const clsDir = path.join(
{III}causeDir,
{III}{typescript_common.string_literal(cls_name_json)}
{II});
{II}if (!fs.existsSync(baseDir)) {{
{III}// No examples of {cls_name_typescript} exist for the failure cause.
{II}if (!fs.existsSync(clsDir)) {{
{III}// NOTE (mristin):
{III}// Some classes indeed lack the invalid examples.
{III}continue;
{II}}}
{II}}}
{II}const pths = Array.from(
{III}TestCommon.findFilesBySuffixRecursively(
{IIII}baseDir,
{IIII}clsDir,
{IIII}".json"
{III})
{II});
Expand Down Expand Up @@ -236,24 +258,39 @@ def _generate_for_contained_in_container(
Stripped(
f"""\
test("{cls_name_typescript} deserialization fail", () => {{
{I}for (const cause of CAUSES_FOR_DESERIALIZATION_FAILURE) {{
{II}const baseDir = path.join(
{III}TestCommon.TEST_DATA_DIR,
{III}"Json",
{III}"ContainedIn{container_cls_json}",
{III}"Unexpected",
{III}cause,
{I}for (
{II}const causeDir of
{II}TestCommon.findImmediateSubdirectories(
{III}path.join(
{IIII}TestCommon.TEST_DATA_DIR,
{IIII}"Json",
{IIII}"ContainedIn{container_cls_json}",
{IIII}"Unexpected",
{IIII}"Unserializable"
{III})
{II})
{I}) {{
{II}// NOTE (mristin):
{II}// Unlike other SDKs, we can not be really sure what additional properties
{II}// JavaScript might bring about. Therefore, we leave out the tests with
{II}// the validation of additional properties.
{II}if (path.basename(causeDir) == "UnexpectedAdditionalProperty") {{
{III}continue;
{II}}}
{II}const clsDir = path.join(
{III}causeDir,
{III}{typescript_common.string_literal(cls_name_json)}
{II});
{II}if (!fs.existsSync(baseDir)) {{
{III}// No examples of {cls_name_typescript} exist for the failure cause.
{II}if (!fs.existsSync(clsDir)) {{
{III}// NOTE (mristin):
{III}// Some classes indeed lack the invalid examples.
{III}continue;
{II}}}
{II}}}
{II}const pths = Array.from(
{III}TestCommon.findFilesBySuffixRecursively(
{IIII}baseDir,
{IIII}clsDir,
{IIII}".json"
{III})
{II});
Expand Down Expand Up @@ -282,24 +319,31 @@ def _generate_for_contained_in_container(
Stripped(
f"""\
test("{cls_name_typescript} verification fail", () => {{
{I}for (const cause of TestCommon.CAUSES_FOR_VERIFICATION_FAILURE) {{
{II}const baseDir = path.join(
{III}TestCommon.TEST_DATA_DIR,
{III}"Json",
{III}"ContainedIn{container_cls_json}",
{III}"Unexpected",
{III}cause,
{I}for (
{II}const causeDir of
{II}TestCommon.findImmediateSubdirectories(
{III}path.join(
{IIII}TestCommon.TEST_DATA_DIR,
{IIII}"Json",
{IIII}"ContainedIn{container_cls_json}",
{IIII}"Unexpected",
{IIII}"Invalid"
{III})
{II})
{I}) {{
{II}const clsDir = path.join(
{III}causeDir,
{III}{typescript_common.string_literal(cls_name_json)}
{II});
{II}if (!fs.existsSync(baseDir)) {{
{III}// No examples of {cls_name_typescript} exist for the failure cause.
{II}if (!fs.existsSync(clsDir)) {{
{III}// NOTE (mristin):
{III}// Some classes indeed lack the invalid examples.
{III}continue;
{II}}}
{II}}}
{II}const pths = Array.from(
{III}TestCommon.findFilesBySuffixRecursively(
{IIII}baseDir,
{IIII}clsDir,
{IIII}".json"
{III})
{II});
Expand Down Expand Up @@ -400,20 +444,6 @@ def main() -> int:
),
Stripped(
f"""\
const CAUSES_FOR_DESERIALIZATION_FAILURE = [
{I}"TypeViolation",
{I}"RequiredViolation",
{I}"EnumViolation",
{I}"NullViolation",
{I}// NOTE (mristin, 2022-12-09):
{I}// Unlike other SDKs, we can not be really sure what additional properties
{I}// JavaScript might bring about. Therefore, we leave out the tests with
{I}// the validation of additional properties.
{I}// "UnexpectedAdditionalProperty"
];"""
),
Stripped(
f"""\
/**
* Assert that the deserialization error equals the expected golden one,
* or, if {{@link common.RECORD_MODE}} set, re-record the expected error.
Expand All @@ -434,7 +464,9 @@ def main() -> int:
{I}}} else {{
{II}if (!fs.existsSync(errorPath)) {{
{III}throw new Error(
{IIII}`The file with the recorded error does not exist: ${{errorPath}}`
{IIII}`The file with the recorded deserialization error does ` +
{IIII}`not exist: ${{errorPath}}; you probably want to set ` +
{IIII}`the environment variable ${{TestCommon.RECORD_MODE_ENVIRONMENT_VARIABLE_NAME}}?`
{III});
{II}}}
Expand Down Expand Up @@ -482,7 +514,9 @@ def main() -> int:
{I}}} else {{
{II}if (!fs.existsSync(errorsPath)) {{
{III}throw new Error(
{IIII}`The file with the recorded errors does not exist: ${{errorsPath}}`
{IIII}`The file with the recorded verification errors ` +
{IIII}`does not exist: ${{errorsPath}}; you probably want to set the environment ` +
{IIII}`variable ${{TestCommon.RECORD_MODE_ENVIRONMENT_VARIABLE_NAME}}?`
{III});
{II}}}
Expand All @@ -498,7 +532,7 @@ def main() -> int:
{II}}}
{I}}}
}}"""
)
),
] # type: List[Stripped]

environment_cls = symbol_table.must_find_concrete_class(Identifier("Environment"))
Expand Down
41 changes: 29 additions & 12 deletions test/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,11 @@ import { Path } from "../src/jsonization";
// It is tedious to record manually all the expected error messages. Therefore we include this variable
// to steer the automatic recording. We intentionally inter-twine the recording code with the test code
// to keep them close to each other so that they are easier to maintain.
export const RECORD_MODE_ENVIRONMENT_VARIABLE_NAME =
"AAS_CORE3_0_TYPESCRIPT_RECORD_MODE";

const RECORD_MODE_TEXT =
process.env["AAS_CORE3_0_TYPESCRIPT_RECORD_MODE"]?.toLowerCase();
process.env[RECORD_MODE_ENVIRONMENT_VARIABLE_NAME]?.toLowerCase();
export const RECORD_MODE: boolean =
RECORD_MODE_TEXT === "true" || RECORD_MODE_TEXT === "1" || RECORD_MODE_TEXT === "on";

Expand Down Expand Up @@ -117,17 +120,6 @@ export function assertNoVerificationErrors(
}
}

export const CAUSES_FOR_VERIFICATION_FAILURE = [
"DateTimeStampUtcViolationOnFebruary29th",
"MaxLengthViolation",
"MinLengthViolation",
"PatternViolation",
"InvalidValueExample",
"InvalidMinMaxExample",
"SetViolation",
"ConstraintViolation"
];

/**
* Assert that `errors` either correspond to the errors recorded to the disk,
* or re-record the errors, if {@link RECORD_MODE} is set.
Expand Down Expand Up @@ -156,6 +148,14 @@ export function assertExpectedOrRecordedVerificationErrors(
if (RECORD_MODE) {
fs.writeFileSync(errorsPath, got, "utf-8");
} else {
if (!fs.existsSync(errorsPath)) {
throw new Error(
`The file with the recorded verification errors does not ` +
`exist: ${errorsPath}; you probably want to set the environment ` +
`variable ${RECORD_MODE_ENVIRONMENT_VARIABLE_NAME}?`
);
}

const expected = fs.readFileSync(errorsPath, "utf-8");
if (expected !== got) {
throw new Error(
Expand Down Expand Up @@ -214,6 +214,23 @@ export function* findFilesBySuffixRecursively(
}
}

/**
* Iterate over all the immediate subdirectories `directory`.
*
* @param directory - to iterate through
*/
export function* findImmediateSubdirectories(
directory: string
): IterableIterator<string> {
for (const filename of fs.readdirSync(directory)) {
const pth = path.join(directory, filename);
const stat = fs.lstatSync(pth);
if (stat.isDirectory()) {
yield pth;
}
}
}

/**
* Signal that two JSON-able structures are unequal.
*/
Expand Down
Loading

0 comments on commit 305d986

Please sign in to comment.