From d7dd02dd3214c4ac18a0f4f8cddbe2094ce418e3 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Thu, 19 Mar 2020 11:36:59 +0100 Subject: [PATCH 01/24] [validation.py] Add Possible DType Check for String Property Values Checks, whether a Property with dtype string might be a different dtype but could not be detected when reading a file or property. --- odml/validation.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/odml/validation.py b/odml/validation.py index 21498f15..73ca6a64 100644 --- a/odml/validation.py +++ b/odml/validation.py @@ -3,6 +3,7 @@ Generic odML validation framework. """ +import re from . import dtypes LABEL_ERROR = 'error' @@ -386,3 +387,53 @@ def property_values_check(prop): Validation.register_handler('property', property_values_check) + + +def property_values_string_check(prop): + """ + PROTOTYPE + + Tests whether values with dtype "string" are maybe of different dtype. + + :param prop: property the validation is applied on. + """ + + if prop.dtype != "string" or not prop.values: + return + + dtype_checks = { + 'int': r'^(-+)?\d+$', + 'date': r'^\d{2,4}-\d{1,2}-\d{1,2}$', + 'datetime': r'^\d{2,4}-\d{1,2}-\d{1,2} \d{2}:\d{2}(:\d{2})?$', + 'time': r'^\d{2}:\d{2}(:\d{2})?$', + 'float': r'^(-+)?\d+\.\d+$', + 'tuple': r'^\((.*?)\)', + 'boolean': r'^TRUE|FALSE|True|False|t|f+$', + 'text': r'[\r\n]'} + + val_dtypes = [] + + for val in prop.values: + curr_dtype = "string" + + for check_dtype in dtype_checks.items(): + if bool(re.compile(check_dtype[1]).match(val.strip())): + if check_dtype[0] == "tuple" and val.count(';') > 0: + curr_dtype = str(val.count(';') + 1) + "-" + check_dtype[0] + else: + curr_dtype = check_dtype[0] + break + if check_dtype[0] == "text" and len(re.findall(check_dtype[1], val.strip())) > 0: + curr_dtype = check_dtype[0] + break + + val_dtypes += [curr_dtype] + + res_dtype = max(set(val_dtypes), key=val_dtypes.count) + + if res_dtype != "string": + msg = 'Dtype of property "%s" currently is "string", but might fit dtype "%s"!' % (prop.name, res_dtype) + yield ValidationError(prop, msg, LABEL_WARNING) + + +Validation.register_handler('property', property_values_string_check) From be4663c34e98550e28a69f24b1796828669b5039 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Thu, 19 Mar 2020 11:41:18 +0100 Subject: [PATCH 02/24] [test_validation.py] Add Test for Property String Dtype Validation --- test/test_validation.py | 44 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/test_validation.py b/test/test_validation.py index d020cd26..f53d9b2c 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -152,3 +152,47 @@ def test_standalone_property(self): for err in validate(prop).errors: assert not err.is_error + + def test_prop_string_values(self): + + """ + Test if property values set as dtype string but could be of different dtype raise validation warning. + """ + + prop1 = odml.Property(name='members', dtype="string", values=['-13', '101', '-11', '0', '-8']) + self.assertError(validate(prop1), + 'Dtype of property "members" currently is "string", but might fit dtype "int"!') + + prop2 = odml.Property(name='potential', dtype="string", values=['-4.8', '10.0', '-11.9', '-10.0', '18.0']) + self.assertError(validate(prop2), + 'Dtype of property "potential" currently is "string", but might fit dtype "float"!') + + prop3 = odml.Property(name='dates', dtype="string", values=['1997-12-14', '00-12-14', '8']) + self.assertError(validate(prop3), + 'Dtype of property "dates" currently is "string", but might fit dtype "date"!') + + prop4 = odml.Property(name='datetimes', dtype="string", + values=['97-12-14 11:11:11', '97-12-14 12:12', '1997-12-14 03:03']) + self.assertError(validate(prop4), + 'Dtype of property "datetimes" currently is "string", but might fit dtype "datetime"!') + + prop5 = odml.Property(name='times', dtype="string", values=['11:11:11', '12:12:12', '03:03:03', '8']) + self.assertError(validate(prop5), + 'Dtype of property "times" currently is "string", but might fit dtype "time"!') + + prop6 = odml.Property(name='sent', dtype="string", values=['False', True, 'TRUE', '0', 't', '12', 'Ft']) + self.assertError(validate(prop6), + 'Dtype of property "sent" currently is "string", but might fit dtype "boolean"!') + + prop7 = odml.Property(name='texts', dtype="string", values=['line1\n line2', 'line3\n line4', '\nline5\n']) + self.assertError(validate(prop7), + 'Dtype of property "texts" currently is "string", but might fit dtype "text"!') + + prop8 = odml.Property(name="Location", dtype='string', values=['(39.12; 67.19)', '(39.12; 67.19)', '(39.12)']) + self.assertError(validate(prop8), + 'Dtype of property "Location" currently is "string", but might fit dtype "2-tuple"!') + + prop9 = odml.Property(name="Coos", dtype='string', + values=['(39.12; 89; 67.19)', '(39.12; 78; 67.19)', '(39.12)']) + self.assertError(validate(prop9), + 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') From 2ec7e5458090eb6e9f91c2827d10c2b02931043a Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Thu, 19 Mar 2020 11:42:46 +0100 Subject: [PATCH 03/24] [test/resources] Add integration.xml for odML Validation Test --- test/resources/integration.xml | 117 +++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 test/resources/integration.xml diff --git a/test/resources/integration.xml b/test/resources/integration.xml new file mode 100644 index 00000000..05761e92 --- /dev/null +++ b/test/resources/integration.xml @@ -0,0 +1,117 @@ + + + + e178c43c-8430-4a3e-bc0d-12e2b844b218 +
+ 2fe02204-29eb-4358-884a-48c91755539e + sec_test_1 +
+
+ 39ec5df0-f01d-4385-8865-1550a6b59c36 + + sec_test_2 +
+
+ 27ca5a42-fc00-4342-af67-45d014c92d90 + test + sec1 + + b237006e-a584-469d-adbc-e30dae9c4678 + members + [-13,101,-11,0,-8] + string + +
+
+ c37f6d71-923e-4223-b67a-373c1bc936aa + test + sec2 + + 10676e90-ea2a-4f7c-8c74-0a618a4d073c + potential + [-4.8,10.0,-11.9,-10.0,18.0] + string + +
+
+ 69c54bbf-2697-4baf-970a-4113f020b983 + test + sec3 + + 00cf0cae-23ff-44cf-84f3-2bbe98493a2a + dates + [1997-12-14,00-12-14,8] + string + +
+
+ f2258147-c6d8-43ee-b764-ee51ef38074a + test + sec4 + + 417b9d0e-285a-44b9-96b4-e4e0f9b13d80 + datetimes + [97-12-14 11:11:11,97-12-14 12:12,1997-12-14 03:03] + string + +
+
+ 03b6ccbc-7e4e-4f8e-99d1-84f0577f3c55 + test + sec5 + + f3fc344d-463b-4a7b-b4fb-a20d6c1edc49 + times + [11:11:11,12:12:12,03:03:03,8] + string + +
+
+ a2aa0243-1db5-447b-a66b-f30b5c56e105 + test + sec6 + + e62ab8f2-c70d-4582-b1ea-ad92cb1bf0a6 + sent + [False,True,TRUE,0,t,12,Ft] + string + +
+
+ 58149830-e716-49a4-8fe0-069277fbf968 + test + sec7 + + b41e04d1-b56d-4909-b7ac-a7e383c39fc1 + texts + ["lineA + lineB","lineC + lineD"," + lineE + "] + string + +
+
+ 68dba805-6042-4107-a12e-be8944cef9b1 + test + sec8 + + 363cab7a-6271-40d7-86c8-dc59c4c178c5 + Location + [(39.12; 67.19),(39.12; 67.19),(39.12)] + string + +
+
+ 41d19531-e4df-48bc-908d-0d89106e5d2e + test + sec9 + + 9ed75191-92fe-46b8-b187-7ae35987b9b0 + Coos + [(39.12; 89; 67.19),(39.12; 78; 67.19),(39.12)] + string + +
+
From 53eba68af80519614472da356f3c02ccfdb55390 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Thu, 19 Mar 2020 11:45:01 +0100 Subject: [PATCH 04/24] [test_validation.py] Add Test for XML File Validation --- test/test_validation.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/test_validation.py b/test/test_validation.py index f53d9b2c..a5b064fc 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -196,3 +196,31 @@ def test_prop_string_values(self): values=['(39.12; 89; 67.19)', '(39.12; 78; 67.19)', '(39.12)']) self.assertError(validate(prop9), 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') + + def test_load_xml(self): + + """ + Test if loading xml document raises validation errors. Errors are: + + - Undefined Section type + - Properties with undefined dtypes + """ + + doc = odml.load("./resources/integration.xml") + + self.assertError(validate(doc), "Section type undefined") + + self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') + self.assertError(validate(doc), + 'Dtype of property "potential" currently is "string", but might fit dtype "float"!') + self.assertError(validate(doc), 'Dtype of property "dates" currently is "string", but might fit dtype "date"!') + self.assertError(validate(doc), + 'Dtype of property "datetimes" currently is "string", but might fit dtype "datetime"!') + self.assertError(validate(doc), 'Dtype of property "times" currently is "string", but might fit dtype "time"!') + self.assertError(validate(doc), + 'Dtype of property "sent" currently is "string", but might fit dtype "boolean"!') + self.assertError(validate(doc), 'Dtype of property "texts" currently is "string", but might fit dtype "text"!') + self.assertError(validate(doc), + 'Dtype of property "Location" currently is "string", but might fit dtype "2-tuple"!') + self.assertError(validate(doc), + 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') From d062f5268c1d3e5e8d55f852dbcdfddd2238f155 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Thu, 19 Mar 2020 11:45:47 +0100 Subject: [PATCH 05/24] [test/resources] Add integration.json for odML Validation Test --- test/resources/integration.json | 191 ++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 test/resources/integration.json diff --git a/test/resources/integration.json b/test/resources/integration.json new file mode 100644 index 00000000..b9ce1091 --- /dev/null +++ b/test/resources/integration.json @@ -0,0 +1,191 @@ +{ + "Document": { + "id": "b2e4f609-d38c-4796-9042-2edce98619f3", + "sections": [ + { + "id": "b40e0cc4-adac-483c-8556-792799e1186e", + "name": "sec_test_1", + "sections": [], + "properties": [] + }, + { + "id": "37b7506e-c859-412d-abda-8e636a152f22", + "type": "", + "name": "sec_test_2", + "sections": [], + "properties": [] + }, + { + "id": "c72ed817-dc62-40cf-9d05-3ed230821024", + "type": "test", + "name": "sec1", + "sections": [], + "properties": [ + { + "id": "1fa95ef7-16cc-4b24-aa98-47d5c3eff13b", + "name": "members", + "value": [ + "-13", + "101", + "-11", + "0", + "-8" + ] + } + ] + }, + { + "id": "f2273f99-b55e-46b5-abd0-4ea918e75801", + "type": "test", + "name": "sec2", + "sections": [], + "properties": [ + { + "id": "32002ac0-12b3-45dd-a0b9-07bb828f8d9e", + "name": "potential", + "value": [ + "-4.8", + "10.0", + "-11.9", + "-10.0", + "18.0" + ], + "type": "string" + } + ] + }, + { + "id": "e82249de-7cfc-41c3-b7e0-f7d4f1e1b367", + "type": "test", + "name": "sec3", + "sections": [], + "properties": [ + { + "id": "df4cdfc0-dea7-443c-ac64-828b000edba7", + "name": "dates", + "value": [ + "1997-12-14", + "00-12-14", + "8" + ], + "type": "string" + } + ] + }, + { + "id": "84ace3cc-b6c7-44d3-bc15-baa8fc3f2c1c", + "type": "test", + "name": "sec4", + "sections": [], + "properties": [ + { + "id": "a373aa89-2f23-4643-aadb-7d073c5531c2", + "name": "datetimes", + "value": [ + "97-12-14 11:11:11", + "97-12-14 12:12", + "1997-12-14 03:03" + ], + "type": "string" + } + ] + }, + { + "id": "5d5968a1-51da-4ccc-926f-0e306a0c0dd4", + "type": "test", + "name": "sec5", + "sections": [], + "properties": [ + { + "id": "830f1310-3640-42da-a78d-49e4bac8fac9", + "name": "times", + "value": [ + "11:11:11", + "12:12:12", + "03:03:03", + "8" + ], + "type": "string" + } + ] + }, + { + "id": "dc4479e2-d757-4dbb-9661-017e5a6d11e9", + "type": "test", + "name": "sec6", + "sections": [], + "properties": [ + { + "id": "febd2862-c01d-4411-8c55-4abd74f3e070", + "name": "sent", + "value": [ + "False", + "True", + "TRUE", + "0", + "t", + "12", + "Ft" + ], + "type": "string" + } + ] + }, + { + "id": "fc69bd30-70ee-44dc-b9e9-fefd101c32f2", + "type": "test", + "name": "sec7", + "sections": [], + "properties": [ + { + "id": "6014f2db-b919-408c-96c8-bee79f1d96b5", + "name": "texts", + "value": [ + "line1\n line2", + "line3\n line4", + "\nline5\n" + ], + "type": "string" + } + ] + }, + { + "id": "feb72b53-3cf6-4a3f-93d5-f3a33f5e97ab", + "type": "test", + "name": "sec8", + "sections": [], + "properties": [ + { + "id": "2f17f6d2-4733-4c61-bbbd-e553659e2fef", + "name": "Location", + "value": [ + "(39.12; 67.19)", + "(39.12; 67.19)", + "(39.12)" + ], + "type": "string" + } + ] + }, + { + "id": "80256f2f-b3f7-4cba-8a70-8745dc96bd5e", + "type": "test", + "name": "sec9", + "sections": [], + "properties": [ + { + "id": "5c0df900-ecf1-4416-9816-df9abd7c3ee5", + "name": "Coos", + "value": [ + "(39.12; 89; 67.19)", + "(39.12; 78; 67.19)", + "(39.12)" + ], + "type": "string" + } + ] + } + ] + }, + "odml-version": "1.1" +} \ No newline at end of file From 3e5aaa752bfc5adf8a4174de932ea166cc908da7 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Thu, 19 Mar 2020 11:46:25 +0100 Subject: [PATCH 06/24] [test_validation.py] Add Test for JSON File Validation --- test/test_validation.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/test_validation.py b/test/test_validation.py index a5b064fc..dc5d7d2a 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -224,3 +224,31 @@ def test_load_xml(self): 'Dtype of property "Location" currently is "string", but might fit dtype "2-tuple"!') self.assertError(validate(doc), 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') + + def test_load_json(self): + + """ + Test if loading json document raises validation errors. Errors are: + + - Undefined Section type + - Properties with undefined dtypes + """ + + doc = odml.load("./resources/integration.json", "JSON") + + self.assertError(validate(doc), "Section type undefined") + + self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') + self.assertError(validate(doc), + 'Dtype of property "potential" currently is "string", but might fit dtype "float"!') + self.assertError(validate(doc), 'Dtype of property "dates" currently is "string", but might fit dtype "date"!') + self.assertError(validate(doc), + 'Dtype of property "datetimes" currently is "string", but might fit dtype "datetime"!') + self.assertError(validate(doc), 'Dtype of property "times" currently is "string", but might fit dtype "time"!') + self.assertError(validate(doc), + 'Dtype of property "sent" currently is "string", but might fit dtype "boolean"!') + self.assertError(validate(doc), 'Dtype of property "texts" currently is "string", but might fit dtype "text"!') + self.assertError(validate(doc), + 'Dtype of property "Location" currently is "string", but might fit dtype "2-tuple"!') + self.assertError(validate(doc), + 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') From 43b4941835ba56c1893953d9d445d631c2fe0cb9 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Thu, 19 Mar 2020 11:47:03 +0100 Subject: [PATCH 07/24] [test/resources] Add integration.yaml for odML Validation Test --- test/resources/integration.yaml | 133 ++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 test/resources/integration.yaml diff --git a/test/resources/integration.yaml b/test/resources/integration.yaml new file mode 100644 index 00000000..66c59bd3 --- /dev/null +++ b/test/resources/integration.yaml @@ -0,0 +1,133 @@ +Document: + id: b4be7fe5-0a1c-4499-8cf5-b34ef73689e6 + sections: + - id: 859e4f67-d59d-477a-8c12-248e30d8da35 + name: sec_test_1 + properties: [] + sections: [] + - id: 8b7785dd-cc83-41ab-ae0a-4f10d362a008 + name: sec_test_2 + properties: [] + sections: [] + type: "" + - id: 42721936-f62e-4e40-9152-2fb4c6857e92 + name: sec1 + properties: + - id: 96f2f6e4-c9f2-426f-a87e-1af2e75f8a27 + name: members + type: string + value: + - '-13' + - '101' + - '-11' + - '0' + - '-8' + sections: [] + type: test + - id: afdcef87-82e9-4cbd-a9ff-1cac79928fac + name: sec2 + properties: + - id: 685d4de7-47ba-4ffb-8eb1-b948e9204e2e + name: potential + value: + - '-4.8' + - '10.0' + - '-11.9' + - '-10.0' + - '18.0' + sections: [] + type: test + - id: 182569c1-cf26-4e14-9159-68ddc97c0188 + name: sec3 + properties: + - id: 777b23ac-4639-49a5-96f4-037390a344d6 + name: dates + type: string + value: + - '1997-12-14' + - 00-12-14 + - '8' + sections: [] + type: test + - id: 3964fad7-12ff-459a-86e2-100f5c986fa6 + name: sec4 + properties: + - id: 1be11c05-4c5f-4bea-8409-c36bf1fbc0d2 + name: datetimes + type: string + value: + - 97-12-14 11:11:11 + - 97-12-14 12:12 + - 1997-12-14 03:03 + sections: [] + type: test + - id: 4f4de2c9-7b2d-4210-a280-f7daa823c3b3 + name: sec5 + properties: + - id: 8404bbb8-6b9f-4a5a-8396-033df7ad6efc + name: times + type: string + value: + - '11:11:11' + - '12:12:12' + - 03:03:03 + - '8' + sections: [] + type: test + - id: c689f7d7-3a84-4747-a8fc-c117d15d781e + name: sec6 + properties: + - id: b0e97058-aedf-44cb-927f-172d5971ebbb + name: sent + type: string + value: + - 'False' + - 'True' + - 'TRUE' + - '0' + - t + - '12' + - Ft + sections: [] + type: test + - id: 10f8eed7-b36a-475a-a392-fd0aa1273a9a + name: sec7 + properties: + - id: 99046f03-71ad-4512-a4be-b6241a10186a + name: texts + type: string + value: + - "line1\n line2" + - "line3\n line4" + - ' + + line5 + + ' + sections: [] + type: test + - id: 5001ca98-0c97-4ec7-b194-048055d79394 + name: sec8 + properties: + - id: 98772d32-d917-46e8-8fc5-728d72db1480 + name: Location + type: string + value: + - (39.12; 67.19) + - (39.12; 67.19) + - (39.12) + sections: [] + type: test + - id: 55c52ce7-1de0-4b1c-a0d6-f63af497a5c9 + name: sec9 + properties: + - id: 01dfd04b-3c61-48c4-b45b-d754d1645f8b + name: Coos + type: string + value: + - (39.12; 89; 67.19) + - (39.12; 78; 67.19) + - (39.12) + sections: [] + type: test +odml-version: '1.1' From 30892b4f373114c879fa4aaaf0c3a6a5e2eee19f Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Thu, 19 Mar 2020 11:47:27 +0100 Subject: [PATCH 08/24] [test_validation.py] Add Test for YAML File Validation --- test/test_validation.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/test_validation.py b/test/test_validation.py index dc5d7d2a..07bdb281 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -252,3 +252,31 @@ def test_load_json(self): 'Dtype of property "Location" currently is "string", but might fit dtype "2-tuple"!') self.assertError(validate(doc), 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') + + def test_load_yaml(self): + + """ + Test if loading yaml document raises validation errors. Errors are: + + - Undefined Section type + - Properties with undefined dtypes + """ + + doc = odml.load("./resources/integration.yaml", "YAML") + + self.assertError(validate(doc), "Section type undefined") + + self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') + self.assertError(validate(doc), + 'Dtype of property "potential" currently is "string", but might fit dtype "float"!') + self.assertError(validate(doc), 'Dtype of property "dates" currently is "string", but might fit dtype "date"!') + self.assertError(validate(doc), + 'Dtype of property "datetimes" currently is "string", but might fit dtype "datetime"!') + self.assertError(validate(doc), 'Dtype of property "times" currently is "string", but might fit dtype "time"!') + self.assertError(validate(doc), + 'Dtype of property "sent" currently is "string", but might fit dtype "boolean"!') + self.assertError(validate(doc), 'Dtype of property "texts" currently is "string", but might fit dtype "text"!') + self.assertError(validate(doc), + 'Dtype of property "Location" currently is "string", but might fit dtype "2-tuple"!') + self.assertError(validate(doc), + 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') From f24eed3d1e59a92f6efaf66c24525d487523b534 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Thu, 19 Mar 2020 12:03:48 +0100 Subject: [PATCH 09/24] [test_validation.py] Specify Section Type Test for File Loading Tests --- test/test_validation.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/test/test_validation.py b/test/test_validation.py index 07bdb281..361464ea 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -208,7 +208,17 @@ def test_load_xml(self): doc = odml.load("./resources/integration.xml") - self.assertError(validate(doc), "Section type undefined") + sec_test_1_err = False + sec_test_2_err = False + + for err in validate(doc).errors: + if err.msg == "Section type undefined" and err.obj.name == "sec_test_1": + sec_test_1_err = True + elif err.msg == "Section type undefined" and err.obj.name == "sec_test_2": + sec_test_2_err = True + + assert sec_test_1_err + assert sec_test_2_err self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') self.assertError(validate(doc), @@ -236,7 +246,17 @@ def test_load_json(self): doc = odml.load("./resources/integration.json", "JSON") - self.assertError(validate(doc), "Section type undefined") + sec_test_1_err = False + sec_test_2_err = False + + for err in validate(doc).errors: + if err.msg == "Section type undefined" and err.obj.name == "sec_test_1": + sec_test_1_err = True + elif err.msg == "Section type undefined" and err.obj.name == "sec_test_2": + sec_test_2_err = True + + assert sec_test_1_err + assert sec_test_2_err self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') self.assertError(validate(doc), @@ -264,7 +284,17 @@ def test_load_yaml(self): doc = odml.load("./resources/integration.yaml", "YAML") - self.assertError(validate(doc), "Section type undefined") + sec_test_1_err = False + sec_test_2_err = False + + for err in validate(doc).errors: + if err.msg == "Section type undefined" and err.obj.name == "sec_test_1": + sec_test_1_err = True + elif err.msg == "Section type undefined" and err.obj.name == "sec_test_2": + sec_test_2_err = True + + assert sec_test_1_err + assert sec_test_2_err self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') self.assertError(validate(doc), From b5d8d0993bb82b46c6dae21ba4bd4424e8fbc4e3 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Thu, 19 Mar 2020 12:36:35 +0100 Subject: [PATCH 10/24] [test_validation.py] Add Test for Property String Dtype Validation --- test/test_validation.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/test_validation.py b/test/test_validation.py index 361464ea..e8bca6cf 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -1,5 +1,6 @@ import unittest import odml +import os import odml.validation import odml.terminology from . import test_samplefile as samplefile @@ -206,7 +207,9 @@ def test_load_xml(self): - Properties with undefined dtypes """ - doc = odml.load("./resources/integration.xml") + dir_path = os.path.dirname(os.path.realpath(__file__)) + path = os.path.join(dir_path, "resources", "integration.xml") + doc = odml.load(path) sec_test_1_err = False sec_test_2_err = False @@ -244,7 +247,9 @@ def test_load_json(self): - Properties with undefined dtypes """ - doc = odml.load("./resources/integration.json", "JSON") + dir_path = os.path.dirname(os.path.realpath(__file__)) + path = os.path.join(dir_path, "resources", "integration.json") + doc = odml.load(path, "JSON") sec_test_1_err = False sec_test_2_err = False @@ -282,7 +287,9 @@ def test_load_yaml(self): - Properties with undefined dtypes """ - doc = odml.load("./resources/integration.yaml", "YAML") + dir_path = os.path.dirname(os.path.realpath(__file__)) + path = os.path.join(dir_path, "resources", "integration.yaml") + doc = odml.load(path, "YAML") sec_test_1_err = False sec_test_2_err = False From e22f7257822975fad3cb40285628e9915357e559 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:11:09 +0100 Subject: [PATCH 11/24] [validation.py] Throw String Dtype Warning Only If All Values Match Dtype --- odml/validation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/odml/validation.py b/odml/validation.py index 73ca6a64..7ea1492b 100644 --- a/odml/validation.py +++ b/odml/validation.py @@ -431,6 +431,9 @@ def property_values_string_check(prop): res_dtype = max(set(val_dtypes), key=val_dtypes.count) + if len(set(val_dtypes)) > 1: + res_dtype = "string" + if res_dtype != "string": msg = 'Dtype of property "%s" currently is "string", but might fit dtype "%s"!' % (prop.name, res_dtype) yield ValidationError(prop, msg, LABEL_WARNING) From f912971111a8f26859668be93f67247fc0f9a79d Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:12:17 +0100 Subject: [PATCH 12/24] [test_validation.py] Move dir_path to setUp --- test/test_validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_validation.py b/test/test_validation.py index e8bca6cf..efc4d494 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -13,6 +13,7 @@ class TestValidation(unittest.TestCase): def setUp(self): self.doc = samplefile.SampleFileCreator().create_document() self.maxDiff = None + self.dir_path = os.path.dirname(os.path.realpath(__file__)) def filter_repository_errors(self, errors): return filter(lambda x: "A section should have an associated " From dde673099afc484141258db94e1c6fab36150c70 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:16:33 +0100 Subject: [PATCH 13/24] [test_validation.py] Adapt Test for String Dtype Validation Behavior Show warning only if all values fit dtype. --- test/test_validation.py | 74 +++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/test/test_validation.py b/test/test_validation.py index efc4d494..7d84519f 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -156,48 +156,56 @@ def test_standalone_property(self): assert not err.is_error def test_prop_string_values(self): - """ - Test if property values set as dtype string but could be of different dtype raise validation warning. + Test if property values set as dtype string but could be of different dtype + raise validation warning. """ - prop1 = odml.Property(name='members', dtype="string", values=['-13', '101', '-11', '0', '-8']) - self.assertError(validate(prop1), - 'Dtype of property "members" currently is "string", but might fit dtype "int"!') + prop1 = odml.Property(name='members', dtype="string", + values=['-13', '101', '-11', '0', '-8']) + self.assertError(validate(prop1), 'Dtype of property "members" currently is "string",' + ' but might fit dtype "int"!') - prop2 = odml.Property(name='potential', dtype="string", values=['-4.8', '10.0', '-11.9', '-10.0', '18.0']) - self.assertError(validate(prop2), - 'Dtype of property "potential" currently is "string", but might fit dtype "float"!') + prop2 = odml.Property(name='potential', dtype="string", + values=['-4.8', '10.0', '-11.9', '-10.0', '18.0']) + self.assertError(validate(prop2),'Dtype of property "potential" currently is "string", ' + 'but might fit dtype "float"!') - prop3 = odml.Property(name='dates', dtype="string", values=['1997-12-14', '00-12-14', '8']) - self.assertError(validate(prop3), - 'Dtype of property "dates" currently is "string", but might fit dtype "date"!') + prop3 = odml.Property(name='dates', dtype="string", + values=['1997-12-14', '00-12-14', '89-07-04']) + self.assertError(validate(prop3), 'Dtype of property "dates" currently is "string", ' + 'but might fit dtype "date"!') prop4 = odml.Property(name='datetimes', dtype="string", - values=['97-12-14 11:11:11', '97-12-14 12:12', '1997-12-14 03:03']) - self.assertError(validate(prop4), - 'Dtype of property "datetimes" currently is "string", but might fit dtype "datetime"!') - - prop5 = odml.Property(name='times', dtype="string", values=['11:11:11', '12:12:12', '03:03:03', '8']) - self.assertError(validate(prop5), - 'Dtype of property "times" currently is "string", but might fit dtype "time"!') - - prop6 = odml.Property(name='sent', dtype="string", values=['False', True, 'TRUE', '0', 't', '12', 'Ft']) - self.assertError(validate(prop6), - 'Dtype of property "sent" currently is "string", but might fit dtype "boolean"!') - - prop7 = odml.Property(name='texts', dtype="string", values=['line1\n line2', 'line3\n line4', '\nline5\n']) - self.assertError(validate(prop7), - 'Dtype of property "texts" currently is "string", but might fit dtype "text"!') - - prop8 = odml.Property(name="Location", dtype='string', values=['(39.12; 67.19)', '(39.12; 67.19)', '(39.12)']) - self.assertError(validate(prop8), - 'Dtype of property "Location" currently is "string", but might fit dtype "2-tuple"!') + values=['97-12-14 11:11:11', '97-12-14 12:12', '1997-12-14 03:03']) + self.assertError(validate(prop4), 'Dtype of property "datetimes" currently is "string", ' + 'but might fit dtype "datetime"!') + + prop5 = odml.Property(name='times', dtype="string", + values=['11:11:11', '12:12:12', '03:03:03']) + self.assertError(validate(prop5), 'Dtype of property "times" currently is "string", ' + 'but might fit dtype "time"!') + + prop6 = odml.Property(name='sent', dtype="string", + values=['False', True, 'TRUE', False, 't']) + self.assertError(validate(prop6), 'Dtype of property "sent" currently is "string", ' + 'but might fit dtype "boolean"!') + + prop7 = odml.Property(name='texts', dtype="string", + values=['line1\n line2', 'line3\n line4', '\nline5\nline6']) + self.assertError(validate(prop7), 'Dtype of property "texts" currently is "string", ' + 'but might fit dtype "text"!') + + prop8 = odml.Property(name="Location", dtype='string', + values=['(39.12; 67.19)', '(39.12; 67.19)', '(39.12; 67.18)']) + self.assertError(validate(prop8), 'Dtype of property "Location" currently is "string", ' + 'but might fit dtype "2-tuple"!') prop9 = odml.Property(name="Coos", dtype='string', - values=['(39.12; 89; 67.19)', '(39.12; 78; 67.19)', '(39.12)']) - self.assertError(validate(prop9), - 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') + values=['(39.12; 89; 67.19)', '(39.12; 78; 67.19)', + '(39.12; 56; 67.18)']) + self.assertError(validate(prop9), 'Dtype of property "Coos" currently is "string", ' + 'but might fit dtype "3-tuple"!') def test_load_xml(self): From 0d41ae8a8d8fbcf9f69850d2bf14563d104c7f9e Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:20:54 +0100 Subject: [PATCH 14/24] [test_validation.py] Add Property with Dtype String for Property String Values Test --- test/test_validation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_validation.py b/test/test_validation.py index 7d84519f..d52cffdf 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -161,6 +161,10 @@ def test_prop_string_values(self): raise validation warning. """ + prop0 = odml.Property(name='words', dtype="string", + values=['-13', '101', '-11', 'hello']) + assert len(validate(prop0).errors) == 0 + prop1 = odml.Property(name='members', dtype="string", values=['-13', '101', '-11', '0', '-8']) self.assertError(validate(prop1), 'Dtype of property "members" currently is "string",' From 4160a8b5113e69daa939431afd60dcd1278bcca3 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:23:43 +0100 Subject: [PATCH 15/24] [test/resources] Replace File for XML Validation Tests Remove integration.xml Add validation_dtypes.xml Add validation_section.xml Split test functionality. --- test/resources/integration.xml | 117 ------------- test/resources/validation_dtypes.xml | 230 ++++++++++++++++++++++++++ test/resources/validation_section.xml | 14 ++ 3 files changed, 244 insertions(+), 117 deletions(-) delete mode 100644 test/resources/integration.xml create mode 100644 test/resources/validation_dtypes.xml create mode 100644 test/resources/validation_section.xml diff --git a/test/resources/integration.xml b/test/resources/integration.xml deleted file mode 100644 index 05761e92..00000000 --- a/test/resources/integration.xml +++ /dev/null @@ -1,117 +0,0 @@ - - - - e178c43c-8430-4a3e-bc0d-12e2b844b218 -
- 2fe02204-29eb-4358-884a-48c91755539e - sec_test_1 -
-
- 39ec5df0-f01d-4385-8865-1550a6b59c36 - - sec_test_2 -
-
- 27ca5a42-fc00-4342-af67-45d014c92d90 - test - sec1 - - b237006e-a584-469d-adbc-e30dae9c4678 - members - [-13,101,-11,0,-8] - string - -
-
- c37f6d71-923e-4223-b67a-373c1bc936aa - test - sec2 - - 10676e90-ea2a-4f7c-8c74-0a618a4d073c - potential - [-4.8,10.0,-11.9,-10.0,18.0] - string - -
-
- 69c54bbf-2697-4baf-970a-4113f020b983 - test - sec3 - - 00cf0cae-23ff-44cf-84f3-2bbe98493a2a - dates - [1997-12-14,00-12-14,8] - string - -
-
- f2258147-c6d8-43ee-b764-ee51ef38074a - test - sec4 - - 417b9d0e-285a-44b9-96b4-e4e0f9b13d80 - datetimes - [97-12-14 11:11:11,97-12-14 12:12,1997-12-14 03:03] - string - -
-
- 03b6ccbc-7e4e-4f8e-99d1-84f0577f3c55 - test - sec5 - - f3fc344d-463b-4a7b-b4fb-a20d6c1edc49 - times - [11:11:11,12:12:12,03:03:03,8] - string - -
-
- a2aa0243-1db5-447b-a66b-f30b5c56e105 - test - sec6 - - e62ab8f2-c70d-4582-b1ea-ad92cb1bf0a6 - sent - [False,True,TRUE,0,t,12,Ft] - string - -
-
- 58149830-e716-49a4-8fe0-069277fbf968 - test - sec7 - - b41e04d1-b56d-4909-b7ac-a7e383c39fc1 - texts - ["lineA - lineB","lineC - lineD"," - lineE - "] - string - -
-
- 68dba805-6042-4107-a12e-be8944cef9b1 - test - sec8 - - 363cab7a-6271-40d7-86c8-dc59c4c178c5 - Location - [(39.12; 67.19),(39.12; 67.19),(39.12)] - string - -
-
- 41d19531-e4df-48bc-908d-0d89106e5d2e - test - sec9 - - 9ed75191-92fe-46b8-b187-7ae35987b9b0 - Coos - [(39.12; 89; 67.19),(39.12; 78; 67.19),(39.12)] - string - -
-
diff --git a/test/resources/validation_dtypes.xml b/test/resources/validation_dtypes.xml new file mode 100644 index 00000000..b9c9a7f0 --- /dev/null +++ b/test/resources/validation_dtypes.xml @@ -0,0 +1,230 @@ + + + + bfc85613-0e25-44e4-a85a-386b32c3c495 +
+ e72e64cd-e6eb-4399-b679-140d01f87365 + no_dtypes + no_dtypes +
+ a028642b-bebf-4392-9575-8cd751c3cdea + sec_string + + 313d13ce-8a46-44de-a333-71c19f54752c + words_no + [hello,-world,3,True] + +
+
+ 2deb48f3-f23b-427c-99c2-73345d5c34bb + int + sec_int + + 5b37aa19-76ef-48dd-bbd5-193a47213f36 + members_no + [-13,101,-11,0,-8] + +
+
+ 4965ce85-5da7-4411-9e66-6af7066fbc46 + float + sec_float + + 48c06a25-a10a-489a-990a-f592f0d87615 + potential_no + [-4.8,10.0,-11.9,-10.0,18.0] + +
+
+ 3d68d344-1fc2-43a5-9561-9fdc52b82fb2 + date + sec_date + + d0780269-20b1-4bc1-a023-4b502ffe63be + dates_no + [1997-12-14,00-12-14,89-07-04] + +
+
+ 9176ec80-7aa2-4a83-acb3-07e9efe7cf66 + datetime + sec_datetime + + 7ad1974a-3102-4e98-b1db-e36143bb8424 + datetimes_no + [97-12-14 11:11:11,97-12-14 12:12,1997-12-14 03:03] + +
+
+ 96756686-8652-46f3-9f7e-23137872fc72 + time + sec_time + + 0201b643-a871-4b8b-9e3a-820f43b81f5a + times_no + [11:11:11,12:12:12,03:03:03] + +
+
+ c7a6950f-dbf0-4478-afc3-f5a045f85914 + boolean + sec_boolean + + c550e679-4d5a-42c8-bbac-41adb9553e8c + sent_no + [False,True,TRUE,False,t] + +
+
+ dea66a08-82ab-448b-be6a-7ec5927316fa + text + sec_text + + 9e008df4-29d1-4def-b3c7-ee054ae6cf25 + texts_no + ["lineA + lineB","lineC + lineD","lineE + lineF"] + +
+
+ d5269b4d-5ef9-4e35-986a-9780fa96a5d6 + 2-tuple + sec_2_tuple + + ba9976ca-ce85-417b-86a7-18a19122fdc5 + Location_no + [(39.12; 67.19),(39.12; 67.19),(39.12; 67.18)] + +
+
+ 67d066e7-ee4c-4ffb-ab8c-4539e6a97fbc + 3-tuple + sec_3_tuple + + df72cfbb-22fc-4470-bd4c-bf9d1c98b170 + Coos_no + [(39.12; 89; 67.19),(39.12; 78; 67.19),(39.12; 56; 67.18)] + +
+
+
+ 2fc36b6f-cf3d-4aef-ae4e-0f56d081c515 + mislabelled_dtypes + mislabelled_dtypes +
+ 7a7ee997-365d-4b3f-9edd-e7580a2bd76e + string + sec_string + + 16037a96-072b-431c-ae7b-2d3312eb8f42 + words_mislabelled + [hello,-world,3,True] + string + +
+
+ 65fe690a-9a0b-4557-8271-e5f10956b4e0 + int + sec_int + + 0c78a624-f60c-4555-8440-2299aaead89d + members_mislabelled + [-13,101,-11,0,-8] + string + +
+
+ fa8b21aa-52d0-414e-b93c-c2197f3020cf + float + sec_float + + 50af87b7-4797-40fa-850a-c8033ba0b85b + potential_mislabelled + [-4.8,10.0,-11.9,-10.0,18.0] + string + +
+
+ 6e56b7da-c779-499d-a4fe-92ed48750421 + date + sec_date + + 2aede904-71a3-4897-8614-1dcdd9ed0ddf + dates_mislabelled + [1997-12-14,00-12-14,89-07-04] + string + +
+
+ 7dec0155-657d-4cba-8c06-8be2ceaf4049 + datetime + sec_datetime + + dc50b6ac-0993-432e-b401-06d13ed9e720 + datetimes_mislabelled + [97-12-14 11:11:11,97-12-14 12:12,1997-12-14 03:03] + string + +
+
+ d0278536-a486-43c2-9708-ef16efc7440a + time + sec_time + + afda136e-70e1-40ee-abdd-0e80f56cf624 + times_mislabelled + [11:11:11,12:12:12,03:03:03] + string + +
+
+ aea047b0-abea-4c6c-9a8c-5b6089fde35c + boolean + sec_boolean + + d959337d-ec34-468e-a374-1b1213fa23d5 + sent_mislabelled + [False,True,TRUE,False,t] + string + +
+
+ 1b078e40-1e23-4a74-bf34-c948c46b8bba + text + sec_text + + 4b7430d6-ba74-4f66-88e5-8009bd289815 + texts_mislabelled + ["lineA + lineB","lineC + lineD","lineE + lineF"] + string + +
+
+ fcf33912-8635-4495-a41e-4f1f92e0dae0 + 2-tuple + sec_2_tuple + + d447d0b1-9586-43f5-9d5a-ae426cab43e9 + Location_mislabelled + [(39.12; 67.19),(39.12; 67.19),(39.12; 67.18)] + string + +
+
+ 23091dcb-3949-400e-8740-7cd26f997edf + 3-tuple + sec_3_tuple + + 66fc02ba-c908-4552-8b91-363348c39f28 + Coos_mislabelled + [(39.12; 89; 67.19),(39.12; 78; 67.19),(39.12; 56; 67.18)] + string + +
+
+
diff --git a/test/resources/validation_section.xml b/test/resources/validation_section.xml new file mode 100644 index 00000000..b6853446 --- /dev/null +++ b/test/resources/validation_section.xml @@ -0,0 +1,14 @@ + + + + 69585a6e-bdfd-4a49-b227-d4f00e42723e +
+ e50cfe1b-1c0b-4f47-b76b-5500508b661a + sec_type_undefined +
+
+ d2fd6a82-2568-4ebc-9bd4-93e386a980b7 + + sec_type_empty +
+
From 7ee2ef6b54379b4b09efb2e53184984e4c55a037 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:29:49 +0100 Subject: [PATCH 16/24] [test_validation.py] Add Test for XML File Loading Sections - Replace part of current XML File Loading Test - Incorporate new file in resources/validation_section.xml --- test/test_validation.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/test/test_validation.py b/test/test_validation.py index d52cffdf..7a9041e5 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -211,30 +211,25 @@ def test_prop_string_values(self): self.assertError(validate(prop9), 'Dtype of property "Coos" currently is "string", ' 'but might fit dtype "3-tuple"!') - def test_load_xml(self): - + def test_load_section_xml(self): """ - Test if loading xml document raises validation errors. Errors are: - - - Undefined Section type - - Properties with undefined dtypes + Test if loading xml document raises validation errors for Sections with undefined type. """ - dir_path = os.path.dirname(os.path.realpath(__file__)) - path = os.path.join(dir_path, "resources", "integration.xml") + path = os.path.join(self.dir_path, "resources", "validation_section.xml") doc = odml.load(path) - sec_test_1_err = False - sec_test_2_err = False + sec_type_undefined_err = False + sec_type_empty_err = False for err in validate(doc).errors: - if err.msg == "Section type undefined" and err.obj.name == "sec_test_1": - sec_test_1_err = True - elif err.msg == "Section type undefined" and err.obj.name == "sec_test_2": - sec_test_2_err = True + if err.msg == "Section type undefined" and err.obj.name == "sec_type_undefined": + sec_type_undefined_err = True + elif err.msg == "Section type undefined" and err.obj.name == "sec_type_empty": + sec_type_empty_err = True - assert sec_test_1_err - assert sec_test_2_err + assert sec_type_undefined_err + assert sec_type_empty_err self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') self.assertError(validate(doc), From 4b1939acedf2142294f71d2367f235d0d71b3074 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:31:36 +0100 Subject: [PATCH 17/24] [test_validation.py] Add Test for XML File Loading Property Dtypes - Replace current XML File Loading Test - Incorporate file at resources/validation_dtypes.xml --- test/test_validation.py | 76 ++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/test/test_validation.py b/test/test_validation.py index 7a9041e5..2780ef23 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -231,25 +231,67 @@ def test_load_section_xml(self): assert sec_type_undefined_err assert sec_type_empty_err - self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') - self.assertError(validate(doc), - 'Dtype of property "potential" currently is "string", but might fit dtype "float"!') - self.assertError(validate(doc), 'Dtype of property "dates" currently is "string", but might fit dtype "date"!') - self.assertError(validate(doc), - 'Dtype of property "datetimes" currently is "string", but might fit dtype "datetime"!') - self.assertError(validate(doc), 'Dtype of property "times" currently is "string", but might fit dtype "time"!') - self.assertError(validate(doc), - 'Dtype of property "sent" currently is "string", but might fit dtype "boolean"!') - self.assertError(validate(doc), 'Dtype of property "texts" currently is "string", but might fit dtype "text"!') - self.assertError(validate(doc), - 'Dtype of property "Location" currently is "string", but might fit dtype "2-tuple"!') - self.assertError(validate(doc), - 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') + def test_load_dtypes_xml(self): + """ + Test if loading xml document raises validation errors for Properties with undefined dtypes. + """ + + path = os.path.join(self.dir_path, "resources", "validation_dtypes.xml") + doc = odml.load(path) + + self.assertError(validate(doc), 'Dtype of property "members_no" currently is "string", ' + 'but might fit dtype "int"!') + + self.assertError(validate(doc), 'Dtype of property "potential_no" currently is "string", ' + 'but might fit dtype "float"!') + + self.assertError(validate(doc), 'Dtype of property "dates_no" currently is "string", ' + 'but might fit dtype "date"!') + + self.assertError(validate(doc), 'Dtype of property "datetimes_no" currently is "string", ' + 'but might fit dtype "datetime"!') + + self.assertError(validate(doc), 'Dtype of property "times_no" currently is "string", ' + 'but might fit dtype "time"!') + + self.assertError(validate(doc), 'Dtype of property "sent_no" currently is "string", ' + 'but might fit dtype "boolean"!') + + self.assertError(validate(doc), 'Dtype of property "Location_no" currently is "string", ' + 'but might fit dtype "2-tuple"!') + + self.assertError(validate(doc), 'Dtype of property "Coos_no" currently is "string", ' + 'but might fit dtype "3-tuple"!') + + self.assertError(validate(doc), 'Dtype of property "members_mislabelled" currently is ' + '"string", but might fit dtype "int"!') + + self.assertError(validate(doc), 'Dtype of property "potential_mislabelled" currently is ' + '"string", but might fit dtype "float"!') + + self.assertError(validate(doc), 'Dtype of property "dates_mislabelled" currently is ' + '"string", but might fit dtype "date"!') + + self.assertError(validate(doc), 'Dtype of property "datetimes_mislabelled" currently is ' + '"string", but might fit dtype "datetime"!') + + self.assertError(validate(doc), 'Dtype of property "times_mislabelled" currently is ' + '"string", but might fit dtype "time"!') + + self.assertError(validate(doc), 'Dtype of property "sent_mislabelled" currently is ' + '"string", but might fit dtype "boolean"!') + + self.assertError(validate(doc), 'Dtype of property "texts_mislabelled" currently is ' + '"string", but might fit dtype "text"!') + + self.assertError(validate(doc), 'Dtype of property "Location_mislabelled" currently is ' + '"string", but might fit dtype "2-tuple"!') + + self.assertError(validate(doc), 'Dtype of property "Coos_mislabelled" currently is ' + '"string", but might fit dtype "3-tuple"!') + - def test_load_json(self): - """ - Test if loading json document raises validation errors. Errors are: - Undefined Section type - Properties with undefined dtypes From 3c78947b6a5d53c45757678daac5c6f46d4cdfe9 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:32:49 +0100 Subject: [PATCH 18/24] [test/resources] Replace File for JSON Validation Tests Remove integration.json Add validation_dtypes.json Add validation_section.json Split test functionality. --- test/resources/integration.json | 191 ------------ test/resources/validation_dtypes.json | 397 +++++++++++++++++++++++++ test/resources/validation_section.json | 21 ++ 3 files changed, 418 insertions(+), 191 deletions(-) delete mode 100644 test/resources/integration.json create mode 100644 test/resources/validation_dtypes.json create mode 100644 test/resources/validation_section.json diff --git a/test/resources/integration.json b/test/resources/integration.json deleted file mode 100644 index b9ce1091..00000000 --- a/test/resources/integration.json +++ /dev/null @@ -1,191 +0,0 @@ -{ - "Document": { - "id": "b2e4f609-d38c-4796-9042-2edce98619f3", - "sections": [ - { - "id": "b40e0cc4-adac-483c-8556-792799e1186e", - "name": "sec_test_1", - "sections": [], - "properties": [] - }, - { - "id": "37b7506e-c859-412d-abda-8e636a152f22", - "type": "", - "name": "sec_test_2", - "sections": [], - "properties": [] - }, - { - "id": "c72ed817-dc62-40cf-9d05-3ed230821024", - "type": "test", - "name": "sec1", - "sections": [], - "properties": [ - { - "id": "1fa95ef7-16cc-4b24-aa98-47d5c3eff13b", - "name": "members", - "value": [ - "-13", - "101", - "-11", - "0", - "-8" - ] - } - ] - }, - { - "id": "f2273f99-b55e-46b5-abd0-4ea918e75801", - "type": "test", - "name": "sec2", - "sections": [], - "properties": [ - { - "id": "32002ac0-12b3-45dd-a0b9-07bb828f8d9e", - "name": "potential", - "value": [ - "-4.8", - "10.0", - "-11.9", - "-10.0", - "18.0" - ], - "type": "string" - } - ] - }, - { - "id": "e82249de-7cfc-41c3-b7e0-f7d4f1e1b367", - "type": "test", - "name": "sec3", - "sections": [], - "properties": [ - { - "id": "df4cdfc0-dea7-443c-ac64-828b000edba7", - "name": "dates", - "value": [ - "1997-12-14", - "00-12-14", - "8" - ], - "type": "string" - } - ] - }, - { - "id": "84ace3cc-b6c7-44d3-bc15-baa8fc3f2c1c", - "type": "test", - "name": "sec4", - "sections": [], - "properties": [ - { - "id": "a373aa89-2f23-4643-aadb-7d073c5531c2", - "name": "datetimes", - "value": [ - "97-12-14 11:11:11", - "97-12-14 12:12", - "1997-12-14 03:03" - ], - "type": "string" - } - ] - }, - { - "id": "5d5968a1-51da-4ccc-926f-0e306a0c0dd4", - "type": "test", - "name": "sec5", - "sections": [], - "properties": [ - { - "id": "830f1310-3640-42da-a78d-49e4bac8fac9", - "name": "times", - "value": [ - "11:11:11", - "12:12:12", - "03:03:03", - "8" - ], - "type": "string" - } - ] - }, - { - "id": "dc4479e2-d757-4dbb-9661-017e5a6d11e9", - "type": "test", - "name": "sec6", - "sections": [], - "properties": [ - { - "id": "febd2862-c01d-4411-8c55-4abd74f3e070", - "name": "sent", - "value": [ - "False", - "True", - "TRUE", - "0", - "t", - "12", - "Ft" - ], - "type": "string" - } - ] - }, - { - "id": "fc69bd30-70ee-44dc-b9e9-fefd101c32f2", - "type": "test", - "name": "sec7", - "sections": [], - "properties": [ - { - "id": "6014f2db-b919-408c-96c8-bee79f1d96b5", - "name": "texts", - "value": [ - "line1\n line2", - "line3\n line4", - "\nline5\n" - ], - "type": "string" - } - ] - }, - { - "id": "feb72b53-3cf6-4a3f-93d5-f3a33f5e97ab", - "type": "test", - "name": "sec8", - "sections": [], - "properties": [ - { - "id": "2f17f6d2-4733-4c61-bbbd-e553659e2fef", - "name": "Location", - "value": [ - "(39.12; 67.19)", - "(39.12; 67.19)", - "(39.12)" - ], - "type": "string" - } - ] - }, - { - "id": "80256f2f-b3f7-4cba-8a70-8745dc96bd5e", - "type": "test", - "name": "sec9", - "sections": [], - "properties": [ - { - "id": "5c0df900-ecf1-4416-9816-df9abd7c3ee5", - "name": "Coos", - "value": [ - "(39.12; 89; 67.19)", - "(39.12; 78; 67.19)", - "(39.12)" - ], - "type": "string" - } - ] - } - ] - }, - "odml-version": "1.1" -} \ No newline at end of file diff --git a/test/resources/validation_dtypes.json b/test/resources/validation_dtypes.json new file mode 100644 index 00000000..411b20bf --- /dev/null +++ b/test/resources/validation_dtypes.json @@ -0,0 +1,397 @@ +{ + "Document": { + "id": "2c9bddd2-0082-410c-89d4-cd6ea44a404b", + "sections": [ + { + "id": "f545ca92-1d4c-4d02-8170-1837b9e16395", + "type": "no_dtypes", + "name": "no_dtypes", + "sections": [ + { + "id": "24d75996-d19d-4138-8d79-8b90d7affbc2", + "name": "sec_string", + "sections": [], + "properties": [ + { + "id": "951db657-ba2d-43b9-9827-f83ef3ce8e64", + "name": "words_no", + "value": [ + "hello", + "-world", + "3", + "True" + ], + "type": "string" + } + ] + }, + { + "id": "2b3540fc-a422-4279-b10d-f87e16e0a2c3", + "type": "int", + "name": "sec_int", + "sections": [], + "properties": [ + { + "id": "55829af7-1058-4cd6-8454-de1d77143935", + "name": "members_no", + "value": [ + "-13", + "101", + "-11", + "0", + "-8" + ], + "type": "string" + } + ] + }, + { + "id": "725b997a-16af-436f-b6a4-bb7a16822958", + "type": "float", + "name": "sec_float", + "sections": [], + "properties": [ + { + "id": "d26b8eed-fd92-4d04-8434-f0846186ceee", + "name": "potential_no", + "value": [ + "-4.8", + "10.0", + "-11.9", + "-10.0", + "18.0" + ], + "type": "string" + } + ] + }, + { + "id": "f8a620b1-b71c-42eb-917b-c5879b4f9e96", + "type": "date", + "name": "sec_date", + "sections": [], + "properties": [ + { + "id": "9d207f37-ca27-4df7-9409-02e369cced7a", + "name": "dates_no", + "value": [ + "1997-12-14", + "00-12-14", + "89-07-04" + ], + "type": "string" + } + ] + }, + { + "id": "857f3300-d6ac-4c48-9d95-12a1d70b9092", + "type": "datetime", + "name": "sec_datetime", + "sections": [], + "properties": [ + { + "id": "8fa99c25-4cc9-491b-9398-59afb563f103", + "name": "datetimes_no", + "value": [ + "97-12-14 11:11:11", + "97-12-14 12:12", + "1997-12-14 03:03" + ], + "type": "string" + } + ] + }, + { + "id": "922d2ab9-bfa6-413d-ba8d-f57cc775b6ac", + "type": "time", + "name": "sec_time", + "sections": [], + "properties": [ + { + "id": "a275b4cc-e850-4a46-95eb-a01772706468", + "name": "times_no", + "value": [ + "11:11:11", + "12:12:12", + "03:03:03" + ], + "type": "string" + } + ] + }, + { + "id": "f18728fc-1e75-43dc-810f-19c9c64f29c0", + "type": "boolean", + "name": "sec_boolean", + "sections": [], + "properties": [ + { + "id": "4080b4ad-fe08-4abc-8071-a84a8169cf30", + "name": "sent_no", + "value": [ + "False", + "True", + "TRUE", + "False", + "t" + ], + "type": "string" + } + ] + }, + { + "id": "22d0732c-afa2-4448-962b-578dd60e0b37", + "type": "text", + "name": "sec_text", + "sections": [], + "properties": [ + { + "id": "3e97d1e2-5ff1-44b1-9498-a488209b326c", + "name": "texts_no", + "value": [ + "lineA \n lineB", + "lineC\n lineD", + "\nlineE\n lineF" + ], + "type": "string" + } + ] + }, + { + "id": "8c49b24b-0b86-4dd6-a4da-45e1c2c504c6", + "type": "2-tuple", + "name": "sec_2_tuple", + "sections": [], + "properties": [ + { + "id": "4f0cfc1b-0175-46f9-9ad4-62ab19fc6625", + "name": "Location_no", + "value": [ + "(39.12; 67.19)", + "(39.12; 67.19)", + "(39.12; 67.18)" + ], + "type": "string" + } + ] + }, + { + "id": "62531fb3-8b5a-4490-a421-8e15cb787f5f", + "type": "3-tuple", + "name": "sec_3_tuple", + "sections": [], + "properties": [ + { + "id": "45eae816-9023-414d-b15f-ae7a8a845f45", + "name": "Coos_no", + "value": [ + "(39.12; 89; 67.19)", + "(39.12; 78; 67.19)", + "(39.12; 56; 67.18)" + ], + "type": "string" + } + ] + } + ], + "properties": [] + }, + { + "id": "bc217fca-d51a-4d01-b37f-7270f02f88a4", + "type": "mislabelled_dtypes", + "name": "mislabelled_dtypes", + "sections": [ + { + "id": "3fd7dd7e-246f-4002-a577-534792fb5357", + "type": "string", + "name": "sec_string", + "sections": [], + "properties": [ + { + "id": "7c987cf5-7bc7-452d-a87f-69a8e68397ea", + "name": "words_mislabelled", + "value": [ + "hello", + "-world", + "3", + "True" + ], + "type": "string" + } + ] + }, + { + "id": "196e9408-3d8a-478a-829d-0834d14c9a32", + "type": "int", + "name": "sec_int", + "sections": [], + "properties": [ + { + "id": "c581ffae-8d7f-44e7-92cd-b96c253e680d", + "name": "members_mislabelled", + "value": [ + "-13", + "101", + "-11", + "0", + "-8" + ], + "type": "string" + } + ] + }, + { + "id": "fbe32258-9b80-43b4-8399-856780e1c67e", + "type": "float", + "name": "sec_float", + "sections": [], + "properties": [ + { + "id": "7302729f-ccc0-463d-8c39-78ec1136237f", + "name": "potential_mislabelled", + "value": [ + "-4.8", + "10.0", + "-11.9", + "-10.0", + "18.0" + ], + "type": "string" + } + ] + }, + { + "id": "daa8fc2f-fc44-408f-b824-d6f890974e36", + "type": "date", + "name": "sec_date", + "sections": [], + "properties": [ + { + "id": "5818ae19-80d5-42c1-ae05-61a36c2b9546", + "name": "dates_mislabelled", + "value": [ + "1997-12-14", + "00-12-14", + "89-07-04" + ], + "type": "string" + } + ] + }, + { + "id": "d2d0b323-4701-4fc1-ac9c-9bb9b04c2710", + "type": "datetime", + "name": "sec_datetime", + "sections": [], + "properties": [ + { + "id": "2be4f2c2-5364-4c6e-86c7-8b940cc8ed5f", + "name": "datetimes_mislabelled", + "value": [ + "97-12-14 11:11:11", + "97-12-14 12:12", + "1997-12-14 03:03" + ], + "type": "string" + } + ] + }, + { + "id": "aa38c228-7870-41a0-89bd-f579406468c0", + "type": "time", + "name": "sec_time", + "sections": [], + "properties": [ + { + "id": "d20c097f-162c-456a-beb4-0a55e261c9b8", + "name": "times_mislabelled", + "value": [ + "11:11:11", + "12:12:12", + "03:03:03" + ], + "type": "string" + } + ] + }, + { + "id": "b4d969e7-78b3-4bc3-942b-0126046ba538", + "type": "boolean", + "name": "sec_boolean", + "sections": [], + "properties": [ + { + "id": "d7f0cf25-a87a-4b77-a7a5-41c4b31763a5", + "name": "sent_mislabelled", + "value": [ + "False", + "True", + "TRUE", + "False", + "t" + ], + "type": "string" + } + ] + }, + { + "id": "fe16efe1-5f94-475e-b652-70b7f5495de5", + "type": "text", + "name": "sec_text", + "sections": [], + "properties": [ + { + "id": "ae863fc4-b998-4da1-b586-96c0d0124ddb", + "name": "texts_mislabelled", + "value": [ + "lineA \n lineB", + "lineC\n lineD", + "\nlineE\n lineF" + ], + "type": "string" + } + ] + }, + { + "id": "513ce09d-2d15-40b2-8d65-10438ba55752", + "type": "2-tuple", + "name": "sec_2_tuple", + "sections": [], + "properties": [ + { + "id": "99318b7a-ce26-4763-bd32-9d3dfb4b956a", + "name": "Location_mislabelled", + "value": [ + "(39.12; 67.19)", + "(39.12; 67.19)", + "(39.12; 67.18)" + ], + "type": "string" + } + ] + }, + { + "id": "f5bb809b-e621-4524-9858-1004e2232a57", + "type": "3-tuple", + "name": "sec_3_tuple", + "sections": [], + "properties": [ + { + "id": "1c41cc57-1704-4494-a463-71552a3f90bb", + "name": "Coos_mislabelled", + "value": [ + "(39.12; 89; 67.19)", + "(39.12; 78; 67.19)", + "(39.12; 56; 67.18)" + ], + "type": "string" + } + ] + } + ], + "properties": [] + } + ] + }, + "odml-version": "1.1" +} \ No newline at end of file diff --git a/test/resources/validation_section.json b/test/resources/validation_section.json new file mode 100644 index 00000000..d7748252 --- /dev/null +++ b/test/resources/validation_section.json @@ -0,0 +1,21 @@ +{ + "Document": { + "id": "318c4323-f83c-4abc-adbd-daf47816fe87", + "sections": [ + { + "id": "552c620d-ac5f-46c4-869e-f2b8170e7a1e", + "name": "sec_type_undefined", + "sections": [], + "properties": [] + }, + { + "id": "52540350-3533-42bc-b438-f031f5ac3641", + "type": "", + "name": "sec_type_empty", + "sections": [], + "properties": [] + } + ] + }, + "odml-version": "1.1" +} \ No newline at end of file From 719e6fb891d771e2405bedc06184237edf609bd5 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:33:43 +0100 Subject: [PATCH 19/24] [test_validation.py] Add Test for JSON File Loading Sections - Replace part of current JSON File Loading Test - Incorporate file at resources/validation_section.json --- test/test_validation.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/test/test_validation.py b/test/test_validation.py index 2780ef23..0c4e5de2 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -290,28 +290,25 @@ def test_load_dtypes_xml(self): self.assertError(validate(doc), 'Dtype of property "Coos_mislabelled" currently is ' '"string", but might fit dtype "3-tuple"!') - - - - - Undefined Section type - - Properties with undefined dtypes + def test_load_section_json(self): + """ + Test if loading json document raises validation errors for Sections with undefined type. """ - dir_path = os.path.dirname(os.path.realpath(__file__)) - path = os.path.join(dir_path, "resources", "integration.json") + path = os.path.join(self.dir_path, "resources", "validation_section.json") doc = odml.load(path, "JSON") - sec_test_1_err = False - sec_test_2_err = False + sec_type_undefined_err = False + sec_type_empty_err = False for err in validate(doc).errors: - if err.msg == "Section type undefined" and err.obj.name == "sec_test_1": - sec_test_1_err = True - elif err.msg == "Section type undefined" and err.obj.name == "sec_test_2": - sec_test_2_err = True + if err.msg == "Section type undefined" and err.obj.name == "sec_type_undefined": + sec_type_undefined_err = True + elif err.msg == "Section type undefined" and err.obj.name == "sec_type_empty": + sec_type_empty_err = True - assert sec_test_1_err - assert sec_test_2_err + assert sec_type_undefined_err + assert sec_type_empty_err self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') self.assertError(validate(doc), From 2488ceb6b0e0fe5bf4fb4a3745cb52addadc1a7d Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:34:56 +0100 Subject: [PATCH 20/24] [test_validation.py] Add Test for JSON File Loading Property Dtypes - Replace current JSON File Loading Test - Incorporate file at resources/validation_dtypes.json --- test/test_validation.py | 76 ++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/test/test_validation.py b/test/test_validation.py index 0c4e5de2..74f25483 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -310,25 +310,67 @@ def test_load_section_json(self): assert sec_type_undefined_err assert sec_type_empty_err - self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') - self.assertError(validate(doc), - 'Dtype of property "potential" currently is "string", but might fit dtype "float"!') - self.assertError(validate(doc), 'Dtype of property "dates" currently is "string", but might fit dtype "date"!') - self.assertError(validate(doc), - 'Dtype of property "datetimes" currently is "string", but might fit dtype "datetime"!') - self.assertError(validate(doc), 'Dtype of property "times" currently is "string", but might fit dtype "time"!') - self.assertError(validate(doc), - 'Dtype of property "sent" currently is "string", but might fit dtype "boolean"!') - self.assertError(validate(doc), 'Dtype of property "texts" currently is "string", but might fit dtype "text"!') - self.assertError(validate(doc), - 'Dtype of property "Location" currently is "string", but might fit dtype "2-tuple"!') - self.assertError(validate(doc), - 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') + def test_load_dtypes_json(self): + """ + Test if loading json document raises validation errors for Properties with undefined dtypes. + """ + + path = os.path.join(self.dir_path, "resources", "validation_dtypes.json") + doc = odml.load(path, "JSON") + + self.assertError(validate(doc), 'Dtype of property "members_no" currently is "string", ' + 'but might fit dtype "int"!') + + self.assertError(validate(doc), 'Dtype of property "potential_no" currently is "string", ' + 'but might fit dtype "float"!') + + self.assertError(validate(doc), 'Dtype of property "dates_no" currently is "string", ' + 'but might fit dtype "date"!') + + self.assertError(validate(doc), 'Dtype of property "datetimes_no" currently is "string", ' + 'but might fit dtype "datetime"!') + + self.assertError(validate(doc), 'Dtype of property "times_no" currently is "string", ' + 'but might fit dtype "time"!') + + self.assertError(validate(doc), 'Dtype of property "sent_no" currently is "string", ' + 'but might fit dtype "boolean"!') + + self.assertError(validate(doc), 'Dtype of property "Location_no" currently is "string", ' + 'but might fit dtype "2-tuple"!') + + self.assertError(validate(doc), 'Dtype of property "Coos_no" currently is "string", ' + 'but might fit dtype "3-tuple"!') + + self.assertError(validate(doc), 'Dtype of property "members_mislabelled" currently is ' + '"string", but might fit dtype "int"!') + + self.assertError(validate(doc), 'Dtype of property "potential_mislabelled" currently is ' + '"string", but might fit dtype "float"!') + + self.assertError(validate(doc), 'Dtype of property "dates_mislabelled" currently is ' + '"string", but might fit dtype "date"!') + + self.assertError(validate(doc), 'Dtype of property "datetimes_mislabelled" currently is ' + '"string", but might fit dtype "datetime"!') + + self.assertError(validate(doc), 'Dtype of property "times_mislabelled" currently is ' + '"string", but might fit dtype "time"!') + + self.assertError(validate(doc), 'Dtype of property "sent_mislabelled" currently is ' + '"string", but might fit dtype "boolean"!') + + self.assertError(validate(doc), 'Dtype of property "texts_mislabelled" currently is ' + '"string", but might fit dtype "text"!') + + self.assertError(validate(doc), 'Dtype of property "Location_mislabelled" currently is ' + '"string", but might fit dtype "2-tuple"!') + + self.assertError(validate(doc), 'Dtype of property "Coos_mislabelled" currently is ' + '"string", but might fit dtype "3-tuple"!') + - def test_load_yaml(self): - """ - Test if loading yaml document raises validation errors. Errors are: - Undefined Section type - Properties with undefined dtypes From 976b10b7001e9f2e858b5ac1033dc3aa8f0ddf8b Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:35:36 +0100 Subject: [PATCH 21/24] [test/resources] Replace File for YAML Validation Tests Remove integration.yaml Add validation_dtypes.yaml Add validation_section.yaml Split test functionality. --- test/resources/integration.yaml | 133 ------------- test/resources/validation_dtypes.yaml | 257 +++++++++++++++++++++++++ test/resources/validation_section.yaml | 13 ++ 3 files changed, 270 insertions(+), 133 deletions(-) delete mode 100644 test/resources/integration.yaml create mode 100644 test/resources/validation_dtypes.yaml create mode 100644 test/resources/validation_section.yaml diff --git a/test/resources/integration.yaml b/test/resources/integration.yaml deleted file mode 100644 index 66c59bd3..00000000 --- a/test/resources/integration.yaml +++ /dev/null @@ -1,133 +0,0 @@ -Document: - id: b4be7fe5-0a1c-4499-8cf5-b34ef73689e6 - sections: - - id: 859e4f67-d59d-477a-8c12-248e30d8da35 - name: sec_test_1 - properties: [] - sections: [] - - id: 8b7785dd-cc83-41ab-ae0a-4f10d362a008 - name: sec_test_2 - properties: [] - sections: [] - type: "" - - id: 42721936-f62e-4e40-9152-2fb4c6857e92 - name: sec1 - properties: - - id: 96f2f6e4-c9f2-426f-a87e-1af2e75f8a27 - name: members - type: string - value: - - '-13' - - '101' - - '-11' - - '0' - - '-8' - sections: [] - type: test - - id: afdcef87-82e9-4cbd-a9ff-1cac79928fac - name: sec2 - properties: - - id: 685d4de7-47ba-4ffb-8eb1-b948e9204e2e - name: potential - value: - - '-4.8' - - '10.0' - - '-11.9' - - '-10.0' - - '18.0' - sections: [] - type: test - - id: 182569c1-cf26-4e14-9159-68ddc97c0188 - name: sec3 - properties: - - id: 777b23ac-4639-49a5-96f4-037390a344d6 - name: dates - type: string - value: - - '1997-12-14' - - 00-12-14 - - '8' - sections: [] - type: test - - id: 3964fad7-12ff-459a-86e2-100f5c986fa6 - name: sec4 - properties: - - id: 1be11c05-4c5f-4bea-8409-c36bf1fbc0d2 - name: datetimes - type: string - value: - - 97-12-14 11:11:11 - - 97-12-14 12:12 - - 1997-12-14 03:03 - sections: [] - type: test - - id: 4f4de2c9-7b2d-4210-a280-f7daa823c3b3 - name: sec5 - properties: - - id: 8404bbb8-6b9f-4a5a-8396-033df7ad6efc - name: times - type: string - value: - - '11:11:11' - - '12:12:12' - - 03:03:03 - - '8' - sections: [] - type: test - - id: c689f7d7-3a84-4747-a8fc-c117d15d781e - name: sec6 - properties: - - id: b0e97058-aedf-44cb-927f-172d5971ebbb - name: sent - type: string - value: - - 'False' - - 'True' - - 'TRUE' - - '0' - - t - - '12' - - Ft - sections: [] - type: test - - id: 10f8eed7-b36a-475a-a392-fd0aa1273a9a - name: sec7 - properties: - - id: 99046f03-71ad-4512-a4be-b6241a10186a - name: texts - type: string - value: - - "line1\n line2" - - "line3\n line4" - - ' - - line5 - - ' - sections: [] - type: test - - id: 5001ca98-0c97-4ec7-b194-048055d79394 - name: sec8 - properties: - - id: 98772d32-d917-46e8-8fc5-728d72db1480 - name: Location - type: string - value: - - (39.12; 67.19) - - (39.12; 67.19) - - (39.12) - sections: [] - type: test - - id: 55c52ce7-1de0-4b1c-a0d6-f63af497a5c9 - name: sec9 - properties: - - id: 01dfd04b-3c61-48c4-b45b-d754d1645f8b - name: Coos - type: string - value: - - (39.12; 89; 67.19) - - (39.12; 78; 67.19) - - (39.12) - sections: [] - type: test -odml-version: '1.1' diff --git a/test/resources/validation_dtypes.yaml b/test/resources/validation_dtypes.yaml new file mode 100644 index 00000000..266898ca --- /dev/null +++ b/test/resources/validation_dtypes.yaml @@ -0,0 +1,257 @@ +Document: + id: 2af46ee0-48c3-4874-a67f-5f8b2951cf51 + sections: + - id: ebb1607f-a450-409f-b286-b9a8aef0900f + name: no_dtypes + properties: [] + sections: + - id: d7f47035-4e49-4438-b61f-227717a8bbe4 + name: sec_string + properties: + - id: a211dd69-a2b2-46f1-95f1-6b30d8b96ba0 + name: words_no + value: + - hello + - -world + - '3' + - 'True' + sections: [] + - id: efa678af-e8a5-491e-a0ea-29653bcb8c7b + name: sec_int + properties: + - id: f1e31bef-3652-4890-b570-b92c3903a8cf + name: members_no + value: + - '-13' + - '101' + - '-11' + - '0' + - '-8' + sections: [] + type: int + - id: ba3cd001-033b-4991-a205-9d4fd1138848 + name: sec_float + properties: + - id: 06ebdcd1-9635-4afc-bf2c-1ef44946f387 + name: potential_no + value: + - '-4.8' + - '10.0' + - '-11.9' + - '-10.0' + - '18.0' + sections: [] + type: float + - id: e0a47494-1d45-4e9e-a1c9-2d9987cc6625 + name: sec_date + properties: + - id: c9d46221-a403-4a48-870c-16f7632b24d7 + name: dates_no + value: + - '1997-12-14' + - 00-12-14 + - 89-07-04 + sections: [] + type: date + - id: 1e06a7d7-076b-45f2-b3e1-8795f9cd71c9 + name: sec_datetime + properties: + - id: 7f98fcff-4ac5-4c38-a071-8f0801ca0aa8 + name: datetimes_no + value: + - 97-12-14 11:11:11 + - 97-12-14 12:12 + - 1997-12-14 03:03 + sections: [] + type: datetime + - id: a8c624fc-1e0a-4da4-9862-64a0ad84fa0e + name: sec_time + properties: + - id: b2d169dd-1a00-43ad-b481-a85561c4d001 + name: times_no + value: + - '11:11:11' + - '12:12:12' + - 03:03:03 + sections: [] + type: time + - id: 4aafafb0-4d07-4689-a54c-565b4805f740 + name: sec_boolean + properties: + - id: 650e60e1-7647-40cb-8371-9f65e025c51c + name: sent_no + value: + - 'False' + - 'True' + - 'TRUE' + - 'False' + - t + sections: [] + type: boolean + - id: 6a1b6122-5d4f-4618-959e-1955a6c1e26b + name: sec_text + properties: + - id: 717d2d4a-bf44-4329-8967-59348c7b623c + name: texts_no + value: + - "lineA \n lineB" + - "lineC\n lineD" + - "\nlineE\n lineF" + sections: [] + type: text + - id: ffe14c58-0ad7-420c-a97f-7805a1ba8bf6 + name: sec_2_tuple + properties: + - id: e52973a8-9186-4739-a789-73286745cf4c + name: Location_no + value: + - (39.12; 67.19) + - (39.12; 67.19) + - (39.12; 67.18) + sections: [] + type: 2-tuple + - id: e47bc5f7-38f6-4575-891a-91bcf64e7872 + name: sec_3_tuple + properties: + - id: 443ef8a0-5e6e-480f-83ad-e305398dd828 + name: Coos_no + value: + - (39.12; 89; 67.19) + - (39.12; 78; 67.19) + - (39.12; 56; 67.18) + sections: [] + type: 3-tuple + type: no_dtypes + - id: cd3e02f7-ff2f-4b12-85d1-cd0a23ca8a83 + name: mislabelled_dtypes + properties: [] + sections: + - id: 2874da49-418d-452d-b4f8-540764d513aa + name: sec_string + properties: + - id: 8a6360e6-dce7-42c3-9477-eca7f9f51926 + name: words_mislabelled + type: string + value: + - hello + - -world + - '3' + - 'True' + sections: [] + type: string + - id: 8c926b02-4d6f-4e8f-aa92-78460eeeec5e + name: sec_int + properties: + - id: f46c9d64-28f8-4cc8-9fcc-31a0e5337ffe + name: members_mislabelled + type: string + value: + - '-13' + - '101' + - '-11' + - '0' + - '-8' + sections: [] + type: int + - id: 0785a91c-e126-41c1-9c64-dabee9750ea8 + name: sec_float + properties: + - id: 07f877fc-5190-4f21-a834-0bfea38dc5b0 + name: potential_mislabelled + type: string + value: + - '-4.8' + - '10.0' + - '-11.9' + - '-10.0' + - '18.0' + sections: [] + type: float + - id: 7684924a-5875-48f5-838d-509a99274097 + name: sec_date + properties: + - id: 36d587fd-0d83-4f71-b2bd-b08628378e03 + name: dates_mislabelled + type: string + value: + - '1997-12-14' + - 00-12-14 + - 89-07-04 + sections: [] + type: date + - id: ecc8be3a-e17c-48e5-92a4-07f829713531 + name: sec_datetime + properties: + - id: b7d415c1-8c1b-4f94-b425-37c3f6c2f131 + name: datetimes_mislabelled + type: string + value: + - 97-12-14 11:11:11 + - 97-12-14 12:12 + - 1997-12-14 03:03 + sections: [] + type: datetime + - id: 1a5d94f6-6ebc-4f5e-b621-93cfa2bebb50 + name: sec_time + properties: + - id: edc1fa24-f3a4-448f-9f1e-9387383a0fa3 + name: times_mislabelled + type: string + value: + - '11:11:11' + - '12:12:12' + - 03:03:03 + sections: [] + type: time + - id: 75fb464d-9ea5-49a0-a820-343b08cfba2d + name: sec_boolean + properties: + - id: d2c2c6d1-1ca7-4544-8379-d8451954d38f + name: sent_mislabelled + type: string + value: + - 'False' + - 'True' + - 'TRUE' + - 'False' + - t + sections: [] + type: boolean + - id: 889124e7-09a5-400a-b7de-7d0e6d019d7e + name: sec_text + properties: + - id: 0a05a652-cfe7-46c2-bba4-fa9d1714a635 + name: texts_mislabelled + type: string + value: + - "lineA \n lineB" + - "lineC\n lineD" + - "\nlineE\n lineF" + sections: [] + type: text + - id: db9d7872-86d4-4155-9a14-2a44a4402b1a + name: sec_2_tuple + properties: + - id: 508517b1-e67a-42d3-aca1-6bd55ddea0e6 + name: Location_mislabelled + type: string + value: + - (39.12; 67.19) + - (39.12; 67.19) + - (39.12; 67.18) + sections: [] + type: 2-tuple + - id: a75c955a-6bef-40a7-99fd-a26160ae79d2 + name: sec_3_tuple + properties: + - id: 84aa21f3-c571-420c-963f-59c5785a45f2 + name: Coos_mislabelled + type: string + value: + - (39.12; 89; 67.19) + - (39.12; 78; 67.19) + - (39.12; 56; 67.18) + sections: [] + type: 3-tuple + type: mislabelled_dtypes +odml-version: '1.1' diff --git a/test/resources/validation_section.yaml b/test/resources/validation_section.yaml new file mode 100644 index 00000000..93da8572 --- /dev/null +++ b/test/resources/validation_section.yaml @@ -0,0 +1,13 @@ +Document: + id: 468dfc92-d794-4ae9-9ffe-a0944a957d36 + sections: + - id: 5279b30a-3dfd-4c44-ac18-2a80e7647449 + name: sec_type_undefined + properties: [] + sections: [] + - id: 2033d777-fc2b-4b73-a8f0-ae164bb1ea10 + name: sec_type_empty + properties: [] + sections: [] + type: '' +odml-version: '1.1' From f85ffe63fc89d00d2f45f3c3abc4e7eda782264e Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:36:28 +0100 Subject: [PATCH 22/24] [test_validation.py] Add Test for YAML File Loading Sections - Replace part of current YAML File Loading Test - Incorporate file at resources/validation_section.yaml --- test/test_validation.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/test/test_validation.py b/test/test_validation.py index 74f25483..4473e381 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -369,25 +369,28 @@ def test_load_dtypes_json(self): self.assertError(validate(doc), 'Dtype of property "Coos_mislabelled" currently is ' '"string", but might fit dtype "3-tuple"!') - - - - - Undefined Section type - - Properties with undefined dtypes + def test_load_section_yaml(self): + """ + Test if loading yaml document raises validation errors for Sections with undefined type. """ - dir_path = os.path.dirname(os.path.realpath(__file__)) - path = os.path.join(dir_path, "resources", "integration.yaml") + path = os.path.join(self.dir_path, "resources", "validation_section.yaml") doc = odml.load(path, "YAML") - sec_test_1_err = False - sec_test_2_err = False + sec_type_undefined_err = False + sec_type_empty_err = False for err in validate(doc).errors: - if err.msg == "Section type undefined" and err.obj.name == "sec_test_1": - sec_test_1_err = True - elif err.msg == "Section type undefined" and err.obj.name == "sec_test_2": - sec_test_2_err = True + if err.msg == "Section type undefined" and err.obj.name == "sec_type_undefined": + sec_type_undefined_err = True + elif err.msg == "Section type undefined" and err.obj.name == "sec_type_empty": + sec_type_empty_err = True + + assert sec_type_undefined_err + assert sec_type_empty_err + + + assert sec_test_1_err assert sec_test_2_err From 52eb412da540bef9b3a320265efcf976dc3d4e08 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Mon, 23 Mar 2020 17:37:09 +0100 Subject: [PATCH 23/24] [test_validation.py] Add Test for YAML File Loading Property Dtypes - Replace current YAML File Loading Test - Incorporate file at resources/validation_dtypes.yaml --- test/test_validation.py | 72 +++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/test/test_validation.py b/test/test_validation.py index 4473e381..7f4a170e 100644 --- a/test/test_validation.py +++ b/test/test_validation.py @@ -389,23 +389,61 @@ def test_load_section_yaml(self): assert sec_type_undefined_err assert sec_type_empty_err + def test_load_dtypes_yaml(self): + """ + Test if loading yaml document raises validation errors for Properties with undefined dtypes. + """ + + path = os.path.join(self.dir_path, "resources", "validation_dtypes.yaml") + doc = odml.load(path, "YAML") + + self.assertError(validate(doc), 'Dtype of property "members_no" currently is "string", ' + 'but might fit dtype "int"!') + + self.assertError(validate(doc), 'Dtype of property "potential_no" currently is "string", ' + 'but might fit dtype "float"!') + + self.assertError(validate(doc), 'Dtype of property "dates_no" currently is "string", ' + 'but might fit dtype "date"!') + + self.assertError(validate(doc), 'Dtype of property "datetimes_no" currently is "string", ' + 'but might fit dtype "datetime"!') + + self.assertError(validate(doc), 'Dtype of property "times_no" currently is "string", ' + 'but might fit dtype "time"!') + + self.assertError(validate(doc), 'Dtype of property "sent_no" currently is "string", ' + 'but might fit dtype "boolean"!') + + self.assertError(validate(doc), 'Dtype of property "Location_no" currently is "string", ' + 'but might fit dtype "2-tuple"!') + + self.assertError(validate(doc), 'Dtype of property "Coos_no" currently is "string", ' + 'but might fit dtype "3-tuple"!') + + self.assertError(validate(doc), 'Dtype of property "members_mislabelled" currently is ' + '"string", but might fit dtype "int"!') + + self.assertError(validate(doc), 'Dtype of property "potential_mislabelled" currently is ' + '"string", but might fit dtype "float"!') + + self.assertError(validate(doc), 'Dtype of property "dates_mislabelled" currently is ' + '"string", but might fit dtype "date"!') + + self.assertError(validate(doc), 'Dtype of property "datetimes_mislabelled" currently is ' + '"string", but might fit dtype "datetime"!') + self.assertError(validate(doc), 'Dtype of property "times_mislabelled" currently is ' + '"string", but might fit dtype "time"!') + self.assertError(validate(doc), 'Dtype of property "sent_mislabelled" currently is ' + '"string", but might fit dtype "boolean"!') - assert sec_test_1_err - assert sec_test_2_err - - self.assertError(validate(doc), 'Dtype of property "members" currently is "string", but might fit dtype "int"!') - self.assertError(validate(doc), - 'Dtype of property "potential" currently is "string", but might fit dtype "float"!') - self.assertError(validate(doc), 'Dtype of property "dates" currently is "string", but might fit dtype "date"!') - self.assertError(validate(doc), - 'Dtype of property "datetimes" currently is "string", but might fit dtype "datetime"!') - self.assertError(validate(doc), 'Dtype of property "times" currently is "string", but might fit dtype "time"!') - self.assertError(validate(doc), - 'Dtype of property "sent" currently is "string", but might fit dtype "boolean"!') - self.assertError(validate(doc), 'Dtype of property "texts" currently is "string", but might fit dtype "text"!') - self.assertError(validate(doc), - 'Dtype of property "Location" currently is "string", but might fit dtype "2-tuple"!') - self.assertError(validate(doc), - 'Dtype of property "Coos" currently is "string", but might fit dtype "3-tuple"!') + self.assertError(validate(doc), 'Dtype of property "texts_mislabelled" currently is ' + '"string", but might fit dtype "text"!') + + self.assertError(validate(doc), 'Dtype of property "Location_mislabelled" currently is ' + '"string", but might fit dtype "2-tuple"!') + + self.assertError(validate(doc), 'Dtype of property "Coos_mislabelled" currently is ' + '"string", but might fit dtype "3-tuple"!') From fd78be49350e77270679068bf21ce42d6c361274 Mon Sep 17 00:00:00 2001 From: fschrader1992 Date: Tue, 24 Mar 2020 13:40:12 +0100 Subject: [PATCH 24/24] [validation_dtypes.json] Remove Wrongly Set Dtype String --- test/resources/validation_dtypes.json | 30 +++++++++------------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/test/resources/validation_dtypes.json b/test/resources/validation_dtypes.json index 411b20bf..3c3d347f 100644 --- a/test/resources/validation_dtypes.json +++ b/test/resources/validation_dtypes.json @@ -20,8 +20,7 @@ "-world", "3", "True" - ], - "type": "string" + ] } ] }, @@ -40,8 +39,7 @@ "-11", "0", "-8" - ], - "type": "string" + ] } ] }, @@ -60,8 +58,7 @@ "-11.9", "-10.0", "18.0" - ], - "type": "string" + ] } ] }, @@ -78,8 +75,7 @@ "1997-12-14", "00-12-14", "89-07-04" - ], - "type": "string" + ] } ] }, @@ -96,8 +92,7 @@ "97-12-14 11:11:11", "97-12-14 12:12", "1997-12-14 03:03" - ], - "type": "string" + ] } ] }, @@ -114,8 +109,7 @@ "11:11:11", "12:12:12", "03:03:03" - ], - "type": "string" + ] } ] }, @@ -134,8 +128,7 @@ "TRUE", "False", "t" - ], - "type": "string" + ] } ] }, @@ -152,8 +145,7 @@ "lineA \n lineB", "lineC\n lineD", "\nlineE\n lineF" - ], - "type": "string" + ] } ] }, @@ -170,8 +162,7 @@ "(39.12; 67.19)", "(39.12; 67.19)", "(39.12; 67.18)" - ], - "type": "string" + ] } ] }, @@ -188,8 +179,7 @@ "(39.12; 89; 67.19)", "(39.12; 78; 67.19)", "(39.12; 56; 67.18)" - ], - "type": "string" + ] } ] }