2819 consolidate general information schema validation (#3473)

* #2819 Initial attempt to consolidate GeneralInformation schemas in order to remove the unnecessary one

* #2819 Initial step in translating GeneralInformation schema into python checks

* #2819 Removed extra general information schema and consolidated existing one

* #2819 Updated test cases and fixtures to meet changes in schema

* #2819 Updated code to only use one schema for general information

* #2819 Removed unused schema

* Make addresses conditionally required by country

* Re-require the address fields

* Display foreign addresses in the summary view

* Utilize two separate gen info validators.

* Whitespace lint.

* #2819 Improved error handling

* Fix test case to reflect validation error handling

* #2819 Increased international address schema size and rebuilt schemas

* #2819 Ensure general information form allows 500 characters max for foreign address

* More schema update. This is a temp update, we will revisit a better approach

---------

Co-authored-by: James Person <jperson1@umbc.edu>

backend/audit/cross_validation/validate_general_information.py

@@ -2,8 +2,52 @@
 from django.conf import settings
 from jsonschema import FormatChecker, validate
 from jsonschema.exceptions import ValidationError as JSONSchemaValidationError
-
+from django.core.exceptions import ValidationError
 from audit.cross_validation.naming import NC
+from audit.validators import validate_general_information_schema_rules
+
+required_fields = {
+    #  `key_in_json_schema: label_for_user`
+    "audit_type": "Type of audit",
+    "auditee_address_line_1": "Auditee street",
+    "auditee_city": "Auditee city",
+    "auditee_zip": "Auditee ZIP",
+    "auditee_contact_name": "Auditee contact name",
+    "auditee_contact_title": "Auditee contact title",
+    "auditee_email": "Auditee email",
+    "auditee_fiscal_period_end": "Fiscal period end date",
+    "auditee_fiscal_period_start": "Fiscal period start date",
+    "auditee_name": "Auditee name",
+    "auditee_phone": "Auditee phone",
+    "auditee_state": "Auditee state",
+    "auditee_uei": "Auditee UEI",
+    "auditor_contact_name": "Auditor contact name",
+    "auditor_contact_title": "Auditor contact title",
+    "auditor_ein": "Auditor EIN",
+    "auditor_ein_not_an_ssn_attestation": "Confirmation that auditor EIN is not an SSN",
+    "auditor_email": "Auditor email",
+    "auditor_firm_name": "Audit firm name",
+    "auditor_phone": "Auditor phone number",
+    "ein": "Auditee EIN",
+    "ein_not_an_ssn_attestation": "Confirmation that auditee EIN is not an SSN",
+    "is_usa_based": "Auditor is USA-based",
+    "met_spending_threshold": "Spending threshold",
+    "multiple_eins_covered": "Multiple EINs covered",
+    "multiple_ueis_covered": "Multiple UEIs covered",
+    "secondary_auditors_exist": "Confirmation that secondary auditors exist",
+    "user_provided_organization_type": "Organization type",
+    "audit_period_covered": "Audit period",
+}
+
+# optionally_required_fields is handled separately from required_fields
+optionally_required_fields = {
+    "auditor_state": "Auditor state",
+    "auditor_zip": "Auditor ZIP",
+    "auditor_address_line_1": "Auditor street",
+    "auditor_city": "Auditor city",
+    "auditor_country": "Auditor country",
+    "auditor_international_address": "Auditor international address",
+}
 
 
 def validate_general_information(sac_dict, *_args, **_kwargs):
@@ -20,11 +64,33 @@ def validate_general_information(sac_dict, *_args, **_kwargs):
     """
     all_sections = sac_dict["sf_sac_sections"]
     general_information = all_sections[NC.GENERAL_INFORMATION]
-    schema_path = settings.SECTION_SCHEMA_DIR / "GeneralInformationComplete.schema.json"
+    schema_path = settings.SECTION_SCHEMA_DIR / "GeneralInformationRequired.schema.json"
     schema = json.loads(schema_path.read_text(encoding="utf-8"))
 
+    errors = _check_required_field(general_information)
+    if errors:
+        return errors
     try:
+        validate_general_information_schema_rules(general_information)
         validate(general_information, schema, format_checker=FormatChecker())
     except JSONSchemaValidationError as err:
         return [{"error": f"General Information: {str(err)}"}]
+    except ValidationError as err:
+        return [{"error": f"General Information: {str(err.message)}"}]
+    return []
+
+
+def _check_required_field(general_information):
+    """
+    Check that all required fields are present in the general information.
+    """
+    # Check that all required fields are present or return a message pointing to the missing fields
+    missing_fields = []
+    for key, label in required_fields.items():
+        if key not in general_information or general_information[key] in [None, ""]:
+            missing_fields.append(label)
+
+    if missing_fields:
+        return [{"error": f"Missing required fields: {', '.join(missing_fields)}"}]
+
     return []

backend/audit/fixtures/json/general-information--test0001test--simple-pass.json

@@ -28,7 +28,7 @@
   "auditor_address_line_1": "Main Hall",
   "met_spending_threshold": true,
   "secondary_auditors_exist": false,
-  "audit_period_other_months": "null",
+  "audit_period_other_months": "",
   "auditee_fiscal_period_end": "2023-01-01",
   "ein_not_an_ssn_attestation": true,
   "auditee_fiscal_period_start": "2022-01-01",

backend/audit/test_schemas.py

@@ -42,7 +42,7 @@ class GeneralInformationSchemaValidityTest(SimpleTestCase):
     """
 
     GENERAL_INFO_SCHEMA = json.loads(
-        (SECTION_SCHEMA_DIR / "GeneralInformation.schema.json").read_text(
+        (SECTION_SCHEMA_DIR / "GeneralInformationRequired.schema.json").read_text(
             encoding="utf-8"
         )
     )
@@ -72,7 +72,7 @@ def test_invalid_auditee_fiscal_period_start(self):
 
         self.assertRaisesRegex(
             exceptions.ValidationError,
-            "'' was expected",  # Value also accepts a blank string, so this error comes back.
+            "'not a date' is not a 'date'",
             validate,
             simple_case,
             schema,
@@ -91,7 +91,7 @@ def test_invalid_auditee_fiscal_period_end(self):
 
         self.assertRaisesRegex(
             exceptions.ValidationError,
-            "'' was expected",  # Value also accepts a blank string, so this error comes back.
+            "'not a date' is not a 'date'",
             validate,
             simple_case,
             schema,
@@ -129,7 +129,7 @@ def test_invalid_ein(self):
 
                 with self.assertRaisesRegex(
                     exceptions.ValidationError,
-                    "is not valid",
+                    "does not match",
                     msg=f"ValidationError not raised with EIN = {bad_ein}",
                 ):
                     validate(instance, schema)
@@ -257,7 +257,7 @@ def test_invalid_zip(self):
 
                     with self.assertRaisesRegex(
                         exceptions.ValidationError,
-                        "is not valid",
+                        "does not match",
                         msg=f"ValidationError not raised with zip = {bad_zip}",
                     ):
                         validate(instance, schema)
@@ -290,7 +290,7 @@ def test_invalid_zip_plus_4(self):
 
                     with self.assertRaisesRegex(
                         exceptions.ValidationError,
-                        "is not valid",
+                        "does not match",
                         msg=f"ValidationError not raised with zip = {bad_zip}",
                     ):
                         validate(instance, schema)
@@ -353,7 +353,7 @@ def test_invalid_phone(self):
 
                     with self.assertRaisesRegex(
                         exceptions.ValidationError,
-                        "is not valid",
+                        "does not match",
                         msg=f"ValidationError not raised with phone = {bad_phone}",
                     ):
                         validate(instance, schema)

backend/audit/test_validators.py

@@ -31,6 +31,8 @@
     MAX_EXCEL_FILE_SIZE_MB,
     validate_additional_ueis_json,
     validate_additional_eins_json,
+    validate_general_information_schema,
+    validate_general_information_schema_rules,
     validate_notes_to_sefa_json,
     validate_corrective_action_plan_json,
     validate_file_content_type,
@@ -994,3 +996,177 @@ def test_error_is_not_gsa_migration_auditor_email(self):
 
         with self.assertRaises(ValidationError):
             validate_general_information_json(gen_info, False)
+
+
+class TestValidateGeneralInformation(SimpleTestCase):
+    def setUp(self):
+        """Set up common test data"""
+        # Example general information that matches the schema
+        self.valid_general_information = {
+            "audit_period_covered": "annual",
+            "auditor_country": "USA",
+            "ein": "123456789",
+            "audit_type": "single-audit",
+            "auditee_uei": "AQDDHJH47DW7",
+            "auditee_zip": "12345",
+            "auditor_ein": "123456789",
+            "auditor_zip": "12345",
+            "auditee_city": "Washington",
+            "auditee_name": "Auditee Name",
+            "auditor_city": "Washington",
+            "is_usa_based": True,
+            "auditee_email": "auditee@email.com",
+            "auditee_phone": "1234567890",
+            "auditee_state": "DC",
+            "auditor_email": "auditor@email.com",
+            "auditor_phone": "1234567890",
+            "auditor_state": "DC",
+            "auditor_country": "USA",
+            "auditor_firm_name": "Auditor Firm Name",
+            "audit_period_covered": "annual",
+            "auditee_contact_name": "Auditee Contact Name",
+            "auditor_contact_name": "Auditor Contact Name",
+            "auditee_contact_title": "Auditee Contact Title",
+            "auditor_contact_title": "Auditor Contact Title",
+            "multiple_eins_covered": False,
+            "multiple_ueis_covered": False,
+            "auditee_address_line_1": "Auditee Address Line 1",
+            "auditor_address_line_1": "Auditor Address Line 1",
+            "met_spending_threshold": True,
+            "secondary_auditors_exist": False,
+            "audit_period_other_months": "",
+            "auditee_fiscal_period_end": "2023-12-31",
+            "ein_not_an_ssn_attestation": False,
+            "auditee_fiscal_period_start": "2023-01-01",
+            "auditor_international_address": "",
+            "user_provided_organization_type": "state",
+            "auditor_ein_not_an_ssn_attestation": False,
+        }
+
+        self.invalid_fiscal_period_end = self.valid_general_information | {
+            "auditee_fiscal_period_end": "Not a date",
+        }
+
+        self.wrong_fiscal_period_end_format = self.valid_general_information | {
+            "auditee_fiscal_period_end": "2023-31-12",
+        }
+
+        self.invalid_auditee_uei = self.valid_general_information | {
+            "auditee_uei": "Invalid",
+        }
+
+        self.invalid_audit_period_other_months = self.valid_general_information | {
+            "audit_period_other_months": "Invalid",
+        }
+
+        self.unexpected_state_and_zip = self.valid_general_information | {
+            "auditor_state": "DC",
+            "auditor_zip": "12345",
+            "auditor_country": "",
+        }
+
+        self.unexpected_audit_period_other_months = self.valid_general_information | {
+            "audit_period_covered": "annual",
+            "audit_period_other_months": "12",
+        }
+
+        self.missing_audit_period_other_months = self.valid_general_information | {
+            "audit_period_covered": "other",
+            "audit_period_other_months": "",
+        }
+
+        self.missing_state_and_zip = self.valid_general_information | {
+            "auditor_state": "",
+            "auditor_zip": "",
+            "auditee_country": "USA",
+        }
+
+        self.invalid_state_and_zip = self.valid_general_information | {
+            "auditor_state": "Not a state",
+            "auditor_zip": "Not a zip",
+        }
+
+        self.invalid_phone = self.valid_general_information | {
+            "auditor_phone": "123-456-789",
+        }
+
+        self.invalid_email = self.valid_general_information | {
+            "auditor_email": "auditor.email.com",
+        }
+
+        self.invalid_audit_period_covered = self.valid_general_information | {
+            "audit_period_covered": "Invalid",
+        }
+
+    def test_validate_general_information_schema_with_valid_data(self):
+        """
+        Test the validation method with valid general information data.
+        """
+        try:
+            validate_general_information_schema(self.valid_general_information)
+        except ValidationError:
+            self.fail(
+                "validate_general_information_schema raised ValidationError unexpectedly!"
+            )
+
+    def test_validate_general_information_schema_with_invalid_data(self):
+        """
+        Test the validation method with invalid general information data.
+        """
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema(self.invalid_fiscal_period_end)
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema(self.wrong_fiscal_period_end_format)
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema(self.invalid_auditee_uei)
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema(self.invalid_audit_period_other_months)
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema(self.invalid_state_and_zip)
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema(self.invalid_phone)
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema(self.invalid_email)
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema(self.invalid_audit_period_covered)
+
+    def test_validate_general_information_schema_rules_with_valid_data(self):
+        """
+        Test the validation method with valid general information data.
+        """
+        try:
+            validate_general_information_schema_rules(self.valid_general_information)
+        except ValidationError:
+            self.fail(
+                "validate_general_information_schema_rules raised ValidationError unexpectedly!"
+            )
+
+    def test_validate_general_information_schema_rules_with_invalid_data(self):
+        """
+        Test the validation method with invalid general information data.
+        """
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema_rules(self.unexpected_state_and_zip)
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema_rules(
+                self.unexpected_audit_period_other_months
+            )
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema_rules(
+                self.missing_audit_period_other_months
+            )
+        with self.assertRaises(ValidationError):
+            validate_general_information_schema_rules(self.missing_state_and_zip)
+
+    def tes_validate_general_information_json(self):
+        """
+        Test the validation method with valid general information data.
+        """
+        try:
+            validate_general_information_json(
+                self.valid_general_information, is_data_migration=False
+            )
+        except ValidationError:
+            self.fail(
+                "validate_general_information_json raised ValidationError unexpectedly!"
+            )