Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feature/remove-is-required-from-constant-validator #163

Merged
merged 1 commit into from
Feb 14, 2022
Merged

feature/remove-is-required-from-constant-validator #163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions cdp_backend/database/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ class Role(Model):
id = fields.IDField()
title = fields.TextField(
required=True,
validator=validators.create_constant_value_validator(RoleTitle, True),
validator=validators.create_constant_value_validator(RoleTitle),
)
person_ref = fields.ReferenceField(Person, required=True, auto_load=False)
body_ref = fields.ReferenceField(Body, auto_load=False)
Expand Down Expand Up @@ -565,9 +565,7 @@ class EventMinutesItem(Model):
)
index = fields.NumberField(required=True)
decision = fields.TextField(
validator=validators.create_constant_value_validator(
EventMinutesItemDecision, False
)
validator=validators.create_constant_value_validator(EventMinutesItemDecision)
)
external_source_id = fields.TextField()

Expand Down Expand Up @@ -619,9 +617,7 @@ class MatterStatus(Model):
event_minutes_item_ref = fields.ReferenceField(EventMinutesItem, auto_load=False)
status = fields.TextField(
required=True,
validator=validators.create_constant_value_validator(
MatterStatusDecision, True
),
validator=validators.create_constant_value_validator(MatterStatusDecision),
)
update_datetime = fields.DateTime(required=True)
external_source_id = fields.TextField()
Expand Down Expand Up @@ -713,7 +709,7 @@ class Vote(Model):
person_ref = fields.ReferenceField(Person, required=True, auto_load=False)
decision = fields.TextField(
required=True,
validator=validators.create_constant_value_validator(VoteDecision, True),
validator=validators.create_constant_value_validator(VoteDecision),
)
in_majority = fields.BooleanField()
external_source_id = fields.TextField()
Expand Down
8 changes: 1 addition & 7 deletions cdp_backend/database/validators.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,18 +156,14 @@ def resource_exists(uri: Optional[str], **kwargs: str) -> bool:
return False


def create_constant_value_validator(
constant_cls: Type, is_required: bool
) -> Callable[[str], bool]:
def create_constant_value_validator(constant_cls: Type) -> Callable[[str], bool]:
"""
Create a validator func that validates a value is one of the valid values.

Parameters
----------
constant_cls: Type
The constant class that contains the valid values.
is_required: bool
Whether the value is required.

Returns
-------
Expand All @@ -189,8 +185,6 @@ def is_valid(value: str) -> bool:
status: bool
The validation status.
"""
if value is None:
return not is_required
return value in get_all_class_attr_values(constant_cls)

return is_valid
14 changes: 4 additions & 10 deletions cdp_backend/tests/database/test_validators.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,21 +136,19 @@ def test_remote_resource_exists(
@pytest.mark.parametrize(
"decision, expected_result",
[
(None, False),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is making me question what the pattern is. Before I approve can you show me code that has a random FireO model with a field thats required fail to push with a None value?

Like I want to make sure that regardless of this change, that None values for required fields are still rejected from upload.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, here's an example code snippet I ran:

matter_status = MatterStatus()
matter_status.matter_ref = Matter.Example()
matter_status.status = None
upload_db_model(matter_status, config.google_credentials_file)

It will raise a FireO RequiredField exception:

Traceback (most recent call last):
  File "/Users/Isaac/Desktop/CDP/cdp-backend/cdp_backend/database/functions.py", line 107, in upload_db_model
    db_model = db_model.upsert(transaction=transaction, batch=batch)
  File "/Users/Isaac/.local/share/virtualenvs/cdp-backend-9V6eR1KV/lib/python3.7/site-packages/fireo/models/model.py", line 352, in upsert
    return self.save(transaction=transaction, batch=batch, merge=True)
  File "/Users/Isaac/.local/share/virtualenvs/cdp-backend-9V6eR1KV/lib/python3.7/site-packages/fireo/models/model.py", line 346, in save
    return self.__class__.collection.create(self, transaction, batch, merge, no_return, **self._get_fields())
  File "/Users/Isaac/.local/share/virtualenvs/cdp-backend-9V6eR1KV/lib/python3.7/site-packages/fireo/managers/managers.py", line 230, in create
    return self.queryset.create(mutable_instance, transaction, batch, merge, no_return, **field_list)
  File "/Users/Isaac/.local/share/virtualenvs/cdp-backend-9V6eR1KV/lib/python3.7/site-packages/fireo/queries/query_set.py", line 56, in create
    return CreateQuery(self.model_cls, mutable_instance, no_return, **kwargs).exec(transaction_or_batch, merge)
  File "/Users/Isaac/.local/share/virtualenvs/cdp-backend-9V6eR1KV/lib/python3.7/site-packages/fireo/queries/create_query.py", line 158, in exec
    return query_wrapper.ModelWrapper.from_query_result(self.model, self._raw_exec(merge=merge))
  File "/Users/Isaac/.local/share/virtualenvs/cdp-backend-9V6eR1KV/lib/python3.7/site-packages/fireo/queries/create_query.py", line 143, in _raw_exec
    ref.set(self._parse_field(), merge=merge)
  File "/Users/Isaac/.local/share/virtualenvs/cdp-backend-9V6eR1KV/lib/python3.7/site-packages/fireo/queries/create_query.py", line 97, in _parse_field
    fv = f.get_value(self.query.get(f.name))
  File "/Users/Isaac/.local/share/virtualenvs/cdp-backend-9V6eR1KV/lib/python3.7/site-packages/fireo/fields/base_field.py", line 122, in get_value
    val = self.field_attribute.parse(val, ignore_required, ignore_default)
  File "/Users/Isaac/.local/share/virtualenvs/cdp-backend-9V6eR1KV/lib/python3.7/site-packages/fireo/fields/field_attribute.py", line 86, in parse
    raise RequiredField(f'"{self.field.__class__.__name__}" is required for model {self.field.model_cls} '
fireo.fields.errors.RequiredField: "TextField" is required for model <class 'cdp_backend.database.models.MatterStatus'> but received no default and no value.

Same thing happens when we don't set status at all

Copy link
Collaborator Author

@isaacna isaacna Feb 10, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If upload fails due to a required field not being present, do we want to fail on the pipeline or keep proceeding? If it's the latter, then I can update the PR to catch RequiredField for uploads.

I'm thinking it's one thing to proceed if validation fails, but if something like a required field is gone, it could be worth stopping the pipeline because that could mean something is wrong with the data source of the ingestion models

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ahhh I think it depends on the case. I wonder if this was why we split them up initially? I will go through this PR and see which ones are needed and which one are truly optional

("Approve", True),
("INVALID", False),
],
)
def test_vote_decision_is_valid(decision: str, expected_result: bool) -> None:
validator_func = validators.create_constant_value_validator(VoteDecision, True)
validator_func = validators.create_constant_value_validator(VoteDecision)
actual_result = validator_func(decision)
assert actual_result == expected_result


@pytest.mark.parametrize(
"decision, expected_result",
[
(None, True),
("Passed", True),
("INVALID", False),
],
Expand All @@ -159,7 +157,7 @@ def test_event_minutes_item_decision_is_valid(
decision: str, expected_result: bool
) -> None:
validator_func = validators.create_constant_value_validator(
EventMinutesItemDecision, False
EventMinutesItemDecision
)
actual_result = validator_func(decision)
assert actual_result == expected_result
Expand All @@ -168,28 +166,24 @@ def test_event_minutes_item_decision_is_valid(
@pytest.mark.parametrize(
"decision, expected_result",
[
(None, False),
("Adopted", True),
("INVALID", False),
],
)
def test_matter_status_decision_is_valid(decision: str, expected_result: bool) -> None:
validator_func = validators.create_constant_value_validator(
MatterStatusDecision, True
)
validator_func = validators.create_constant_value_validator(MatterStatusDecision)
actual_result = validator_func(decision)
assert actual_result == expected_result


@pytest.mark.parametrize(
"title, expected_result",
[
(None, False),
("Councilmember", True),
("INVALID", False),
],
)
def test_role_title_is_valid(title: str, expected_result: bool) -> None:
validator_func = validators.create_constant_value_validator(RoleTitle, True)
validator_func = validators.create_constant_value_validator(RoleTitle)
actual_result = validator_func(title)
assert actual_result == expected_result