Use filtering/sorting from django-ansible-base (ansible#14726)

* Move filtering to DAB

* add comment to trigger building a new image

Signed-off-by: jessicamack <jmack@redhat.com>

* remove unneeded comment

Signed-off-by: jessicamack <jmack@redhat.com>

* remove unused imports

Signed-off-by: jessicamack <jmack@redhat.com>

* change mock import

Signed-off-by: jessicamack <jmack@redhat.com>

---------

Signed-off-by: jessicamack <jmack@redhat.com>
Co-authored-by: jessicamack <jmack@redhat.com>

awx/api/generics.py

@@ -30,12 +30,13 @@
 from rest_framework.renderers import StaticHTMLRenderer
 from rest_framework.negotiation import DefaultContentNegotiation
 
+from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend
+from ansible_base.utils.models import get_all_field_names
+
 # AWX
-from awx.api.filters import FieldLookupBackend
 from awx.main.models import UnifiedJob, UnifiedJobTemplate, User, Role, Credential, WorkflowJobTemplateNode, WorkflowApprovalTemplate
 from awx.main.access import optimize_queryset
 from awx.main.utils import camelcase_to_underscore, get_search_fields, getattrd, get_object_or_400, decrypt_field, get_awx_version
-from awx.main.utils.db import get_all_field_names
 from awx.main.utils.licensing import server_product_name
 from awx.main.views import ApiErrorView
 from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer

awx/api/serializers.py

@@ -43,6 +43,8 @@
 # Django-Polymorphic
 from polymorphic.models import PolymorphicModel
 
+from ansible_base.utils.models import get_type_for_model
+
 # AWX
 from awx.main.access import get_user_capabilities
 from awx.main.constants import ACTIVE_STATES, CENSOR_VALUE
@@ -102,7 +104,6 @@
 from awx.main.models.rbac import role_summary_fields_generator, RoleAncestorEntry
 from awx.main.fields import ImplicitRoleField
 from awx.main.utils import (
-    get_type_for_model,
     get_model_for_type,
     camelcase_to_underscore,
     getattrd,

awx/conf/models.py

@@ -7,8 +7,10 @@
 # Django
 from django.db import models
 
+from ansible_base.utils.models import prevent_search
+
 # AWX
-from awx.main.models.base import CreatedModifiedModel, prevent_search
+from awx.main.models.base import CreatedModifiedModel
 from awx.main.utils import encrypt_field
 from awx.conf import settings_registry
 

awx/main/access.py

@@ -20,11 +20,12 @@
 # Django OAuth Toolkit
 from awx.main.models.oauth import OAuth2Application, OAuth2AccessToken
 
+from ansible_base.utils.validation import to_python_boolean
+
 # AWX
 from awx.main.utils import (
     get_object_or_400,
     get_pk_from_dict,
-    to_python_boolean,
     get_licenser,
 )
 from awx.main.models import (

awx/main/models/__init__.py

@@ -6,8 +6,10 @@
 from django.db import connection
 from django.db.models.signals import pre_delete  # noqa
 
+from ansible_base.utils.models import prevent_search
+
 # AWX
-from awx.main.models.base import BaseModel, PrimordialModel, prevent_search, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES  # noqa
+from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES  # noqa
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded  # noqa
 from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership  # noqa
 from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env  # noqa

awx/main/models/ad_hoc_commands.py

@@ -12,9 +12,11 @@
 from django.utils.translation import gettext_lazy as _
 from django.core.exceptions import ValidationError
 
+from ansible_base.utils.models import prevent_search
+
 # AWX
 from awx.api.versioning import reverse
-from awx.main.models.base import prevent_search, AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty
+from awx.main.models.base import AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty
 from awx.main.models.events import AdHocCommandEvent, UnpartitionedAdHocCommandEvent
 from awx.main.models.unified_jobs import UnifiedJob
 from awx.main.models.notifications import JobNotificationMixin, NotificationTemplate

awx/main/models/base.py

@@ -15,7 +15,6 @@
 from awx.main.constants import CLOUD_PROVIDERS
 
 __all__ = [
-    'prevent_search',
     'VarsDictProperty',
     'BaseModel',
     'CreatedModifiedModel',
@@ -384,23 +383,6 @@ class Meta:
     notification_templates_started = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_started')
 
 
-def prevent_search(relation):
-    """
-    Used to mark a model field or relation as "restricted from filtering"
-    e.g.,
-
-    class AuthToken(BaseModel):
-        user = prevent_search(models.ForeignKey(...))
-        sensitive_data = prevent_search(models.CharField(...))
-
-    The flag set by this function is used by
-    `awx.api.filters.FieldLookupBackend` to block fields and relations that
-    should not be searchable/filterable via search query params
-    """
-    setattr(relation, '__prevent_search__', True)
-    return relation
-
-
 def accepts_json(relation):
     """
     Used to mark a model field as allowing JSON e.g,. JobTemplate.extra_vars

awx/main/models/ha.py

@@ -17,14 +17,16 @@
 import redis
 from solo.models import SingletonModel
 
+from ansible_base.utils.models import prevent_search
+
 # AWX
 from awx import __version__ as awx_application_version
 from awx.main.utils import is_testing
 from awx.api.versioning import reverse
 from awx.main.fields import ImplicitRoleField
 from awx.main.managers import InstanceManager, UUID_DEFAULT
 from awx.main.constants import JOB_FOLDER_PREFIX
-from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
+from awx.main.models.base import BaseModel, HasEditsMixin
 from awx.main.models.rbac import (
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
     ROLE_SINGLETON_SYSTEM_AUDITOR,

awx/main/models/inventory.py

@@ -25,6 +25,8 @@
 # REST Framework
 from rest_framework.exceptions import ParseError
 
+from ansible_base.utils.models import prevent_search
+
 # AWX
 from awx.api.versioning import reverse
 from awx.main.constants import CLOUD_PROVIDERS
@@ -35,7 +37,7 @@
     OrderedManyToManyField,
 )
 from awx.main.managers import HostManager, HostMetricActiveManager
-from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, prevent_search, accepts_json
+from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, accepts_json
 from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
 from awx.main.models.mixins import (

awx/main/models/jobs.py

@@ -20,13 +20,14 @@
 # REST Framework
 from rest_framework.exceptions import ParseError
 
+from ansible_base.utils.models import prevent_search
+
 # AWX
 from awx.api.versioning import reverse
 from awx.main.constants import HOST_FACTS_FIELDS
 from awx.main.models.base import (
     BaseModel,
     CreatedModifiedModel,
-    prevent_search,
     accepts_json,
     JOB_TYPE_CHOICES,
     NEW_JOB_TYPE_CHOICES,

awx/main/models/mixins.py

@@ -16,8 +16,9 @@
 from django.utils.crypto import get_random_string
 from django.utils.translation import gettext_lazy as _
 
+from ansible_base.utils.models import prevent_search
+
 # AWX
-from awx.main.models.base import prevent_search
 from awx.main.models.rbac import Role, RoleAncestorEntry
 from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic
 from awx.main.utils.execution_environments import get_default_execution_environment

awx/main/models/notifications.py

@@ -15,9 +15,11 @@
 from jinja2 import sandbox, ChainableUndefined
 from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
 
+from ansible_base.utils.models import prevent_search
+
 # AWX
 from awx.api.versioning import reverse
-from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel, prevent_search
+from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel
 from awx.main.utils import encrypt_field, decrypt_field, set_environ
 from awx.main.notifications.email_backend import CustomEmailBackend
 from awx.main.notifications.slack_backend import SlackBackend

awx/main/models/unified_jobs.py

@@ -30,8 +30,10 @@
 # Django-Polymorphic
 from polymorphic.models import PolymorphicModel
 
+from ansible_base.utils.models import prevent_search, get_type_for_model
+
 # AWX
-from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel, prevent_search
+from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel
 from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.control import Control as ControlDispatcher
 from awx.main.registrar import activity_stream_registrar
@@ -42,7 +44,6 @@
     _inventory_updates,
     copy_model_by_class,
     copy_m2m_relationships,
-    get_type_for_model,
     parse_yaml_or_json,
     getattr_dne,
     ScheduleDependencyManager,

awx/main/models/workflow.py

@@ -23,9 +23,11 @@
 from jinja2 import sandbox
 from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
 
+from ansible_base.utils.models import prevent_search
+
 # AWX
 from awx.api.versioning import reverse
-from awx.main.models import prevent_search, accepts_json, UnifiedJobTemplate, UnifiedJob
+from awx.main.models import accepts_json, UnifiedJobTemplate, UnifiedJob
 from awx.main.models.notifications import NotificationTemplate, JobNotificationMixin
 from awx.main.models.base import CreatedModifiedModel, VarsDictProperty
 from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR

awx/main/scheduler/task_manager.py

@@ -17,6 +17,8 @@
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 
+from ansible_base.utils.models import get_type_for_model
+
 # AWX
 from awx.main.dispatch.reaper import reap_job
 from awx.main.models import (
@@ -34,7 +36,6 @@
 from awx.main.scheduler.dag_workflow import WorkflowDAG
 from awx.main.utils.pglock import advisory_lock
 from awx.main.utils import (
-    get_type_for_model,
     ScheduleTaskManager,
     ScheduleWorkflowManager,
 )

awx/main/tests/functional/api/test_survey_spec.py

@@ -2,12 +2,12 @@
 import pytest
 import json
 
+from ansible_base.utils.models import get_type_for_model
 
 from awx.api.versioning import reverse
 from awx.main.models.jobs import JobTemplate, Job
 from awx.main.models.activity_stream import ActivityStream
 from awx.main.access import JobTemplateAccess
-from awx.main.utils.common import get_type_for_model
 
 
 @pytest.fixture

awx/main/tests/unit/api/test_filters.py

@@ -3,15 +3,13 @@
 import pytest
 
 # Django
-from django.core.exceptions import FieldDoesNotExist
+from rest_framework.exceptions import PermissionDenied
 
-from rest_framework.exceptions import PermissionDenied, ParseError
+from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend
 
-from awx.api.filters import FieldLookupBackend, OrderByBackend, get_field_from_path
 from awx.main.models import (
     AdHocCommand,
     ActivityStream,
-    Credential,
     Job,
     JobTemplate,
     SystemJob,
@@ -20,88 +18,11 @@
     WorkflowJob,
     WorkflowJobTemplate,
     WorkflowJobOptions,
-    InventorySource,
-    JobEvent,
 )
 from awx.main.models.oauth import OAuth2Application
 from awx.main.models.jobs import JobOptions
 
 
-def test_related():
-    field_lookup = FieldLookupBackend()
-    lookup = '__'.join(['inventory', 'organization', 'pk'])
-    field, new_lookup = field_lookup.get_field_from_lookup(InventorySource, lookup)
-    print(field)
-    print(new_lookup)
-
-
-def test_invalid_filter_key():
-    field_lookup = FieldLookupBackend()
-    # FieldDoesNotExist is caught and converted to ParseError by filter_queryset
-    with pytest.raises(FieldDoesNotExist) as excinfo:
-        field_lookup.value_to_python(JobEvent, 'event_data.task_action', 'foo')
-    assert 'has no field named' in str(excinfo)
-
-
-def test_invalid_field_hop():
-    with pytest.raises(ParseError) as excinfo:
-        get_field_from_path(Credential, 'organization__description__user')
-    assert 'No related model for' in str(excinfo)
-
-
-def test_invalid_order_by_key():
-    field_order_by = OrderByBackend()
-    with pytest.raises(ParseError) as excinfo:
-        [f for f in field_order_by._validate_ordering_fields(JobEvent, ('event_data.task_action',))]
-    assert 'has no field named' in str(excinfo)
-
-
-@pytest.mark.parametrize(u"empty_value", [u'', ''])
-def test_empty_in(empty_value):
-    field_lookup = FieldLookupBackend()
-    with pytest.raises(ValueError) as excinfo:
-        field_lookup.value_to_python(JobTemplate, 'project__name__in', empty_value)
-    assert 'empty value for __in' in str(excinfo.value)
-
-
-@pytest.mark.parametrize(u"valid_value", [u'foo', u'foo,'])
-def test_valid_in(valid_value):
-    field_lookup = FieldLookupBackend()
-    value, new_lookup, _ = field_lookup.value_to_python(JobTemplate, 'project__name__in', valid_value)
-    assert 'foo' in value
-
-
-def test_invalid_field():
-    invalid_field = u"ヽヾ"
-    field_lookup = FieldLookupBackend()
-    with pytest.raises(ValueError) as excinfo:
-        field_lookup.value_to_python(WorkflowJobTemplate, invalid_field, 'foo')
-    assert 'is not an allowed field name. Must be ascii encodable.' in str(excinfo.value)
-
-
-def test_valid_iexact():
-    field_lookup = FieldLookupBackend()
-    value, new_lookup, _ = field_lookup.value_to_python(JobTemplate, 'project__name__iexact', 'foo')
-    assert 'foo' in value
-
-
-def test_invalid_iexact():
-    field_lookup = FieldLookupBackend()
-    with pytest.raises(ValueError) as excinfo:
-        field_lookup.value_to_python(Job, 'id__iexact', '1')
-    assert 'is not a text field and cannot be filtered by case-insensitive search' in str(excinfo.value)
-
-
-@pytest.mark.parametrize('lookup_suffix', ['', 'contains', 'startswith', 'in'])
-@pytest.mark.parametrize('password_field', Credential.PASSWORD_FIELDS)
-def test_filter_on_password_field(password_field, lookup_suffix):
-    field_lookup = FieldLookupBackend()
-    lookup = '__'.join(filter(None, [password_field, lookup_suffix]))
-    with pytest.raises(PermissionDenied) as excinfo:
-        field, new_lookup = field_lookup.get_field_from_lookup(Credential, lookup)
-    assert 'not allowed' in str(excinfo.value)
-
-
 @pytest.mark.parametrize(
     'model, query',
     [
@@ -128,10 +49,3 @@ def test_filter_sensitive_fields_and_relations(model, query):
     with pytest.raises(PermissionDenied) as excinfo:
         field, new_lookup = field_lookup.get_field_from_lookup(model, query)
     assert 'not allowed' in str(excinfo.value)
-
-
-def test_looping_filters_prohibited():
-    field_lookup = FieldLookupBackend()
-    with pytest.raises(ParseError) as loop_exc:
-        field_lookup.get_field_from_lookup(Job, 'job_events__job__job_events')
-    assert 'job_events' in str(loop_exc.value)

awx/main/tests/unit/utils/test_common.py

@@ -12,6 +12,8 @@
 
 from rest_framework.exceptions import ParseError
 
+from ansible_base.utils.models import get_type_for_model
+
 from awx.main.utils import common
 from awx.api.validators import HostnameRegexValidator
 
@@ -106,7 +108,7 @@ def test_set_environ():
 # Cases relied on for scheduler dependent jobs list
 @pytest.mark.parametrize('model,name', TEST_MODELS)
 def test_get_type_for_model(model, name):
-    assert common.get_type_for_model(model) == name
+    assert get_type_for_model(model) == name
 
 
 def test_get_model_for_invalid_type():