Implement REST API versioning and consolidate Job REST API endpoints (n…

…autobot#1521)

* Combine JobViewSet and JobModelViewSet, branch job-list view depending on requested API version

* Update REST API documentation, rework API-Version reporting in responses

* Linting

* More docs updates

* Apply suggestions from code review

Co-authored-by: Jathan McCollum <jathan@gmail.com>

* Improve Swagger documentation of Job API endpoints

* Add JobModel PK to JobClass serializer to ease migration

Co-authored-by: Jathan McCollum <jathan@gmail.com>

nautobot/core/api/versioning.py

@@ -0,0 +1,11 @@
+from django.conf import settings
+from django.utils.translation import gettext_lazy as _
+from rest_framework.versioning import AcceptHeaderVersioning
+
+
+class NautobotAcceptHeaderVersioning(AcceptHeaderVersioning):
+    """Extend the DRF AcceptHeaderVersioning class with a more verbose rejection message."""
+
+    invalid_version_message = _('Invalid version in "Accept" header. Supported versions are %(versions)s') % {
+        "versions": ", ".join(settings.REST_FRAMEWORK["ALLOWED_VERSIONS"])
+    }

nautobot/core/api/views.py

@@ -51,6 +51,20 @@
 #
 
 
+class NautobotAPIVersionMixin:
+    """Add Nautobot-specific handling to the base APIView class."""
+
+    def finalize_response(self, request, response, *args, **kwargs):
+        """Returns the final response object."""
+        response = super().finalize_response(request, response, *args, **kwargs)
+        try:
+            # Add the API version to the response, if available
+            response["API-Version"] = request.version
+        except AttributeError:
+            pass
+        return response
+
+
 class BulkUpdateModelMixin:
     """
     Support bulk modification of objects using the list endpoint for a model. Accepts a PATCH action with a list of one
@@ -197,6 +211,12 @@ def initial(self, request, *args, **kwargs):
         """
         super().initial(request, *args, **kwargs)
 
+        # Django Rest Framework stores the raw API version string e.g. "1.2" as request.version.
+        # For convenience we split it out into integer major/minor versions as well.
+        major, minor = request.version.split(".")
+        request.major_version = int(major)
+        request.minor_version = int(minor)
+
         self.restrict_queryset(request, *args, **kwargs)
 
     def dispatch(self, request, *args, **kwargs):
@@ -212,7 +232,13 @@ def dispatch(self, request, *args, **kwargs):
             return self.finalize_response(request, Response({"detail": msg}, status=409), *args, **kwargs)
 
 
-class ModelViewSet(BulkUpdateModelMixin, BulkDestroyModelMixin, ModelViewSetMixin, ModelViewSet_):
+class ModelViewSet(
+    NautobotAPIVersionMixin,
+    BulkUpdateModelMixin,
+    BulkDestroyModelMixin,
+    ModelViewSetMixin,
+    ModelViewSet_,
+):
     """
     Extend DRF's ModelViewSet to support bulk update and delete functions.
     """
@@ -265,7 +291,7 @@ def perform_destroy(self, instance):
         return super().perform_destroy(instance)
 
 
-class ReadOnlyModelViewSet(ModelViewSetMixin, ReadOnlyModelViewSet_):
+class ReadOnlyModelViewSet(NautobotAPIVersionMixin, ModelViewSetMixin, ReadOnlyModelViewSet_):
     """
     Extend DRF's ReadOnlyModelViewSet to support queryset restriction.
     """
@@ -276,7 +302,7 @@ class ReadOnlyModelViewSet(ModelViewSetMixin, ReadOnlyModelViewSet_):
 #
 
 
-class APIRootView(APIView):
+class APIRootView(NautobotAPIVersionMixin, APIView):
     """
     This is the root of the REST API. API endpoints are arranged by app and model name; e.g. `/api/dcim/sites/`.
     """
@@ -336,7 +362,7 @@ def get(self, request, format=None):
         )
 
 
-class StatusView(APIView):
+class StatusView(NautobotAPIVersionMixin, APIView):
     """
     A lightweight read-only endpoint for conveying the current operational status.
     """
@@ -385,7 +411,7 @@ def get(self, request):
 #
 
 
-class GraphQLDRFAPIView(APIView):
+class GraphQLDRFAPIView(NautobotAPIVersionMixin, APIView):
     """
     API View for GraphQL to integrate properly with DRF authentication mecanism.
     The code is a stripped down version of graphene-django default View

nautobot/core/middleware.py

@@ -97,21 +97,6 @@ def __call__(self, request):
         return response
 
 
-class APIVersionMiddleware(object):
-    """
-    If the request is for an API endpoint, include the API version as a response header.
-    """
-
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        response = self.get_response(request)
-        if is_api_request(request):
-            response["API-Version"] = settings.REST_FRAMEWORK_VERSION
-        return response
-
-
 class ExceptionHandlingMiddleware(object):
     """
     Intercept certain exceptions which are likely indicative of installation issues and provide helpful instructions

nautobot/core/settings.py

@@ -142,8 +142,14 @@
 #
 
 REST_FRAMEWORK_VERSION = VERSION.rsplit(".", 1)[0]  # Use major.minor as API version
+current_major, current_minor = REST_FRAMEWORK_VERSION.split(".")
+# We support all major.minor API versions from 1.2 to the present latest version.
+# This will need to be elaborated upon when we move to version 2.0
+assert current_major == "1", f"REST_FRAMEWORK_ALLOWED_VERSIONS needs to be updated to handle version {current_major}"
+REST_FRAMEWORK_ALLOWED_VERSIONS = [f"{current_major}.{minor}" for minor in range(2, int(current_minor) + 1)]
+
 REST_FRAMEWORK = {
-    "ALLOWED_VERSIONS": [REST_FRAMEWORK_VERSION],
+    "ALLOWED_VERSIONS": REST_FRAMEWORK_ALLOWED_VERSIONS,
     "DEFAULT_AUTHENTICATION_CLASSES": (
         "rest_framework.authentication.SessionAuthentication",
         "nautobot.core.api.authentication.TokenAuthentication",
@@ -156,8 +162,10 @@
         "rest_framework.renderers.JSONRenderer",
         "nautobot.core.api.renderers.FormlessBrowsableAPIRenderer",
     ),
-    "DEFAULT_VERSION": REST_FRAMEWORK_VERSION,
-    "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.AcceptHeaderVersioning",
+    # Version to use if the client doesn't request otherwise.
+    # This should only change (if at all) with Nautobot major (breaking) releases.
+    "DEFAULT_VERSION": "1.2",
+    "DEFAULT_VERSIONING_CLASS": "nautobot.core.api.versioning.NautobotAcceptHeaderVersioning",
     "PAGE_SIZE": None,
     "SCHEMA_COERCE_METHOD_NAMES": {
         # Default mappings
@@ -311,7 +319,6 @@
     "nautobot.core.middleware.ExceptionHandlingMiddleware",
     "nautobot.core.middleware.RemoteUserMiddleware",
     "nautobot.core.middleware.ExternalAuthMiddleware",
-    "nautobot.core.middleware.APIVersionMiddleware",
     "nautobot.core.middleware.ObjectChangeMiddleware",
     "django_prometheus.middleware.PrometheusAfterMiddleware",
 ]

nautobot/docs/additional-features/jobs.md

@@ -451,6 +451,9 @@ The `class_path` is often represented as a string in the format of `<grouping_na
 `local/example/MyJobWithNoVars` or `plugins/nautobot_golden_config.jobs/BackupJob`. Understanding the definitions of these
 elements will be important in running jobs programmatically.
 
+!!! note
+    In Nautobot 1.3 and later, with the addition of Job database models, it is now generally possible and preferable to refer to a job by its UUID primary key, similar to other Nautobot database models, rather than its `class_path`.
+
 ### Via the Web UI
 
 Jobs can be run via the web UI by navigating to the job, completing any required form data (if any), and clicking the "Run Job" button.
@@ -459,19 +462,16 @@ Once a job has been run, the latest [`JobResult`](../models/extras/jobresult.md)
 
 ### Via the API
 
-To run a job via the REST API, issue a POST request to the job's endpoint `/api/extras/jobs/<class_path>/run`. You can optionally provide JSON data to set the `commit` flag, specify any required user input `data`, and/or provide optional scheduling information as described in [the section on scheduling and approvals](./job-scheduling-and-approvals.md).
-
-!!! note
-    [See above](#jobs-and-class_path) for information on constructing the `class_path` for any given Job.
+To run a job via the REST API, issue a POST request to the job's endpoint `/api/extras/jobs/<uuid>/run/`. You can optionally provide JSON data to set the `commit` flag, specify any required user input `data`, and/or provide optional scheduling information as described in [the section on scheduling and approvals](./job-scheduling-and-approvals.md).
 
 For example, to run a job with no user inputs and without committing any anything to the database:
 
 ```no-highlight
 curl -X POST \
 -H "Authorization: Token $TOKEN" \
 -H "Content-Type: application/json" \
--H "Accept: application/json; indent=4" \
-http://nautobot/api/extras/jobs/local/example/MyJobWithNoVars/run/
+-H "Accept: application/json; version=1.3; indent=4" \
+http://nautobot/api/extras/jobs/$JOB_ID/run/
 ```
 
 Or to run a job that expects user inputs, and commit changes to the database:
@@ -480,8 +480,8 @@ Or to run a job that expects user inputs, and commit changes to the database:
 curl -X POST \
 -H "Authorization: Token $TOKEN" \
 -H "Content-Type: application/json" \
--H "Accept: application/json; indent=4" \
-http://nautobot/api/extras/jobs/local/example/MyJobWithVars/run/ \
+-H "Accept: application/json; version=1.3; indent=4" \
+http://nautobot/api/extras/jobs/$JOB_ID/run/ \
 --data '{"data": {"string_variable": "somevalue", "integer_variable": 123}, "commit": true}'
 ```
 
@@ -546,8 +546,8 @@ from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.test import TransactionTestCase
 
-from nautobot.extras.jobs import get_job, run_job
-from nautobot.extras.models import JobResult, JobLogEntry
+from nautobot.extras.jobs import run_job
+from nautobot.extras.models import Job, JobResult, JobLogEntry
 
 
 if "job_logs" in settings.DATABASES:
@@ -561,10 +561,12 @@ class MyJobTestCase(TransactionTestCase):
 
     def test_my_job(self):
         # Testing of Job "MyJob" in file "my_job_file.py" in $JOBS_ROOT
-        job_class = get_job("local/my_job_file/MyJob")
+        job = Job.objects.get(job_class_name="MyJob", module_name="my_job_file", source="local")
+        # or, job = Job.objects.get_for_class_path("local/my_job_file/MyJob")
         job_result = JobResult.objects.create(
-            name=job_class.class_path,
+            name=job.class_path,
             obj_type=ContentType.objects.get(app_label="extras", model="job"),
+            job_model=job,
             job_id=uuid.uuid4(),
         )
         run_job(data={"my_variable": "my_value"}, request=None, commit=False, job_result_pk=job_result.pk)

nautobot/docs/release-notes/version-1.3.md

@@ -28,11 +28,8 @@ Installed Jobs are now represented by a data model in the Nautobot database. Thi
 - The Jobs listing UI view can now be filtered and searched like most other Nautobot table/list views.
 - Job attributes (name, description, approval requirements, etc.) can now be managed via the Nautobot UI by an administrator or user with appropriate permissions to customize or override the attributes defined in the Job source code.
 - Jobs can now be identified by a `slug` as well as by their `class_path`.
-- A new set of REST API endpoints have been added at `api/extras/job-models`. The existing `api/extras/jobs` REST API continues to work but should be considered as deprecated.
-
-!!! warning
-    The new Jobs REST API endpoint URL is likely to change before the final release of Nautobot 1.3.
-
+- A new set of REST API endpoints have been added to `/api/extras/jobs/<uuid>/`. The existing `/api/extras/jobs/<class_path>/` REST API endpoints continue to work but should be considered as deprecated.
+  - A new version of the REST API `/api/extras/jobs/` list endpoint has been implemented as well, but by default this endpoint continues to demonstrate the pre-1.3 behavior unless the REST API client explicitly requests API `version=1.3`. See the section on REST API versioning, below, for more details.
 - As a minor security measure, newly installed Jobs default to `enabled = False`, preventing them from being run until an administrator or user with appropriate permissions updates them to be enabled for running.
 
 !!! note
@@ -52,6 +49,18 @@ A [data model](../models/circuits/providernetwork.md) has been added to support
 
 Python 3.10 is officially supported by Nautobot now, and we are building and publishing Docker images with Python 3.10 now.
 
+#### REST API Versioning ([#1465](https://github.com/nautobot/nautobot/issues/1465))
+
+Nautobot's REST API now supports multiple versions, which may requested by modifying the HTTP Accept header on any requests sent by a REST API client. Details are in the [REST API documentation](../rest-api/overview.md#versioning), but in brief:
+
+- The only REST API endpoint that is versioned in the 1.3.0 release is the `/api/extras/jobs/` listing endpoint, as described above. All others are currently un-versioned. However, over time more versioned REST APIs will be developed, so this is important to understand for all REST API consumers.
+- If a REST API client does not request a specific REST API version (in other words, requests `Accept: application/json` rather than `Accept: application/json; version=1.3`) the API behavior will be compatible with Nautobot 1.2, at a minimum for the remainder of the Nautobot 1.x release cycle.
+- The API behavior may change to a newer default version in a Nautobot major release (e.g. 2.0).
+- To request an updated (non-backwards-compatible) API endpoint, an API version must be requested corresponding at a minimum to the Nautobot `major.minor` version where the updated API endpoint was introduced (so to interact with the new Jobs REST API, `Accept: application/json; version=1.3`).
+
+!!! tip
+    As a best practice, when developing a Nautobot REST API integration, your client should _always_ request the current API version it is being developed against, rather than relying on the default API behavior (which may change with a new Nautobot major release, as noted, and which also may not include the latest and greatest API endpoints already available but not yet made default in the current release).
+
 ### Changed
 
 #### Update Jinja2 to 3.x ([#1474](https://github.com/nautobot/nautobot/pull/1474))