diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 28f36f131e..b2a3a1e95e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,19 +1,62 @@ { - "dockerComposeFile": ["../development/docker-compose.yml", "../development/docker-compose.debug.yml"], + "name": "Nautobot Dev Container", + "dockerComposeFile": [ + "../development/docker-compose.yml", + "../development/docker-compose.postgres.yml", + "../development/docker-compose.dev.yml" + ], + "features": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { } + }, "service": "nautobot", - "workspaceFolder": "/source/", - "extensions": [ - "ms-python.vscode-pylance", - "streetsidesoftware.code-spell-checker", - "eamodio.gitlens", - "oderwat.indent-rainbow", - "ms-python.python", - "ms-vsliveshare.vsliveshare" - ], - "settings": { - "python.pythonPath": "/usr/local/bin/python", - "python.analysis.extraPaths": ["/source"], - "python.linting.pylintEnabled": true, - "python.linting.enabled": true - } + "shutdownAction": "stopCompose", + "customizations": { + "vscode": { + "extensions": [ + "charliermarsh.ruff", + "DavidAnson.vscode-markdownlint", + "eamodio.gitlens", + "EditorConfig.EditorConfig", + "GitHub.vscode-pull-request-github", + "ms-python.python", + "ms-python.vscode-pylance", + "samuelcolvin.jinjahtml", + "tamasfe.even-better-toml" + ], + "settings": { + // Global editor settings + "editor.codeActionsOnSave": { + "source.fixAll.markdownlint": "explicit" + }, + "editor.formatOnPaste": true, + "editor.formatOnSave": true, + // JSON settings + "json.format.keepLines": true, + "workbench.settings.useSplitJSON": false, + // Markdown settings + "[markdown]": { + "editor.defaultFormatter": "DavidAnson.vscode-markdownlint" + }, + // Python settings + "[python]": { + "editor.codeActionsOnSave": { + "source.fixAll": "explicit", + "source.organizeImports": "explicit" + }, + // When rebuilding the devcontainer, `Developer: Reload Window` must be + // executed for the Ruff formatter to be recognized by the settings. + // See: https://github.com/microsoft/vscode/issues/189839 + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true + }, + "python.analysis.extraPaths": [ + "${workspaceFolder}" + ], + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.pythonPath": "/usr/local/bin/python" + } + } + }, + "workspaceFolder": "/source", + "postCreateCommand": "bash .devcontainer/postCreateCommand.sh" } diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh new file mode 100644 index 0000000000..343175ce4b --- /dev/null +++ b/.devcontainer/postCreateCommand.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -ex + +git config --global --add safe.directory /source diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..6e2fb00472 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +# EditorConfig is awesome: https://EditorConfig.org + +root = true + +# All +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{yaml,yml}] +indent_size = 2 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 37426c4ac7..b727ca9b0a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,18 +7,19 @@ Please indicate the relevant feature request or bug report below. --> -# Closes: # +# Closes # # What's Changed +# Screenshots + - - # TODO +## v1.6.10 (2024-01-22) + +### Security + +- [#5109](https://github.com/nautobot/nautobot/issues/5109) - Removed `/files/get/` URL endpoint (for viewing FileAttachment files in the browser), as it was unused and could potentially pose security issues. +- [#5134](https://github.com/nautobot/nautobot/issues/5134) - Fixed an XSS vulnerability ([GHSA-v4xv-795h-rv4h](https://github.com/nautobot/nautobot/security/advisories/GHSA-v4xv-795h-rv4h)) in the `render_markdown()` utility function used to render comments, notes, job log entries, etc. + +### Added + +- [#5134](https://github.com/nautobot/nautobot/issues/5134) - Enhanced Markdown-supporting fields (`comments`, `description`, Notes, Job log entries, etc.) to also permit the use of a limited subset of "safe" HTML tags and attributes. + +### Changed + +- [#5132](https://github.com/nautobot/nautobot/issues/5132) - Updated poetry version for development Docker image to match 2.0. + +### Dependencies + +- [#5087](https://github.com/nautobot/nautobot/issues/5087) - Updated GitPython to version 3.1.41 to address Windows security vulnerability [GHSA-2mqj-m65w-jghx](https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-2mqj-m65w-jghx). +- [#5087](https://github.com/nautobot/nautobot/issues/5087) - Updated Jinja2 to version 3.1.3 to address to address XSS security vulnerability [GHSA-h5c8-rqwp-cp95](https://github.com/pallets/jinja/security/advisories/GHSA-h5c8-rqwp-cp95). +- [#5134](https://github.com/nautobot/nautobot/issues/5134) - Added `nh3` HTML sanitization library as a dependency. + +## v1.6.9 (2024-01-08) + +### Fixed + +- [#5042](https://github.com/nautobot/nautobot/issues/5042) - Fixed early return conditional in `ensure_git_repository`. + +## v1.6.8 (2023-12-21) + +### Security + +- [#4876](https://github.com/nautobot/nautobot/issues/4876) - Updated `cryptography` to `41.0.7` due to CVE-2023-49083. As this is not a direct dependency of Nautobot, it will not auto-update when upgrading. Please be sure to upgrade your local environment. +- [#4988](https://github.com/nautobot/nautobot/issues/4988) - Fixed missing object-level permissions enforcement when running a JobButton ([GHSA-vf5m-xrhm-v999](https://github.com/nautobot/nautobot/security/advisories/GHSA-vf5m-xrhm-v999)). +- [#4988](https://github.com/nautobot/nautobot/issues/4988) - Removed the requirement for users to have both `extras.run_job` and `extras.run_jobbutton` permissions to run a Job via a Job Button. Only `extras.run_job` permission is now required. +- [#5002](https://github.com/nautobot/nautobot/issues/5002) - Updated `paramiko` to `3.4.0` due to CVE-2023-48795. As this is not a direct dependency of Nautobot, it will not auto-update when upgrading. Please be sure to upgrade your local environment. + +### Added + +- [#4965](https://github.com/nautobot/nautobot/issues/4965) - Added MMF OM5 cable type to cable type choices. + +### Removed + +- [#4988](https://github.com/nautobot/nautobot/issues/4988) - Removed redundant `/extras/job-button//run/` URL endpoint; Job Buttons now use `/extras/jobs//run/` endpoint like any other job. + +### Fixed + +- [#4977](https://github.com/nautobot/nautobot/issues/4977) - Fixed early return conditional in `ensure_git_repository`. + +### Housekeeping + +- [#4988](https://github.com/nautobot/nautobot/issues/4988) - Fixed some bugs in `example_plugin.jobs.ExampleComplexJobButtonReceiver`. + +## v1.6.7 (2023-12-12) + +### Security + +- [#4959](https://github.com/nautobot/nautobot/issues/4959) - Enforce authentication and object permissions on DB file storage views ([GHSA-75mc-3pjc-727q](https://github.com/nautobot/nautobot/security/advisories/GHSA-75mc-3pjc-727q)). + +### Added + +- [#4873](https://github.com/nautobot/nautobot/issues/4873) - Added QSFP112 interface type to interface type choices. + +### Removed + +- [#4797](https://github.com/nautobot/nautobot/issues/4797) - Removed erroneous `custom_fields` decorator from InterfaceRedundancyGroupAssociation as it's not a supported feature for this model. +- [#4857](https://github.com/nautobot/nautobot/issues/4857) - Removed Jathan McCollum as a point of contact in `SECURITY.md`. + +### Fixed + +- [#4142](https://github.com/nautobot/nautobot/issues/4142) - Fixed unnecessary git operations when calling `ensure_git_repository` while the desired commit is already checked out. +- [#4917](https://github.com/nautobot/nautobot/issues/4917) - Fixed slow performance on location hierarchy html template. +- [#4921](https://github.com/nautobot/nautobot/issues/4921) - Fixed inefficient queries in `Location.base_site`. + +## v1.6.6 (2023-11-21) + +### Security + +- [#4833](https://github.com/nautobot/nautobot/issues/4833) - Fixed cross-site-scripting (XSS) potential with maliciously crafted Custom Links, Computed Fields, and Job Buttons (GHSA-cf9f-wmhp-v4pr). + +### Changed + +- [#4833](https://github.com/nautobot/nautobot/issues/4833) - Changed the `render_jinja2()` API to no longer automatically call `mark_safe()` on the output. + +### Fixed + +- [#3179](https://github.com/nautobot/nautobot/issues/3179) - Fixed the error that occurred when fetching the API response for CircuitTermination with a cable connected to CircuitTermination, FrontPort, or RearPort. +- [#4799](https://github.com/nautobot/nautobot/issues/4799) - Reduced size of Nautobot `sdist` and `wheel` packages from 69 MB to 29 MB. + +### Dependencies + +- [#4799](https://github.com/nautobot/nautobot/issues/4799) - Updated `mkdocs` development dependency to `1.5.3`. + +### Housekeeping + +- [#4799](https://github.com/nautobot/nautobot/issues/4799) - Updated docs configuration for `examples/example_plugin`. +- [#4833](https://github.com/nautobot/nautobot/issues/4833) - Added `ruff` to invoke tasks and CI. + ## v1.6.5 (2023-11-13) ### Security diff --git a/nautobot/docs/release-notes/version-2.1.md b/nautobot/docs/release-notes/version-2.1.md index 6b1499f5bc..a15f920adf 100644 --- a/nautobot/docs/release-notes/version-2.1.md +++ b/nautobot/docs/release-notes/version-2.1.md @@ -72,6 +72,76 @@ Support for versions of PostgreSQL prior to 12.0 has been removed as these versi Support for `HIDE_RESTRICTED_UI` has been removed. UI elements requiring specific permissions will now always be hidden from users lacking those permissions. Additionally, users not logged in will now be automatically redirected to the login page. +## v2.1.2 (2024-01-22) + +### Security + +- [#5054](https://github.com/nautobot/nautobot/issues/5054) - Added validation of redirect URLs to the "Add a new IP Address" and "Assign an IP Address" views. +- [#5109](https://github.com/nautobot/nautobot/issues/5109) - Removed `/files/get/` URL endpoint (for viewing FileAttachment files in the browser), as it was unused and could potentially pose security issues. +- [#5133](https://github.com/nautobot/nautobot/issues/5133) - Fixed an XSS vulnerability ([GHSA-v4xv-795h-rv4h](https://github.com/nautobot/nautobot/security/advisories/GHSA-v4xv-795h-rv4h)) in the `render_markdown()` utility function used to render comments, notes, job log entries, etc. + +### Added + +- [#3877](https://github.com/nautobot/nautobot/issues/3877) - Added global filtering to Job Result log table, enabling search across all pages. +- [#5102](https://github.com/nautobot/nautobot/issues/5102) - Enhanced the `sanitize` function to also handle sanitization of lists and tuples of strings. +- [#5133](https://github.com/nautobot/nautobot/issues/5133) - Enhanced Markdown-supporting fields (`comments`, `description`, Notes, Job log entries, etc.) to also permit the use of a limited subset of "safe" HTML tags and attributes. + +### Changed + +- [#5102](https://github.com/nautobot/nautobot/issues/5102) - Changed the `nautobot-server runjob` management command to check whether the requested user has permission to run the requested job. +- [#5102](https://github.com/nautobot/nautobot/issues/5102) - Changed the `nautobot-server runjob` management command to check whether the requested job is installed and enabled. +- [#5102](https://github.com/nautobot/nautobot/issues/5102) - Changed the `nautobot-server runjob` management command to check whether a Celery worker is running when invoked without the `--local` flag. +- [#5131](https://github.com/nautobot/nautobot/issues/5131) - Improved the performance of the `/api/dcim/locations/` REST API. + +### Removed + +- [#5078](https://github.com/nautobot/nautobot/issues/5078) - Removed `nautobot-server startplugin` management command. + +### Fixed + +- [#4075](https://github.com/nautobot/nautobot/issues/4075) - Fixed sorting of Device Bays list view by installed device status. +- [#4444](https://github.com/nautobot/nautobot/issues/4444) - Fixed Sync Git Repository requires non-matching permissions for UI vs API. +- [#4998](https://github.com/nautobot/nautobot/issues/4998) - Fixed inability to import CSVs where later rows include references to records defined by earlier rows. +- [#5024](https://github.com/nautobot/nautobot/issues/5024) - Improved performance of the Job Result list view by optimizing the way JobLogEntry records are queried. +- [#5024](https://github.com/nautobot/nautobot/issues/5024) - Improved performance of the Device list view by including the manufacturer name in the table queryset. +- [#5024](https://github.com/nautobot/nautobot/issues/5024) - Improved performance of most ObjectListViews by optimizing how Custom fields, Computed fields, and Relationships are queried. +- [#5024](https://github.com/nautobot/nautobot/issues/5024) - Fixed a bug that caused IPAddress objects to query their parent Prefix and Namespace every time they were instantiated. +- [#5024](https://github.com/nautobot/nautobot/issues/5024) - Improved performance of the IPAddress list view by including the namespace in the table queryset. +- [#5024](https://github.com/nautobot/nautobot/issues/5024) - Updated bulk-edit and bulk-delete views to auto-hide any "actions" column in the table of objects being edited or deleted. +- [#5031](https://github.com/nautobot/nautobot/issues/5031) - Updated the default sanitizer pattern to include secret(s) and to be flexible with python dictionaries. +- [#5043](https://github.com/nautobot/nautobot/issues/5043) - Fixed early return conditional in `ensure_git_repository`. +- [#5045](https://github.com/nautobot/nautobot/issues/5045) - Adjusted Bootstrap grid breakpoints to account for the space occupied by the sidebar, fixing various page rendering. +- [#5054](https://github.com/nautobot/nautobot/issues/5054) - Fixed missing search logic on the "Assign an IP Address" view. +- [#5058](https://github.com/nautobot/nautobot/issues/5058) - Changed filter query parameters from `location_id` to `location` in `virtualization/forms.py`. +- [#5081](https://github.com/nautobot/nautobot/issues/5081) - Fixed core.tables.BaseTable to terminate dynamic queryset's building of pre-fetched fields upon first non-RelatedField of a column. +- [#5095](https://github.com/nautobot/nautobot/issues/5095) - Fixed a couple of potential `KeyError` when refreshing Git repository Jobs. +- [#5095](https://github.com/nautobot/nautobot/issues/5095) - Fixed color highlighting of `error` and `critical` log entries when viewing a Job Result. +- [#5102](https://github.com/nautobot/nautobot/issues/5102) - Fixed missing log messages when errors occur during `Job.__call__()` initial setup. +- [#5102](https://github.com/nautobot/nautobot/issues/5102) - Fixed misleading "Job completed" message from being logged when a Job aborted. +- [#5102](https://github.com/nautobot/nautobot/issues/5102) - Fixed an error in `nautobot-server runjob` if a job returned data other than a dict. +- [#5102](https://github.com/nautobot/nautobot/issues/5102) - Fixed misleading "SUCCESS" message when `nautobot-server runjob` resulted in any JobResult status other than "FAILED". +- [#5102](https://github.com/nautobot/nautobot/issues/5102) - Fixed incorrect JobResult data when using `nautobot-server runjob --local` or `JobResult.execute_job()`. +- [#5111](https://github.com/nautobot/nautobot/issues/5111) - Fixed rack group and rack filtering by the location selected in the device bulk edit form. + +### Dependencies + +- [#5083](https://github.com/nautobot/nautobot/issues/5083) - Updated GitPython to version 3.1.41 to address Windows security vulnerability [GHSA-2mqj-m65w-jghx](https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-2mqj-m65w-jghx). +- [#5086](https://github.com/nautobot/nautobot/issues/5086) - Updated Jinja2 to version 3.1.3 to address to address XSS security vulnerability [GHSA-h5c8-rqwp-cp95](https://github.com/pallets/jinja/security/advisories/GHSA-h5c8-rqwp-cp95). +- [#5133](https://github.com/nautobot/nautobot/issues/5133) - Added `nh3` HTML sanitization library as a dependency. + +### Documentation + +- [#5078](https://github.com/nautobot/nautobot/issues/5078) - Added a link to the `cookiecutter-nautobot-app` project in the App developer documentation. + +### Housekeeping + +- [#4906](https://github.com/nautobot/nautobot/issues/4906) - Added automatic superuser creation environment variables to docker development environment. +- [#4906](https://github.com/nautobot/nautobot/issues/4906) - Updated VS Code Dev Containers configuration and documentation. +- [#5076](https://github.com/nautobot/nautobot/issues/5076) - Updated `packaging` dependency to permit newer versions since it follows CalVer rather than SemVer. +- [#5079](https://github.com/nautobot/nautobot/issues/5079) - Increased overly-brief `start_period` for development `nautobot` container to allow sufficient time for initial migrations to run. +- [#5079](https://github.com/nautobot/nautobot/issues/5079) - Fixed bug with invoke cli and invoke nbshell. +- [#5118](https://github.com/nautobot/nautobot/issues/5118) - Updated PR template to encourage inclusion of screenshots. + ## v2.1.1 (2024-01-08) ### Added diff --git a/nautobot/docs/requirements.txt b/nautobot/docs/requirements.txt index 75002af484..1b89d21abc 100644 --- a/nautobot/docs/requirements.txt +++ b/nautobot/docs/requirements.txt @@ -1,4 +1,4 @@ -Jinja2==3.1.2 +Jinja2==3.1.3 Markdown==3.3.7 markdown-data-tables==1.0.0 mkdocs==1.5.3 diff --git a/nautobot/docs/user-guide/administration/configuration/optional-settings.md b/nautobot/docs/user-guide/administration/configuration/optional-settings.md index 06c4667573..3b0fff6975 100644 --- a/nautobot/docs/user-guide/administration/configuration/optional-settings.md +++ b/nautobot/docs/user-guide/administration/configuration/optional-settings.md @@ -883,12 +883,35 @@ Default: ```python [ (re.compile(r"(https?://)?\S+\s*@", re.IGNORECASE), r"\1{replacement}@"), - (re.compile(r"(username|password|passwd|pwd)((?:\s+is.?|:)?\s+)\S+", re.IGNORECASE), r"\1\2{replacement}"), + ( + re.compile(r"(username|password|passwd|pwd|secret|secrets)([\"']?(?:\s+is.?|:)?\s+)\S+[\"']?", re.IGNORECASE), + r"\1\2{replacement}", + ), ] ``` List of (regular expression, replacement pattern) tuples used by the `nautobot.core.utils.logging.sanitize()` function. As of Nautobot 1.3.4 this function is used primarily for sanitization of Job log entries, but it may be used in other scopes in the future. +This pattern catches patterns such as: + +| Pattern Match Examples | +| --- | +| Password is1234 | +| Password: is1234 | +| Password is: is1234 | +| Password is is1234 | +| secret is: is1234 | +| secret is is1234 | +| secrets is: is1234 | +| secrets is is1234 | +| {"username": "is1234"} | +| {"password": "is1234"} | +| {"secret": "is1234"} | +| {"secrets": "is1234"} | + +!!! info + is1234 would be replaced in the Job logs with `(redacted)`. + --- ## STORAGE_BACKEND @@ -941,7 +964,7 @@ If set to `False`, unknown/unrecognized filter parameters will be discarded and Default: `""` -A message to include on error pages (status code 403, 404, 500, etc.) when an error occurs. You can configure this to direct users to the appropriate contact(s) within your organization that provide support for Nautobot. Markdown formatting is supported within this message (raw HTML is not). +A message to include on error pages (status code 403, 404, 500, etc.) when an error occurs. You can configure this to direct users to the appropriate contact(s) within your organization that provide support for Nautobot. Markdown formatting is supported within this message, as well as [a limited subset of HTML](../../platform-functionality/template-filters.md#render_markdown). If unset, the default message that will appear is `If further assistance is required, please join the #nautobot channel on [Network to Code's Slack community](https://slack.networktocode.com) and post your question.` diff --git a/nautobot/docs/user-guide/administration/tools/nautobot-server.md b/nautobot/docs/user-guide/administration/tools/nautobot-server.md index 053865cc28..5a42b78481 100644 --- a/nautobot/docs/user-guide/administration/tools/nautobot-server.md +++ b/nautobot/docs/user-guide/administration/tools/nautobot-server.md @@ -774,23 +774,6 @@ spawned uWSGI worker 5 (pid: 112171, cores: 3) Please see the guide on [Deploying Nautobot Services](../installation/services.md) for our recommended configuration for use with uWSGI. -### `startplugin` - -`nautobot-server startplugin [directory]` - -Create a new plugin with `name`. - -This command is similar to the django-admin [startapp](https://docs.djangoproject.com/en/stable/ref/django-admin/#startapp) command, but with a default template directory (`--template`) of `nautobot/core/templates/plugin_template`. This command assists with creating a basic file structure for beginning development of a new Nautobot plugin. - -Without passing in the destination directory, `nautobot-server startplugin` will use your current directory and the `name` argument provided to create a new directory. We recommend providing a directory so that the plugin can be installed or published easily. Here is an example: - -```no-highlight -mkdir -p ~/myplugin/myplugin -nautobot-server startplugin myplugin ~/myplugin/myplugin -``` - -Additional options such as `--name` or `--extension` can be found in the Django [documentation](https://docs.djangoproject.com/en/stable/ref/django-admin/#startapp). - ### `trace_paths` `nautobot-server trace_paths` diff --git a/nautobot/docs/user-guide/platform-functionality/note.md b/nautobot/docs/user-guide/platform-functionality/note.md index d97e14214c..5665b3b103 100644 --- a/nautobot/docs/user-guide/platform-functionality/note.md +++ b/nautobot/docs/user-guide/platform-functionality/note.md @@ -4,4 +4,4 @@ Notes provide a place for you to store notes or general information on an object, such as a Device, that may not require a specific field for. This could be a note on a recent upgrade, a warning about a problematic device, or the reason the Rack was marked with the Status `Retired`. -The note field supports [Markdown Basic Syntax](https://www.markdownguide.org/cheat-sheet/#basic-syntax). +The note field supports [Markdown Basic Syntax](https://www.markdownguide.org/cheat-sheet/#basic-syntax) as well as [a limited subset of HTML](template-filters.md#render_markdown). diff --git a/nautobot/docs/user-guide/platform-functionality/template-filters.md b/nautobot/docs/user-guide/platform-functionality/template-filters.md index c210e4c46f..785378a3d9 100644 --- a/nautobot/docs/user-guide/platform-functionality/template-filters.md +++ b/nautobot/docs/user-guide/platform-functionality/template-filters.md @@ -216,12 +216,86 @@ Render a dictionary as formatted JSON. ### render_markdown -Render text as Markdown. +Render and sanitize Markdown text into HTML. A limited subset of HTML tags and attributes are permitted in the text as well; non-permitted HTML will be stripped from the output for security. ```django {{ text | render_markdown }} ``` +#### Permitted HTML Tags and Attributes + ++++ 2.1.2 + +The set of permitted HTML tags is defined in `nautobot.core.constants.HTML_ALLOWED_TAGS`, and their permitted attributes are defined in `nautobot.core.constants.HTML_ALLOWED_ATTRIBUTES`. As of Nautobot 2.1.2 the following are permitted: + +??? info "Full list of HTML tags and attributes" + | Tag | Attributes | + | -------------- | -------------------------------------------------------------------- | + | `` | `href`, `hreflang` | + | `` | | + | `` | | + | `` | | + | `` | | + | `` | `dir` | + | `
` | `cite` | + | `
` | | + | `` | | + | `
` | | + | `` | | + | `` | | + | `` | `align`, `char`, `charoff`, `span` | + | `` | `align`, `char`, `charoff`, `span` | + | `
` | | + | `` | `cite`, `datetime` | + | `
` | | + | `
` | | + | `
` | | + | `
` | | + | `` | | + | `

` | | + | `

` | | + | `

` | | + | `

` | | + | `

` | | + | `
` | | + | `
` | | + | `
` | `align`, `size`, `width` | + | `` | | + | `` | `align`, `alt`, `height`, `src`, `width` | + | `` | `cite`, `datetime` | + | `` | | + | `
  • ` | | + | `` | | + | `
      ` | `start` | + | `

      ` | | + | `

      `        |                                                                      |
      +    | ``          | `cite`                                                               |
      +    | ``         |                                                                      |
      +    | ``         |                                                                      |
      +    | ``        |                                                                      |
      +    | ``       |                                                                      |
      +    | ``          |                                                                      |
      +    | ``       |                                                                      |
      +    | ``      |                                                                      |
      +    | ``       |                                                                      |
      +    | ``     |                                                                      |
      +    | ``     |                                                                      |
      +    | ``        |                                                                      |
      +    | ``    |                                                                      |
      +    | ``        |                                                                      |
      +    | ``      | `align`, `char`, `charoff`, `summary`                                |
      +    | ``      | `align`, `char`, `charoff`                                           |
      +    | ``      | `align`, `char`, `charoff`                                           |
      +    | ``         | `align`, `char`, `charoff`                                           |
      +    | ``         |                                                                      |
      +    | ``          |                                                                      |
      +    | `
        ` | | + | `` | | + | `` | | + ### render_yaml Render a dictionary as formatted YAML. diff --git a/nautobot/extras/api/views.py b/nautobot/extras/api/views.py index 12490bc5ae..920d365f4b 100644 --- a/nautobot/extras/api/views.py +++ b/nautobot/extras/api/views.py @@ -375,7 +375,10 @@ class GitRepositoryViewSet(NautobotModelViewSet): filterset_class = filters.GitRepositoryFilterSet @extend_schema(methods=["post"], request=serializers.GitRepositorySerializer) - @action(detail=True, methods=["post"]) + # Since we are explicitly checking for `extras:change_gitrepository` in the API sync() method + # We explicitly set the permission_classes to IsAuthenticated in the @action decorator + # bypassing the default DRF permission check for `extras:add_gitrepository` and the permission check fall through to the function itself. + @action(detail=True, methods=["post"], permission_classes=[IsAuthenticated]) def sync(self, request, pk): """ Enqueue pull git repository and refresh data. diff --git a/nautobot/extras/datasources/git.py b/nautobot/extras/datasources/git.py index ffe363cce3..056e9fc108 100644 --- a/nautobot/extras/datasources/git.py +++ b/nautobot/extras/datasources/git.py @@ -153,10 +153,9 @@ def ensure_git_repository(repository_record, logger=None, head=None): # pylint: if head is not None: # If the repo exists and has HEAD already checked out, the repo is present and has the correct branch selected. with suppress(InvalidGitRepositoryError): - if ( - Path(repository_record.filesystem_path).exists() - and Repo(repository_record.filesystem_path).rev_parse("HEAD") == head - ): + if Path(repository_record.filesystem_path).exists() and str( + Repo(repository_record.filesystem_path).rev_parse("HEAD") + ) == str(head): return False from_url, to_path, from_branch = get_repo_from_url_to_path_and_from_branch(repository_record) @@ -743,14 +742,15 @@ def refresh_code_from_repository(repository_slug, consumer=None, skip_reimport=F logger.debug("Unloading module %s", module_name) if module_name in app.loader.task_modules: app.loader.task_modules.remove(module_name) - del sys.modules[module_name] + if module_name in sys.modules: + del sys.modules[module_name] # Unregister any previous Celery tasks from this module for task_name in list(app.tasks): if task_name.startswith(f"{repository_slug}."): logger.debug("Unregistering Celery task %s", task_name) app.tasks.unregister(task_name) - if consumer is not None: + if consumer is not None and task_name in consumer.strategies: del consumer.strategies[task_name] if not skip_reimport: diff --git a/nautobot/extras/filters/mixins.py b/nautobot/extras/filters/mixins.py index 037109b380..e5fa23c1f1 100644 --- a/nautobot/extras/filters/mixins.py +++ b/nautobot/extras/filters/mixins.py @@ -14,7 +14,6 @@ ) from nautobot.dcim.models import Device from nautobot.extras.choices import ( - CustomFieldFilterLogicChoices, CustomFieldTypeChoices, RelationshipSideChoices, ) @@ -70,9 +69,7 @@ def __init__(self, *args, **kwargs): CustomFieldTypeChoices.TYPE_SELECT: CustomFieldMultiSelectFilter, } - custom_fields = CustomField.objects.filter( - content_types=ContentType.objects.get_for_model(self._meta.model) - ).exclude(filter_logic=CustomFieldFilterLogicChoices.FILTER_DISABLED) + custom_fields = CustomField.objects.get_for_model(self._meta.model, exclude_filter_disabled=True) for cf in custom_fields: # Determine filter class for this CustomField type, default to CustomFieldCharFilter new_filter_name = cf.add_prefix_to_cf_key() @@ -235,16 +232,13 @@ def _append_relationships(self, model): """ Append form fields for all Relationships assigned to this model. """ - query = Q(source_type=self.obj_type, source_hidden=False) | Q( - destination_type=self.obj_type, destination_hidden=False - ) - relationships = Relationship.objects.select_related("source_type", "destination_type").filter(query) - - for rel in relationships.iterator(): - if rel.source_type == self.obj_type and not rel.source_hidden: - self._append_relationships_side([rel], RelationshipSideChoices.SIDE_SOURCE, model) - if rel.destination_type == self.obj_type and not rel.destination_hidden: - self._append_relationships_side([rel], RelationshipSideChoices.SIDE_DESTINATION, model) + src_relationships, dst_relationships = Relationship.objects.get_for_model(model=model, hidden=False) + + for rel in src_relationships: + self._append_relationships_side([rel], RelationshipSideChoices.SIDE_SOURCE, model) + + for rel in dst_relationships: + self._append_relationships_side([rel], RelationshipSideChoices.SIDE_DESTINATION, model) def _append_relationships_side(self, relationships, initial_side, model): """ diff --git a/nautobot/extras/forms/forms.py b/nautobot/extras/forms/forms.py index 4c979f6e70..9b59fea959 100644 --- a/nautobot/extras/forms/forms.py +++ b/nautobot/extras/forms/forms.py @@ -368,6 +368,12 @@ class ConfigContextSchemaFilterForm(BootstrapMixin, forms.Form): ) +class CustomFieldDescriptionField(CommentField): + @property + def default_helptext(self): + return "Also used as the help text when editing models using this custom field.
        " + super().default_helptext + + class CustomFieldForm(BootstrapMixin, forms.ModelForm): label = forms.CharField(required=True, max_length=50, help_text="Name of the field as displayed to users.") key = SlugField( @@ -376,11 +382,9 @@ class CustomFieldForm(BootstrapMixin, forms.ModelForm): slug_source="label", help_text="Internal name of this field. Please use underscores rather than dashes.", ) - description = forms.CharField( + description = CustomFieldDescriptionField( + label="Description", required=False, - help_text="Also used as the help text when editing models using this custom field.
        " - '' - "Markdown syntax is supported.", ) content_types = MultipleContentTypeField( feature="custom_fields", help_text="The object(s) to which this field applies." @@ -1098,7 +1102,7 @@ class JobButtonFilterForm(BootstrapMixin, forms.Form): class NoteForm(BootstrapMixin, forms.ModelForm): - note = CommentField + note = CommentField() class Meta: model = Note diff --git a/nautobot/extras/forms/mixins.py b/nautobot/extras/forms/mixins.py index 29774fa5a6..dd32585f60 100644 --- a/nautobot/extras/forms/mixins.py +++ b/nautobot/extras/forms/mixins.py @@ -13,7 +13,6 @@ ) from nautobot.core.utils.deprecation import class_deprecated_in_favor_of from nautobot.extras.choices import ( - CustomFieldFilterLogicChoices, RelationshipSideChoices, RelationshipTypeChoices, ) @@ -63,13 +62,9 @@ class CustomFieldModelFilterFormMixin(forms.Form): def __init__(self, *args, **kwargs): - self.obj_type = ContentType.objects.get_for_model(self.model) - super().__init__(*args, **kwargs) - custom_fields = CustomField.objects.filter(content_types=self.obj_type).exclude( - filter_logic=CustomFieldFilterLogicChoices.FILTER_DISABLED - ) + custom_fields = CustomField.objects.get_for_model(self.model, exclude_filter_disabled=True) self.custom_fields = [] for cf in custom_fields: field_name = cf.add_prefix_to_cf_key() @@ -653,16 +648,13 @@ def _append_relationships(self): """ Append form fields for all Relationships assigned to this model. """ - query = Q(source_type=self.obj_type, source_hidden=False) | Q( - destination_type=self.obj_type, destination_hidden=False - ) - relationships = Relationship.objects.select_related("source_type", "destination_type").filter(query) + src_relationships, dst_relationships = Relationship.objects.get_for_model(model=self.model, hidden=False) + + for rel in src_relationships: + self._append_relationships_side([rel], RelationshipSideChoices.SIDE_SOURCE) - for rel in relationships.iterator(): - if rel.source_type == self.obj_type and not rel.source_hidden: - self._append_relationships_side([rel], RelationshipSideChoices.SIDE_SOURCE) - if rel.destination_type == self.obj_type and not rel.destination_hidden: - self._append_relationships_side([rel], RelationshipSideChoices.SIDE_DESTINATION) + for rel in dst_relationships: + self._append_relationships_side([rel], RelationshipSideChoices.SIDE_DESTINATION) def _append_relationships_side(self, relationships, initial_side): """ diff --git a/nautobot/extras/jobs.py b/nautobot/extras/jobs.py index 7776fbb2fd..d83f2df673 100644 --- a/nautobot/extras/jobs.py +++ b/nautobot/extras/jobs.py @@ -42,7 +42,7 @@ ) from nautobot.core.utils.config import get_settings_or_config from nautobot.core.utils.lookup import get_model_from_name -from nautobot.extras.choices import ObjectChangeActionChoices, ObjectChangeEventContextChoices +from nautobot.extras.choices import JobResultStatusChoices, ObjectChangeActionChoices, ObjectChangeEventContextChoices from nautobot.extras.context_managers import web_request_context from nautobot.extras.forms import JobForm from nautobot.extras.models import ( @@ -122,6 +122,7 @@ def __call__(self, *args, **kwargs): try: deserialized_kwargs = self.deserialize_data(kwargs) except Exception as err: + self.logger.error("%s", err) raise RunJobTaskFailed("Error initializing job") from err if isinstance(self, JobHookReceiver): change_context = ObjectChangeEventContextChoices.CONTEXT_JOB_HOOK @@ -309,7 +310,8 @@ def after_return(self, status, retval, task_id, args, kwargs, einfo): if file_ids: self._delete_file_proxies(*file_ids) - self.logger.info("Job completed", extra={"grouping": "post_run"}) + if status == JobResultStatusChoices.STATUS_SUCCESS: + self.logger.info("Job completed", extra={"grouping": "post_run"}) # TODO(gary): document this in job author docs # Super.after_return must be called for chords to function properly diff --git a/nautobot/extras/management/commands/runjob.py b/nautobot/extras/management/commands/runjob.py index 12968bd70e..4d47870c6f 100644 --- a/nautobot/extras/management/commands/runjob.py +++ b/nautobot/extras/management/commands/runjob.py @@ -8,6 +8,7 @@ from nautobot.extras.choices import JobResultStatusChoices, LogLevelChoices from nautobot.extras.jobs import get_job from nautobot.extras.models import Job, JobLogEntry, JobResult +from nautobot.extras.utils import get_worker_count class Command(BaseCommand): @@ -52,9 +53,24 @@ def handle(self, *args, **options): if options.get("data"): data = json.loads(options["data"]) except json.decoder.JSONDecodeError as error: - raise CommandError(f"Invalid JSON data:\n{error!s}") + raise CommandError(f"Invalid JSON data:\n{error!s}") from error - job_model = Job.objects.get_for_class_path(options["job"]) + try: + job_model = Job.objects.get_for_class_path(options["job"]) + except Job.DoesNotExist as error: + raise CommandError(f"Job {options['job']} does not exist.") from error + + try: + job_model = Job.objects.restrict(user, "run").get_for_class_path(options["job"]) + except Job.DoesNotExist: + raise CommandError(f"User {options['username']} does not have permission to run this Job") from None + + if not job_model.installed or job_model.job_class is None: + raise CommandError("Job is not presently installed") + if not job_model.enabled: + raise CommandError("Job is not presently enabled to be run") + if not options["local"] and not get_worker_count(): + raise CommandError("No Celery worker detected. Perhaps you meant to specify --local?") # Run the job and create a new JobResult self.stdout.write(f"[{timezone.now():%H:%M:%S}] Running {options['job']}...") @@ -100,12 +116,14 @@ def handle(self, *args, **options): self.stdout.write(f"\t\t{status}: {log_entry.message}") if job_result.result: - self.stdout.write(job_result.result) + self.stdout.write(str(job_result.result)) if job_result.status == JobResultStatusChoices.STATUS_FAILURE: status = self.style.ERROR("FAILURE") - else: + elif job_result.status == JobResultStatusChoices.STATUS_SUCCESS: status = self.style.SUCCESS("SUCCESS") + else: + status = self.style.WARNING(job_result.status) self.stdout.write(f"[{timezone.now():%H:%M:%S}] {options['job']}: {status}") # Wrap things up diff --git a/nautobot/extras/models/customfields.py b/nautobot/extras/models/customfields.py index cf1f0ff97b..28f513e813 100644 --- a/nautobot/extras/models/customfields.py +++ b/nautobot/extras/models/customfields.py @@ -1,5 +1,6 @@ from collections import OrderedDict from datetime import date, datetime +from functools import lru_cache import logging import re @@ -45,6 +46,7 @@ class ComputedFieldManager(BaseManager.from_queryset(RestrictedQuerySet)): use_in_migrations = True + @lru_cache(maxsize=128) def get_for_model(self, model): """ Return all ComputedFields assigned to the given model. @@ -296,12 +298,20 @@ def get_computed_fields(self, label_as_key=False, advanced_ui=None): class CustomFieldManager(BaseManager.from_queryset(RestrictedQuerySet)): use_in_migrations = True - def get_for_model(self, model): + @lru_cache(maxsize=128) + def get_for_model(self, model, exclude_filter_disabled=False): """ Return all CustomFields assigned to the given model. + + Args: + model: The django model to which custom fields are registered + exclude_filter_disabled: Exclude any custom fields which have filter logic disabled """ content_type = ContentType.objects.get_for_model(model._meta.concrete_model) - return self.get_queryset().filter(content_types=content_type) + qs = self.get_queryset().filter(content_types=content_type) + if exclude_filter_disabled: + qs = qs.exclude(filter_logic=CustomFieldFilterLogicChoices.FILTER_DISABLED) + return qs @extras_features("webhooks") diff --git a/nautobot/extras/models/jobs.py b/nautobot/extras/models/jobs.py index 36229b86e3..0b64802d3d 100644 --- a/nautobot/extras/models/jobs.py +++ b/nautobot/extras/models/jobs.py @@ -108,7 +108,10 @@ class Job(PrimaryModel): help_text="Human-readable name of this job", unique=True, ) - description = models.TextField(blank=True, help_text="Markdown formatting is supported") + description = models.TextField( + blank=True, + help_text="Markdown formatting and a limited subset of HTML are supported", + ) # Control flags installed = models.BooleanField( @@ -663,12 +666,22 @@ def enqueue_job( redirect_logger = get_logger("celery.redirected") proxy = LoggingProxy(redirect_logger, app.conf.worker_redirect_stdouts_level) with contextlib.redirect_stdout(proxy), contextlib.redirect_stderr(proxy): - job_model.job_task.apply( + eager_result = job_model.job_task.apply( args=job_args, kwargs=job_kwargs, task_id=str(job_result.id), **job_celery_kwargs ) # copy fields from eager result to job result job_result.refresh_from_db() + # Emulate prepare_exception() behavior + if isinstance(eager_result.result, Exception): + job_result.result = { + "exc_type": type(eager_result.result).__name__, + "exc_message": sanitize(str(eager_result.result)), + } + else: + job_result.result = sanitize(eager_result.result) + job_result.status = eager_result.status + job_result.traceback = sanitize(eager_result.traceback) job_result.date_done = timezone.now() job_result.save() else: diff --git a/nautobot/extras/models/relationships.py b/nautobot/extras/models/relationships.py index c1401aaca1..0a561b45e9 100644 --- a/nautobot/extras/models/relationships.py +++ b/nautobot/extras/models/relationships.py @@ -1,3 +1,4 @@ +from functools import lru_cache import logging from django import forms @@ -328,16 +329,56 @@ def required_related_objects_errors( class RelationshipManager(BaseManager.from_queryset(RestrictedQuerySet)): use_in_migrations = True - def get_for_model(self, model): + @lru_cache(maxsize=128) + def get_for_model(self, model, hidden=None): """ Return all Relationships assigned to the given model. + + Args: + model (Model): The django model to which relationships are registered + hidden (bool): Filter based on the value of the hidden flag, or None to not apply this filter + + Returns a tuple of source and destination scoped relationship querysets. """ - content_type = ContentType.objects.get_for_model(model._meta.concrete_model) return ( - self.get_queryset().filter(source_type=content_type), - self.get_queryset().filter(destination_type=content_type), + self.get_for_model_source(model, hidden=hidden), + self.get_for_model_destination(model, hidden=hidden), ) + @lru_cache(maxsize=128) + def get_for_model_source(self, model, hidden=None): + """ + Return all Relationships assigned to the given model for the source side only. + + Args: + model (Model): The django model to which relationships are registered + hidden (bool): Filter based on the value of the hidden flag, or None to not apply this filter + """ + content_type = ContentType.objects.get_for_model(model._meta.concrete_model) + result = ( + self.get_queryset().filter(source_type=content_type).select_related("source_type", "destination_type") + ) # You almost always will want access to the source_type/destination_type + if hidden is not None: + result = result.filter(source_hidden=hidden) + return result + + @lru_cache(maxsize=128) + def get_for_model_destination(self, model, hidden=None): + """ + Return all Relationships assigned to the given model for the destination side only. + + Args: + model (Model): The django model to which relationships are registered + hidden (bool): Filter based on the value of the hidden flag, or None to not apply this filter + """ + content_type = ContentType.objects.get_for_model(model._meta.concrete_model) + result = ( + self.get_queryset().filter(destination_type=content_type).select_related("source_type", "destination_type") + ) # You almost always will want access to the source_type/destination_type + if hidden is not None: + result = result.filter(destination_hidden=hidden) + return result + def get_required_for_model(self, model): """ Return a queryset with all required Relationships on the given model. diff --git a/nautobot/extras/querysets.py b/nautobot/extras/querysets.py index 233dfd087f..8ae02eab8c 100644 --- a/nautobot/extras/querysets.py +++ b/nautobot/extras/querysets.py @@ -148,7 +148,7 @@ def _get_config_context_filters(self): location_query_string = "cluster__location" location_query = Q(locations=None) | Q(locations=OuterRef(location_query_string)) - for _ in range(Location.objects.max_tree_depth() + 1): + for _ in range(Location.objects.max_depth + 1): location_query_string += "__parent" location_query |= Q(locations=OuterRef(location_query_string)) @@ -156,7 +156,7 @@ def _get_config_context_filters(self): tenant_group_query_string = "tenant__tenant_group" tenant_group_query = Q(tenant_groups=None) | Q(tenant_groups=OuterRef(tenant_group_query_string)) - for _ in range(TenantGroup.objects.max_tree_depth() + 1): + for _ in range(TenantGroup.objects.max_depth + 1): tenant_group_query_string += "__parent" tenant_group_query |= Q(tenant_groups=OuterRef(tenant_group_query_string)) base_query.add((tenant_group_query), Q.AND) diff --git a/nautobot/extras/signals.py b/nautobot/extras/signals.py index 1091001f3e..c520205565 100644 --- a/nautobot/extras/signals.py +++ b/nautobot/extras/signals.py @@ -23,12 +23,14 @@ from nautobot.extras.choices import JobResultStatusChoices, ObjectChangeActionChoices from nautobot.extras.constants import CHANGELOG_MAX_CHANGE_CONTEXT_DETAIL from nautobot.extras.models import ( + ComputedField, CustomField, DynamicGroup, DynamicGroupMembership, GitRepository, JobResult, ObjectChange, + Relationship, ) from nautobot.extras.querysets import NotesQuerySet from nautobot.extras.tasks import delete_custom_field_data, provision_field @@ -58,6 +60,29 @@ def _get_user_if_authenticated(user, instance): return None +@receiver(post_save) +@receiver(m2m_changed) +@receiver(post_delete) +def invalidate_lru_cache(sender, **kwargs): + """Invalidate the LRU cache for ComputedFields, CustomFields and Relationships.""" + if sender is CustomField.content_types.through: + manager = CustomField.objects + elif sender in (ComputedField, CustomField, Relationship): + manager = sender.objects + else: + return + + cached_methods = ( + "get_for_model", + "get_for_model_source", + "get_for_model_destination", + ) + + for method in cached_methods: + if hasattr(manager, method): + getattr(manager, method).cache_clear() + + @receiver(post_save) @receiver(m2m_changed) def _handle_changed_object(sender, instance, raw=False, **kwargs): diff --git a/nautobot/extras/tables.py b/nautobot/extras/tables.py index 5b19ef9327..e42e9fc292 100644 --- a/nautobot/extras/tables.py +++ b/nautobot/extras/tables.py @@ -17,7 +17,6 @@ ) from nautobot.core.templatetags.helpers import render_boolean, render_markdown -from .choices import LogLevelChoices from .models import ( ComputedField, ConfigContext, @@ -538,7 +537,7 @@ def log_object_link(value, record): def log_entry_color_css(record): - if record.log_level.lower() == "failure": + if record.log_level.lower() in ("error", "critical"): return "danger" return record.log_level.lower() @@ -724,20 +723,15 @@ def render_summary(self, record): """ Define custom rendering for the summary column. """ - log_objects = record.job_log_entries.all() - debug = log_objects.filter(log_level=LogLevelChoices.LOG_DEBUG).count() - info = log_objects.filter(log_level=LogLevelChoices.LOG_INFO).count() - warning = log_objects.filter(log_level=LogLevelChoices.LOG_WARNING).count() - error = log_objects.filter(log_level__in=[LogLevelChoices.LOG_ERROR, LogLevelChoices.LOG_CRITICAL]).count() return format_html( """ """, - debug, - info, - warning, - error, + record.debug_log_count, + record.info_log_count, + record.warning_log_count, + record.error_log_count, ) class Meta(BaseTable.Meta): diff --git a/nautobot/extras/templates/extras/inc/jobresult.html b/nautobot/extras/templates/extras/inc/jobresult.html index 9205a50c12..05900e4b49 100644 --- a/nautobot/extras/templates/extras/inc/jobresult.html +++ b/nautobot/extras/templates/extras/inc/jobresult.html @@ -77,15 +77,14 @@
      ` | `align`, `char`, `charoff`, `colspan`, `headers`, `rowspan` | + | `` | `align`, `char`, `charoff`, `colspan`, `headers`, `rowspan`, `scope` | + | `
  • -
    - {% csrf_token %} -
    -
    - Logs -
    - -
    + +
    +
    + Logs +
    +
    - {% ajax_table "log_table" "extras:jobresult_log-table" pk=result.pk %}
    - + {% ajax_table "log_table" "extras:jobresult_log-table" pk=result.pk %} +
    + diff --git a/nautobot/extras/templates/extras/inc/jobresult_js.html b/nautobot/extras/templates/extras/inc/jobresult_js.html index 942688fe7f..877c0e2528 100644 --- a/nautobot/extras/templates/extras/inc/jobresult_js.html +++ b/nautobot/extras/templates/extras/inc/jobresult_js.html @@ -3,10 +3,11 @@