Merge pull request #4582 from nautobot/develop-to-next-post-2.0.0

Develop to next post 2.0.0

.github/actions/build-nautobot-image/action.yaml

@@ -86,4 +86,4 @@ runs:
         build-args: |
           PYTHON_VER=${{ inputs.python-version }}
           DEPENDENCIES_BASE_BRANCH=${{ inputs.branch }}
-          POETRY_PARALLEL=true
+          POETRY_INSTALLER_PARALLEL=true

.github/workflows/ci_integration.yml

@@ -126,7 +126,7 @@ jobs:
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
       - name: "Install NPM Dependencies"
         working-directory: ./nautobot/ui
-        run: "npm install"
+        run: "npm ci"
       - name: "Setup base aliases for tests"
         working-directory: ./nautobot/ui
         run: "mkdir generated && cp src/file_templates/jsconfig-base.json generated/jsconfig.paths.json"
@@ -251,7 +251,7 @@ jobs:
     runs-on: "ubuntu-22.04"
     if: |
       github.event_name == 'push' &&
-      (github.ref_name == 'develop' || github.ref_name == 'next')
+      (github.ref_name == 'develop' || github.ref_name == 'next' || github.ref_name == 'ltm/1.6')
     needs:
       - "check-schema"
       - "integration-test"
@@ -269,12 +269,16 @@ jobs:
         id: "config"
         shell: "bash"
         run: |
-          export BRANCH="${{ github.ref_name }}"
+          if [[ "${{ github.ref_name }}" == "ltm/1.6" ]]; then
+            export BRANCH="ltm-1.6"
+          else
+            export BRANCH="${{ github.ref_name }}"
+          fi
           export TAG_LATEST="false"
           export TAG_LATEST_FOR_BRANCH="false"
           export TAG_LATEST_FOR_PY="false"
 
-          if [[ $BRANCH == "develop" ]]; then
+          if [[ $BRANCH == "ltm-1.6" ]]; then
             export TAG_LATEST_FOR_PY="true"
           fi
 

.github/workflows/ci_pullrequest.yml

@@ -42,7 +42,7 @@ jobs:
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
       - name: "Install NPM Dependencies"
         working-directory: "./nautobot/ui"
-        run: "npm install"
+        run: "npm ci"
       - name: "Linting: prettier"
         run: "poetry run invoke prettier"
   eslint:
@@ -56,7 +56,7 @@ jobs:
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
       - name: "Install NPM Dependencies"
         working-directory: "./nautobot/ui"
-        run: "npm install"
+        run: "npm ci"
       - name: "Linting: eslint"
         run: "poetry run invoke eslint"
   yamllint:
@@ -81,7 +81,7 @@ jobs:
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
       - name: "Install NPM Dependencies"
         working-directory: "./nautobot/ui"
-        run: "npm install"
+        run: "npm ci"
       - name: "Linting: markdownlint"
         run: "poetry run invoke markdownlint"
   hadolint:
@@ -274,7 +274,7 @@ jobs:
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
       - name: "Install NPM Dependencies"
         working-directory: ./nautobot/ui
-        run: "npm install"
+        run: "npm ci"
       - name: "Setup base aliases for tests"
         working-directory: ./nautobot/ui
         run: "mkdir generated && cp src/file_templates/jsconfig-base.json generated/jsconfig.paths.json"

.github/workflows/plugin_upstream_testing_base.yml

@@ -20,13 +20,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        nautobot-version: ["next", "develop"]
+        nautobot-version: ["develop", "ltm-1.6"]
     runs-on: "ubuntu-20.04"
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
         with:
-          ref: "${{ matrix.nautobot-version == 'next' && 'next-2.0' || env.GITHUB_REF_NAME || 'develop' }}"
+          ref: "${{ matrix.nautobot-version == 'develop' && 'next-2.0' || matrix.nautobot-version == 'ltm-1.6' && 'develop' || env.GITHUB_REF_NAME || 'develop' }}"
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
       - name: "Set up Docker Buildx"

.github/workflows/prerelease.yml

@@ -103,7 +103,7 @@ jobs:
           context: "."
           build-args: |
             PYTHON_VER=${{ matrix.python-version }}
-            POETRY_PARALLEL=true
+            POETRY_INSTALLER_PARALLEL=true
       - name: "Docker Dev Metadata"
         id: "dockerdevmeta"
         uses: "docker/metadata-action@v3"
@@ -130,7 +130,7 @@ jobs:
           context: "."
           build-args: |
             PYTHON_VER=${{ matrix.python-version }}
-            POETRY_PARALLEL=true
+            POETRY_INSTALLER_PARALLEL=true
 
   slack-notify:
     needs:

.github/workflows/release.yml

@@ -107,7 +107,7 @@ jobs:
           context: "."
           build-args: |
             PYTHON_VER=${{ matrix.python-version }}
-            POETRY_PARALLEL=true
+            POETRY_INSTALLER_PARALLEL=true
       - name: "Docker Dev Metadata"
         id: "dockerdevmeta"
         uses: "docker/metadata-action@v3"
@@ -138,7 +138,7 @@ jobs:
           context: "."
           build-args: |
             PYTHON_VER=${{ matrix.python-version }}
-            POETRY_PARALLEL=true
+            POETRY_INSTALLER_PARALLEL=true
 
   slack-notify:
     needs:

development/docker-compose.dev.yml

@@ -46,4 +46,4 @@ services:
       - "6006:6006"
     entrypoint: ""
     # TODO this needs to ensure the version of nautobot-ui remains in sync with package-lock.json
-    command: bash -c "git clone https://github.com/nautobot/nautobot-ui.git && cd nautobot-ui && npm install && npm run storybook|| npm run storybook"
+    command: bash -c "git clone https://github.com/nautobot/nautobot-ui.git && cd nautobot-ui && npm ci && npm run storybook|| npm run storybook"

docker/Dockerfile

@@ -49,25 +49,37 @@ ENV PYTHONUNBUFFERED=1 \
     NAUTOBOT_ROOT=/opt/nautobot \
     prometheus_multiproc_dir=/prom_cache
 
+ARG NODE_MAJOR=18
+
 # DL3008 - Require pinned dependencies for apt install
 # DL3009 - Delete the apt-get lists after installing something
-# DL3013 - pin all Python package versions
-# DL3042 - run pip install with --no-cache-dir (https://github.com/hadolint/hadolint/issues/497)
 # DL4006 - Set the SHELL option -o pipefail before RUN with a pipe in
-# hadolint ignore=DL3008,DL3009,DL3013,DL3042,DL4006
-RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \
+# hadolint ignore=DL3008,DL3009,DL4006
+RUN --mount=type=cache,target="/var/cache/apt",sharing=locked \
+    --mount=type=cache,target="/var/lib/apt/lists",sharing=locked \
+    apt-get update && \
+    apt-get install --no-install-recommends -y ca-certificates curl gnupg && \
+    mkdir -p /etc/apt/keyrings && \
+    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
     apt-get update && \
     apt-get upgrade -y && \
-    apt-get install --no-install-recommends -y curl && \
-    curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \
     apt-get install --no-install-recommends -y git mime-support libxml2 libmariadb3 openssl nodejs && \
-    apt-get autoremove -y && \
-    apt-get clean all && \
-    rm -rf /var/lib/apt/lists/* && \
-    npm install -g npm@latest && \
-    rm -rf /root/.npm && \
-    pip install --upgrade pip wheel && \
-    rm -rf /tmp/tmp*
+    apt-get autoremove -y
+
+# Ensure the npm install is cached, updated, and has a long timeout for slow connections
+# The Node.JS install above should install npm 9, but we pin it here just in case
+# npm 10 has shown issues with slow CI environments
+RUN --mount=type=cache,target="/root/.npm",sharing=locked \
+    npm config set fetch-retry-maxtimeout 300000 && \
+    npm install -g npm@9
+
+# DL3013 - pin all Python package versions
+# DL3042 - run pip install with --no-cache-dir (https://github.com/hadolint/hadolint/issues/497)
+# hadolint ignore=DL3013,DL3042
+RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \
+    --mount=type=cache,target="/tmp",sharing=locked \
+    pip install --upgrade pip wheel
 
 HEALTHCHECK --interval=5s --timeout=5s --start-period=5s --retries=1 CMD curl --fail http://localhost:8080/health/ || exit 1
 
@@ -116,48 +128,47 @@ COPY --chown=nautobot:nautobot nautobot/ui/package.json nautobot/ui/package-lock
 WORKDIR /opt/nautobot/ui
 
 # Install (non-development) Node dependencies of Nautobot UI
-RUN npm install --omit=dev
+# npm ci is used instead of npm install to ensure that the package-lock.json is used
+RUN npm ci --omit=dev
 
 ################################ Stage: node-dev-dependencies (intermediate build target)
 
 FROM node-dependencies AS node-dev-dependencies
 
 # Install development Node dependencies of Nautobot UI
-RUN npm install --include=dev
-
-################################ Stage: python-dependencies (intermediate build target)
-
-# hadolint ignore=DL3006
-FROM system-dev-dependencies-$TARGETARCH AS python-dependencies
+# npm ci is used instead of npm install to ensure that the package-lock.json is used
+RUN npm ci --include=dev
 
-USER nautobot
+################################ Stage: poetry (stub for poetry only)
 
+FROM python:${PYTHON_VER}-slim AS poetry
 # Install Poetry manually via its installer script;
 # if we instead used "pip install poetry" it would install its own dependencies globally which may conflict with ours.
 # https://python-poetry.org/docs/master/#installing-with-the-official-installer
 # This also makes it so that Poetry will *not* be included in the "final" image since it's not installed to /usr/local/
-RUN --mount=type=cache,target="/opt/nautobot/.cache",uid=999,gid=999,sharing=locked \
-    curl -sSL https://install.python-poetry.org -o /tmp/install-poetry.py && \
-    python /tmp/install-poetry.py --version 1.5.1 && \
-    rm -f /tmp/install-poetry.py
+ARG POETRY_HOME=/opt/poetry
+ARG POETRY_INSTALLER_PARALLEL=true
+ARG POETRY_VERSION=1.5.1
+ARG POETRY_VIRTUALENVS_CREATE=false
+ADD https://install.python-poetry.org /tmp/install-poetry.py
+RUN python /tmp/install-poetry.py
 
 # Add poetry install location to the $PATH
-ENV PATH="${PATH}:/opt/nautobot/.local/bin"
+ENV PATH="${POETRY_HOME}/bin:${PATH}"
 
-RUN --mount=type=cache,target="/opt/nautobot/.cache",uid=999,gid=999,sharing=locked \
-    python -m venv /opt/nautobot
+RUN poetry config virtualenvs.create ${POETRY_VIRTUALENVS_CREATE} && \
+    poetry config installer.parallel "${POETRY_INSTALLER_PARALLEL}" && \
+    poetry config installer.no-binary pyuwsgi
 
-ENV PATH="/opt/nautobot/bin:${PATH}"
-ENV VIRTUAL_ENV="/opt/nautobot"
+################################ Stage: python-dependencies (intermediate build target)
 
-# Poetry 1.1.0 added parallel installation as an option;
-# unfortunately it seems to have some issues with installing/updating "requests" and "certifi"
-# while simultaneously atttempting to *use* those packages to install other packages.
-# This is disabled by default (safer), but can be re-enabled by setting POETRY_PARALLEL=true
-# pyuwsgi wheel doesn't support ssl so we build it from source
-# https://github.com/nautobot/nautobot/issues/193
-RUN poetry config installer.parallel "${POETRY_PARALLEL:-false}" && \
-    poetry config installer.no-binary pyuwsgi
+# hadolint ignore=DL3006
+FROM system-dev-dependencies-$TARGETARCH AS python-dependencies
+
+ARG POETRY_HOME=/opt/poetry
+COPY --from=poetry ${POETRY_HOME} ${POETRY_HOME}
+COPY --from=poetry /root/.config/pypoetry /root/.config/pypoetry
+ENV PATH="${POETRY_HOME}/bin:${PATH}"
 
 # The example_plugin is only a dev dependency, but Poetry fails to install non-dev dependencies if its source is missing
 COPY --chown=nautobot:nautobot pyproject.toml poetry.lock README.md /source/
@@ -166,7 +177,7 @@ COPY --chown=nautobot:nautobot examples /source/examples
 WORKDIR /source
 
 # Install (non-development) Python dependencies of Nautobot
-RUN --mount=type=cache,target="/opt/nautobot/.cache",uid=999,gid=999,sharing=locked \
+RUN --mount=type=cache,target="/root/.cache",sharing=locked \
     poetry install --no-root --only main --no-ansi --extras all && \
     rm -rf /tmp/tmp*
 
@@ -175,7 +186,7 @@ RUN --mount=type=cache,target="/opt/nautobot/.cache",uid=999,gid=999,sharing=loc
 FROM python-dependencies AS python-dev-dependencies
 
 # Add development-specific dependencies of Nautobot to the installation
-RUN --mount=type=cache,target="/opt/nautobot/.cache",uid=999,gid=999,sharing=locked \
+RUN --mount=type=cache,target="/root/.cache",sharing=locked \
     poetry install --no-root --no-ansi --extras all && \
     rm -rf /tmp/tmp*
 
@@ -202,7 +213,7 @@ COPY --chown=nautobot:nautobot nautobot /source/nautobot
 
 COPY --from=build-nautobot --chown=nautobot:nautobot /source/nautobot/project-static/docs /source/nautobot/project-static/docs
 
-RUN --mount=type=cache,target="/opt/nautobot/.cache",uid=999,gid=999,sharing=locked \
+RUN --mount=type=cache,target="/root/.cache",sharing=locked \
     poetry install --no-ansi --extras all && \
     rm -rf /tmp/tmp*
 
@@ -212,10 +223,6 @@ COPY --chown=nautobot:nautobot development/nautobot_config.py /opt/nautobot/naut
 
 RUN nautobot-server build_ui && rm -rf /source/*
 
-# switch to root user for dev stage: https://github.com/nautobot/nautobot/issues/4164
-# hadolint ignore=DL3002
-USER root
-
 # Run Nautobot development server by default
 EXPOSE 8080
 CMD ["nautobot-server", "runserver", "0.0.0.0:8080", "--insecure"]
@@ -226,17 +233,18 @@ FROM python-dev-dependencies AS final-dev
 
 COPY --chown=nautobot:nautobot nautobot/ui /opt/nautobot/ui
 COPY --from=node-dev-dependencies --chown=nautobot:nautobot /opt/nautobot/ui/node_modules /opt/nautobot/ui/node_modules
-
 COPY --from=build-nautobot --chown=nautobot:nautobot /source/dist /source/dist
 
 # Install Nautobot wheel, and uninstall example apps as they're not included in the final-dev image
 # DL3042 - run pip install with --no-cache-dir (https://github.com/hadolint/hadolint/issues/497)
 # hadolint ignore=DL3042
-RUN --mount=type=cache,target="/opt/nautobot/.cache",uid=999,gid=999,sharing=locked \
+RUN --mount=type=cache,target="/root/.cache",sharing=locked \
     pip install --no-deps /source/dist/*.whl && \
     pip uninstall -y example-plugin example-plugin-with-view-override && \
     rm -rf /source/*
 
+USER nautobot
+
 WORKDIR /opt/nautobot
 
 # Don't send install metrics as this is a development target, not a deployment one
@@ -257,15 +265,9 @@ CMD ["nautobot-server", "runserver", "0.0.0.0:8080", "--insecure"]
 
 FROM system-dependencies AS final
 
-USER nautobot
-ENV PATH="/opt/nautobot/bin:${PATH}"
-ENV VIRTUAL_ENV="/opt/nautobot"
-
-COPY --from=python-dependencies --chown=nautobot:nautobot /opt/nautobot/bin /opt/nautobot/bin
-COPY --from=python-dependencies --chown=nautobot:nautobot /opt/nautobot/include /opt/nautobot/include
-COPY --from=python-dependencies --chown=nautobot:nautobot /opt/nautobot/lib /opt/nautobot/lib
-COPY --from=python-dependencies --chown=nautobot:nautobot /opt/nautobot/lib64 /opt/nautobot/lib64
-COPY --from=python-dependencies --chown=nautobot:nautobot /opt/nautobot/pyvenv.cfg /opt/nautobot/pyvenv.cfg
+ARG PYTHON_VER
+COPY --from=python-dependencies /usr/local/lib/python${PYTHON_VER}/site-packages /usr/local/lib/python${PYTHON_VER}/site-packages
+COPY --from=python-dependencies /usr/local/bin /usr/local/bin
 
 COPY --from=build-nautobot --chown=nautobot:nautobot /source/dist /source/dist
 
@@ -274,10 +276,12 @@ COPY --from=final-dev --chown=nautobot:nautobot /opt/nautobot/ui /opt/nautobot/u
 
 # DL3042 - run pip install with --no-cache-dir (https://github.com/hadolint/hadolint/issues/497)
 # hadolint ignore=DL3042
-RUN --mount=type=cache,target="/opt/nautobot/.cache",uid=999,gid=999,sharing=locked \
+RUN --mount=type=cache,target="/root/.cache",sharing=locked \
     pip install --no-deps /source/dist/*.whl && \
     rm -rf /source/*
 
+USER nautobot
+
 # Generate self-signed SSL certs
 RUN openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj \
     '/C=US/ST=NY/L=NYC/O=Nautobot/CN=nautobot.local' \

examples/example_plugin/example_plugin/__init__.py

@@ -27,6 +27,9 @@ class ExamplePluginConfig(NautobotAppConfig):
         "SAMPLE_VARIABLE": ConstanceConfigItem(
             default="example_default_value", help_text="Example of supplying a setting through Django Constance."
         ),
+        "lowercase_example": ConstanceConfigItem(
+            default="example_lowercase_variable", help_text="Example of all lowercase variable name."
+        ),
     }
     searchable_models = ["examplemodel"]