Finalized MySQL CI/CD integration

.travis.yml

@@ -1,31 +1,61 @@
 # Linting/Testing available at https://config.travis-ci.com/explore
 os: linux
 dist: xenial
-services:
-  - postgresql
-  - redis
-addons:
-  postgresql: "9.6"
-language: python
+
+# Specify the explicit order of the build stages here.
+stages:
+  - lint
+  - test
+  - integration
+
+# Cache settings to explicitly cache Pip & Poetry files
+cache:
+  pip: true
+  directories:
+    - ~/.cache/pypoetry
+
+#
+## Begin "test" stage global config
+#
+
+# Environment variables passed to the job nodes
 env:
+
+  # Environment variables passed to all jobs
   global:
     - INVOKE_NAUTOBOT_LOCAL=True
     - NAUTOBOT_SELENIUM_URL=http://localhost:4444/wd/hub
     - NAUTOBOT_SELENIUM_HOST=$(hostname -f)
-jobs:
-  fast_finish: true
-  include:
-    - stage: Tests
-      python: "3.6"
-    - python: "3.7"
-    - python: "3.8"
-    - python: "3.9"
-  allow_failures:
-    - python: "3.9"
+  # Environment variables passed to "test" stage jobs
+  jobs:
+    - DB=postgres NAUTOBOT_DB_ENGINE=django.db.backends.postgresql
+    - DB=mysql NAUTOBOT_DB_USER=root NAUTOBOT_DB_ENGINE=django.db.backends.mysql
+
+# We be using Python
+language: python
+
+# Test these Python versions
+python:
+  - "3.6"
+  - "3.7"
+  - "3.8"
+  - "3.9"
+
+# Services we want installed on the VM(s)
+services:
+  - mysql
+  - postgresql
+  - redis
+
+# Explicitly install PostgreSQL 9.6
+addons:
+  postgresql: "9.6"
+
+# Things to do before install phase
 before_install:
   - pip install poetry
-  - curl -Lo /home/travis/bin/hadolint https://github.com/hadolint/hadolint/releases/download/v2.0.0/hadolint-Linux-x86_64
-  - chmod +x /home/travis/bin/hadolint
+
+# Install phase
 install:
   - poetry config virtualenvs.create false
   # Poetry 1.1.0 added parallel installation as an option;
@@ -34,8 +64,105 @@ install:
   # For now we disable it.
   - poetry config installer.parallel false
   - poetry install
+
+# Things to do before the script phase
 before_script:
-  - psql -U postgres -c 'create database nautobot;'
+  # If postgres: Create the database
+  - sh -c "if [ '$DB' = 'postgres' ]; then echo 'Creating PostgreSQL database...'; fi"
+  - sh -c "if [ '$DB' = 'postgres' ]; then psql -c 'CREATE DATABASE nautobot;' -U postgres; fi"
+  # If mysql: Upgrade/install MySQL 8.x, then create the database
+  - sh -c "if [ '$DB' = 'mysql' ]; then echo 'Upgrading MySQL to v8.x...'; fi"
+  - sh -c "if [ '$DB' = 'mysql' ]; then wget https://repo.mysql.com//mysql-apt-config_0.8.17-1_all.deb; fi"
+  - sh -c "if [ '$DB' = 'mysql' ]; then sudo dpkg -i mysql-apt-config_0.8.17-1_all.deb; fi"
+  - sh -c "if [ '$DB' = 'mysql' ]; then sudo apt-get update -q; fi"
+  - sh -c "if [ '$DB' = 'mysql' ]; then sudo apt-get install -q -y --allow-unauthenticated -o Dpkg::Options::=--force-confnew mysql-server; fi"
+  - sh -c "if [ '$DB' = 'mysql' ]; then sudo systemctl restart mysql; fi"
+  - sh -c "if [ '$DB' = 'mysql' ]; then sudo mysql_upgrade; fi"
+  - sh -c "if [ '$DB' = 'mysql' ]; then mysql --version; fi"
+  - sh -c "if [ '$DB' = 'mysql' ]; then echo 'Creating MySQL database...'; fi"
+  - sh -c "if [ '$DB' = 'mysql' ]; then mysql -e 'CREATE DATABASE IF NOT EXISTS nautobot;'; fi"
+  # Install the mysqlclient lib
+  - sh -c "if [ '$DB' = 'mysql' ]; then poetry install --extras mysql; fi"
+  # Install Docker Compose
   - pip install docker-compose
+
+# Script phase
 script:
-  - poetry run ./scripts/cibuild.sh
+  # Run unit tests
+  - invoke unittest --failfast --keepdb || travis_terminate 1
+  # Generate unit test coverage report
+  - invoke unittest-coverage || travis_terminate 1
+
+#
+## End "test" stage global config
+#
+
+# Job/stage matrix
+jobs:
+
+  # Terminate build matrix as soon as any job fails.
+  fast_finish: true
+
+  # Job definitions for custom stages
+  include:
+
+    - stage: lint
+
+      before_install:
+        - pip install poetry
+        # - pip install invoke requests black=="20.8b1" flake8=="3.9.2"  # Until there is a poetry install --dev-only install the linting tools here to speed up this step
+        - curl -Lo /home/travis/bin/hadolint https://github.com/hadolint/hadolint/releases/download/v2.0.0/hadolint-Linux-x86_64
+        - chmod +x /home/travis/bin/hadolint
+
+      # Zero out the globals for this stage
+      # install: []  # nautobot needs to be installed for checking migrations
+      services: []
+      addons:
+        postgresql: ""
+      before_script:
+        - psql -U postgres -c 'CREATE DATABASE nautobot;'
+
+      # Python 3.9; lint only
+      python: "3.9"
+      script: "invoke tests --lint-only"
+
+    - stage: integration
+
+      python: "3.9"
+
+      # Services we want installed on the VM(s)
+      services:
+        - postgresql
+        - redis
+
+      # Explicitly install PostgreSQL 9.6
+      addons:
+        postgresql: "9.6"
+
+      # Install phase
+      install:
+        - poetry config virtualenvs.create false
+        # Poetry 1.1.0 added parallel installation as an option;
+        # unfortunately it seems to have some issues with installing/updating "requests" and "certifi"
+        # while simultaneously atttempting to *use* those packages to install other packages.
+        # For now we disable it.
+        - poetry config installer.parallel false
+        - poetry install
+
+      # Things to do before the script phase
+      before_script:
+        # Create Nautobot database
+        - psql -U postgres -c 'CREATE DATABASE nautobot;'
+        # Install Docker compose
+        - pip install docker-compose
+        # Login to Docker if password is in the environment
+        - sh -c "if [[ -n '$DOCKER_HUB_PASSWORD' ]]; then echo -e '\n>> Attempting login to Docker Hub...'; echo '$DOCKER_HUB_PASSWORD' | docker login -u '$DOCKER_HUB_USERNAME' --password-stdin; fi"
+
+      # Script phase
+      script:
+        # Start Selenium container
+        - invoke start --service selenium
+        # Run integration tests
+        - invoke integration-test --keepdb --append
+        # Generate integration test coverage report
+        - invoke unittest-coverage || travis_terminate 1

development/nautobot_config.py

@@ -5,15 +5,15 @@
 from nautobot.core.settings import *
 from nautobot.core.settings_funcs import is_truthy, parse_redis_connection
 
-ALLOWED_HOSTS = os.environ.get("NAUTOBOT_ALLOWED_HOSTS", "").split(" ")
+ALLOWED_HOSTS = os.getenv("NAUTOBOT_ALLOWED_HOSTS", "").split(" ")
 
 DATABASES = {
     "default": {
-        "NAME": os.environ.get("NAUTOBOT_DB_NAME", "nautobot"),
-        "USER": os.environ.get("NAUTOBOT_DB_USER", ""),
-        "PASSWORD": os.environ.get("NAUTOBOT_DB_PASSWORD", ""),
-        "HOST": os.environ.get("NAUTOBOT_DB_HOST", "localhost"),
-        "PORT": os.environ.get("NAUTOBOT_DB_PORT", ""),
+        "NAME": os.getenv("NAUTOBOT_DB_NAME", "nautobot"),
+        "USER": os.getenv("NAUTOBOT_DB_USER", ""),
+        "PASSWORD": os.getenv("NAUTOBOT_DB_PASSWORD", ""),
+        "HOST": os.getenv("NAUTOBOT_DB_HOST", "localhost"),
+        "PORT": os.getenv("NAUTOBOT_DB_PORT", ""),
         "CONN_MAX_AGE": int(os.getenv("NAUTOBOT_DB_TIMEOUT", 300)),
         "ENGINE": os.getenv("NAUTOBOT_DB_ENGINE", "django.db.backends.postgresql"),
     }
@@ -90,9 +90,9 @@
 # REDIS CACHEOPS
 CACHEOPS_REDIS = parse_redis_connection(redis_database=1)
 
-HIDE_RESTRICTED_UI = os.environ.get("HIDE_RESTRICTED_UI", False)
+HIDE_RESTRICTED_UI = os.getenv("HIDE_RESTRICTED_UI", False)
 
-SECRET_KEY = os.environ.get("NAUTOBOT_SECRET_KEY", "")
+SECRET_KEY = os.getenv("NAUTOBOT_SECRET_KEY", "")
 
 # Django Debug Toolbar
 DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda _request: DEBUG and not TESTING}

invoke.yml.example

@@ -3,8 +3,9 @@ nautobot:
   local: False
   python_ver: "3.7"
   compose_dir: "/full/path/to/nautobot/development"
-  compose_file: "docker-compose.yml"
-  compose_override_file: "docker-compose.override.yml"
+  compose_files:
+    - "docker-compose.yml"
+    - "docker-compose.override.yml"
   docker_image_names_main:
     - "networktocode/nautobot"
     - "ghcr.io/nautobot/nautobot"

nautobot/core/runner/runner.py

@@ -92,7 +92,7 @@ def configure_app(
 
     # normalize path
     if settings_envvar in os.environ:
-        default_config_path = os.environ.get(settings_envvar)
+        default_config_path = os.getenv(settings_envvar)
     else:
         default_config_path = os.path.normpath(os.path.abspath(os.path.expanduser(default_config_path)))
 

nautobot/core/settings.py

@@ -51,7 +51,7 @@
 BANNER_TOP = ""
 
 # Base directory wherein all created files (jobs, git repositories, file uploads, static files) will be stored)
-NAUTOBOT_ROOT = os.environ.get("NAUTOBOT_ROOT", os.path.expanduser("~/.nautobot"))
+NAUTOBOT_ROOT = os.getenv("NAUTOBOT_ROOT", os.path.expanduser("~/.nautobot"))
 
 CHANGELOG_RETENTION = 90
 DOCS_ROOT = os.path.join(os.path.dirname(BASE_DIR), "docs")
@@ -71,9 +71,9 @@
 )
 
 EXEMPT_VIEW_PERMISSIONS = []
-GIT_ROOT = os.environ.get("NAUTOBOT_GIT_ROOT", os.path.join(NAUTOBOT_ROOT, "git").rstrip("/"))
+GIT_ROOT = os.getenv("NAUTOBOT_GIT_ROOT", os.path.join(NAUTOBOT_ROOT, "git").rstrip("/"))
 HTTP_PROXIES = None
-JOBS_ROOT = os.environ.get("NAUTOBOT_JOBS_ROOT", os.path.join(NAUTOBOT_ROOT, "jobs").rstrip("/"))
+JOBS_ROOT = os.getenv("NAUTOBOT_JOBS_ROOT", os.path.join(NAUTOBOT_ROOT, "jobs").rstrip("/"))
 MAINTENANCE_MODE = False
 MAX_PAGE_SIZE = 1000
 
@@ -271,7 +271,7 @@
 }
 
 # The secret key is used to encrypt session keys and salt passwords.
-SECRET_KEY = os.environ.get("SECRET_KEY")
+SECRET_KEY = os.getenv("SECRET_KEY")
 
 # Default overrides
 ALLOWED_HOSTS = []

nautobot/core/tests/nautobot_config.py

@@ -17,9 +17,9 @@
         "USER": os.getenv("NAUTOBOT_DB_USER", ""),
         "PASSWORD": os.getenv("NAUTOBOT_DB_PASSWORD", ""),
         "HOST": os.getenv("NAUTOBOT_DB_HOST", "localhost"),
-        "PORT": "",
-        "CONN_MAX_AGE": 300,
-        "ENGINE": "django.db.backends.postgresql",
+        "PORT": os.getenv("NAUTOBOT_DB_PORT", ""),
+        "CONN_MAX_AGE": int(os.getenv("NAUTOBOT_DB_TIMEOUT", 300)),
+        "ENGINE": os.getenv("NAUTOBOT_DB_ENGINE", "django.db.backends.postgresql"),
     }
 }
 

tasks.py

@@ -13,11 +13,12 @@
 """
 
 from distutils.util import strtobool
+import os
+from time import sleep
+
 from invoke import Collection, task as invoke_task
 from invoke.exceptions import Exit
-import os
 import requests
-from time import sleep
 import toml
 
 
@@ -46,8 +47,10 @@ def is_truthy(arg):
             "python_ver": "3.6",
             "local": False,
             "compose_dir": os.path.join(os.path.dirname(__file__), "development/"),
-            "compose_file": "docker-compose.yml",
-            "compose_override_file": "docker-compose.dev.yml",
+            "compose_files": [
+                "docker-compose.yml",
+                "docker-compose.dev.yml",
+            ],
             "docker_image_names_main": [
                 "networktocode/nautobot",
                 "ghcr.io/nautobot/nautobot",
@@ -92,11 +95,12 @@ def docker_compose(context, command, **kwargs):
         command (str): Command string to append to the "docker-compose ..." command, such as "build", "up", etc.
         **kwargs: Passed through to the context.run() call.
     """
-    compose_file_path = os.path.join(context.nautobot.compose_dir, context.nautobot.compose_file)
-    compose_command = f'docker-compose --project-name {context.nautobot.project_name} --project-directory "{context.nautobot.compose_dir}" -f "{compose_file_path}"'
-    compose_override_path = os.path.join(context.nautobot.compose_dir, context.nautobot.compose_override_file)
-    if os.path.isfile(compose_override_path):
-        compose_command += f' -f "{compose_override_path}"'
+    compose_command = f'docker-compose --project-name {context.nautobot.project_name} --project-directory "{context.nautobot.compose_dir}"'
+
+    for compose_file in context.nautobot.compose_files:
+        compose_file_path = os.path.join(context.nautobot.compose_dir, compose_file)
+        compose_command += f' -f "{compose_file_path}"'
+
     compose_command += f" {command}"
 
     # If `service` was passed as a kwarg, add it to the end.
@@ -426,7 +430,7 @@ def unittest(
     """Run Nautobot unit tests."""
 
     append_arg = " --append" if append else ""
-    command = f"coverage run{append_arg} --module nautobot.core.cli test {label} --config=nautobot/core/tests/nautobot_config.py"
+    command = f"coverage run{append_arg} --module nautobot.core.cli --config=nautobot/core/tests/nautobot_config.py test {label}"
     # booleans
     if keepdb:
         command += " --keepdb"
@@ -508,6 +512,6 @@ def tests(context, lint_only=False):
     black(context)
     flake8(context)
     hadolint(context)
-    check_migrations(context)
+    # check_migrations(context)
     if not lint_only:
         unittest(context)