diff --git a/.travis.yml b/.travis.yml index 1da2f68b33..88a385f0b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,31 +1,61 @@ # Linting/Testing available at https://config.travis-ci.com/explore os: linux dist: xenial -services: - - postgresql - - redis -addons: - postgresql: "9.6" -language: python + +# Specify the explicit order of the build stages here. +stages: + - lint + - test + - integration + +# Cache settings to explicitly cache Pip & Poetry files +cache: + pip: true + directories: + - ~/.cache/pypoetry + +# +## Begin "test" stage global config +# + +# Environment variables passed to the job nodes env: + + # Environment variables passed to all jobs global: - INVOKE_NAUTOBOT_LOCAL=True - NAUTOBOT_SELENIUM_URL=http://localhost:4444/wd/hub - NAUTOBOT_SELENIUM_HOST=$(hostname -f) -jobs: - fast_finish: true - include: - - stage: Tests - python: "3.6" - - python: "3.7" - - python: "3.8" - - python: "3.9" - allow_failures: - - python: "3.9" + # Environment variables passed to "test" stage jobs + jobs: + - DB=postgres NAUTOBOT_DB_ENGINE=django.db.backends.postgresql + - DB=mysql NAUTOBOT_DB_USER=root NAUTOBOT_DB_ENGINE=django.db.backends.mysql + +# We be using Python +language: python + +# Test these Python versions +python: + - "3.6" + - "3.7" + - "3.8" + - "3.9" + +# Services we want installed on the VM(s) +services: + - mysql + - postgresql + - redis + +# Explicitly install PostgreSQL 9.6 +addons: + postgresql: "9.6" + +# Things to do before install phase before_install: - pip install poetry - - curl -Lo /home/travis/bin/hadolint https://github.com/hadolint/hadolint/releases/download/v2.0.0/hadolint-Linux-x86_64 - - chmod +x /home/travis/bin/hadolint + +# Install phase install: - poetry config virtualenvs.create false # Poetry 1.1.0 added parallel installation as an option; @@ -34,8 +64,106 @@ install: # For now we disable it. - poetry config installer.parallel false - poetry install + +# Things to do before the script phase before_script: - - psql -U postgres -c 'create database nautobot;' + # If postgres: Create the database + - sh -c "if [ '$DB' = 'postgres' ]; then echo 'Creating PostgreSQL database...'; fi" + - sh -c "if [ '$DB' = 'postgres' ]; then psql -c 'CREATE DATABASE nautobot;' -U postgres; fi" + # If mysql: Upgrade/install MySQL 8.x, then create the database + # We have to upgrade MySQL from v5.7 to get the Travis-specific underpinnings + - sh -c "if [ '$DB' = 'mysql' ]; then echo 'Upgrading MySQL to v8.x...'; fi" + - sh -c "if [ '$DB' = 'mysql' ]; then wget https://repo.mysql.com//mysql-apt-config_0.8.17-1_all.deb; fi" + - sh -c "if [ '$DB' = 'mysql' ]; then sudo dpkg -i mysql-apt-config_0.8.17-1_all.deb; fi" + - sh -c "if [ '$DB' = 'mysql' ]; then sudo apt-get update -q; fi" + - sh -c "if [ '$DB' = 'mysql' ]; then sudo apt-get install -q -y --allow-unauthenticated -o Dpkg::Options::=--force-confnew mysql-server; fi" + - sh -c "if [ '$DB' = 'mysql' ]; then sudo systemctl restart mysql; fi" + - sh -c "if [ '$DB' = 'mysql' ]; then sudo mysql_upgrade; fi" + - sh -c "if [ '$DB' = 'mysql' ]; then mysql --version; fi" + - sh -c "if [ '$DB' = 'mysql' ]; then echo 'Creating MySQL database...'; fi" + - sh -c "if [ '$DB' = 'mysql' ]; then mysql -e 'CREATE DATABASE IF NOT EXISTS nautobot;'; fi" + # Install the mysqlclient lib + - sh -c "if [ '$DB' = 'mysql' ]; then poetry install --extras mysql; fi" + # Install Docker Compose - pip install docker-compose + +# Script phase script: - - poetry run ./scripts/cibuild.sh + # Run unit tests + - invoke unittest --failfast --keepdb || travis_terminate 1 + # Generate unit test coverage report + - invoke unittest-coverage || travis_terminate 1 + +# +## End "test" stage global config +# + +# Job/stage matrix +jobs: + + # Terminate build matrix as soon as any job fails. + fast_finish: true + + # Job definitions for custom stages + include: + + - stage: lint + + before_install: + - pip install poetry + - curl -Lo /home/travis/bin/hadolint https://github.com/hadolint/hadolint/releases/download/v2.0.0/hadolint-Linux-x86_64 + - chmod +x /home/travis/bin/hadolint + + # Zero out the globals for this stage + # install: [] # nautobot needs to be installed for checking migrations + services: [] + addons: + postgresql: "" + before_script: + # This is required so that we can run `invoke check-migrations` + - psql -U postgres -c 'CREATE DATABASE nautobot;' + + # Python 3.9; lint only + python: "3.9" + script: "invoke tests --lint-only" + + - stage: integration + + python: "3.9" + + # Services we want installed on the VM(s) + services: + - postgresql + - redis + + # Explicitly install PostgreSQL 9.6 + addons: + postgresql: "9.6" + + # Install phase + install: + - poetry config virtualenvs.create false + # Poetry 1.1.0 added parallel installation as an option; + # unfortunately it seems to have some issues with installing/updating "requests" and "certifi" + # while simultaneously atttempting to *use* those packages to install other packages. + # For now we disable it. + - poetry config installer.parallel false + - poetry install + + # Things to do before the script phase + before_script: + # Create Nautobot database + - psql -U postgres -c 'CREATE DATABASE nautobot;' + # Install Docker compose + - pip install docker-compose + # Login to Docker if password is in the environment + - sh -c "if [[ -n '$DOCKER_HUB_PASSWORD' ]]; then echo -e '\n>> Attempting login to Docker Hub...'; echo '$DOCKER_HUB_PASSWORD' | docker login -u '$DOCKER_HUB_USERNAME' --password-stdin; fi" + + # Script phase + script: + # Start Selenium container + - invoke start --service selenium + # Run integration tests + - invoke integration-test --keepdb --append + # Generate integration test coverage report + - invoke unittest-coverage || travis_terminate 1 diff --git a/development/nautobot_config.py b/development/nautobot_config.py index 0ae38eb40a..7d64e05c02 100644 --- a/development/nautobot_config.py +++ b/development/nautobot_config.py @@ -5,15 +5,15 @@ from nautobot.core.settings import * from nautobot.core.settings_funcs import is_truthy, parse_redis_connection -ALLOWED_HOSTS = os.environ.get("NAUTOBOT_ALLOWED_HOSTS", "").split(" ") +ALLOWED_HOSTS = os.getenv("NAUTOBOT_ALLOWED_HOSTS", "").split(" ") DATABASES = { "default": { - "NAME": os.environ.get("NAUTOBOT_DB_NAME", "nautobot"), - "USER": os.environ.get("NAUTOBOT_DB_USER", ""), - "PASSWORD": os.environ.get("NAUTOBOT_DB_PASSWORD", ""), - "HOST": os.environ.get("NAUTOBOT_DB_HOST", "localhost"), - "PORT": os.environ.get("NAUTOBOT_DB_PORT", ""), + "NAME": os.getenv("NAUTOBOT_DB_NAME", "nautobot"), + "USER": os.getenv("NAUTOBOT_DB_USER", ""), + "PASSWORD": os.getenv("NAUTOBOT_DB_PASSWORD", ""), + "HOST": os.getenv("NAUTOBOT_DB_HOST", "localhost"), + "PORT": os.getenv("NAUTOBOT_DB_PORT", ""), "CONN_MAX_AGE": int(os.getenv("NAUTOBOT_DB_TIMEOUT", 300)), "ENGINE": os.getenv("NAUTOBOT_DB_ENGINE", "django.db.backends.postgresql"), } @@ -90,9 +90,9 @@ # REDIS CACHEOPS CACHEOPS_REDIS = parse_redis_connection(redis_database=1) -HIDE_RESTRICTED_UI = os.environ.get("HIDE_RESTRICTED_UI", False) +HIDE_RESTRICTED_UI = os.getenv("HIDE_RESTRICTED_UI", False) -SECRET_KEY = os.environ.get("NAUTOBOT_SECRET_KEY", "") +SECRET_KEY = os.getenv("NAUTOBOT_SECRET_KEY", "") # Django Debug Toolbar DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda _request: DEBUG and not TESTING} diff --git a/invoke.yml.example b/invoke.yml.example index 50f6c93039..4b9c246b3e 100644 --- a/invoke.yml.example +++ b/invoke.yml.example @@ -3,8 +3,9 @@ nautobot: local: False python_ver: "3.7" compose_dir: "/full/path/to/nautobot/development" - compose_file: "docker-compose.yml" - compose_override_file: "docker-compose.override.yml" + compose_files: + - "docker-compose.yml" + - "docker-compose.override.yml" docker_image_names_main: - "networktocode/nautobot" - "ghcr.io/nautobot/nautobot" diff --git a/nautobot/core/runner/runner.py b/nautobot/core/runner/runner.py index 57212c0ffb..b25d72e5c1 100644 --- a/nautobot/core/runner/runner.py +++ b/nautobot/core/runner/runner.py @@ -92,7 +92,7 @@ def configure_app( # normalize path if settings_envvar in os.environ: - default_config_path = os.environ.get(settings_envvar) + default_config_path = os.getenv(settings_envvar) else: default_config_path = os.path.normpath(os.path.abspath(os.path.expanduser(default_config_path))) diff --git a/nautobot/core/settings.py b/nautobot/core/settings.py index df368dd7ea..0f6525dcb3 100644 --- a/nautobot/core/settings.py +++ b/nautobot/core/settings.py @@ -51,7 +51,7 @@ BANNER_TOP = "" # Base directory wherein all created files (jobs, git repositories, file uploads, static files) will be stored) -NAUTOBOT_ROOT = os.environ.get("NAUTOBOT_ROOT", os.path.expanduser("~/.nautobot")) +NAUTOBOT_ROOT = os.getenv("NAUTOBOT_ROOT", os.path.expanduser("~/.nautobot")) CHANGELOG_RETENTION = 90 DOCS_ROOT = os.path.join(os.path.dirname(BASE_DIR), "docs") @@ -71,9 +71,9 @@ ) EXEMPT_VIEW_PERMISSIONS = [] -GIT_ROOT = os.environ.get("NAUTOBOT_GIT_ROOT", os.path.join(NAUTOBOT_ROOT, "git").rstrip("/")) +GIT_ROOT = os.getenv("NAUTOBOT_GIT_ROOT", os.path.join(NAUTOBOT_ROOT, "git").rstrip("/")) HTTP_PROXIES = None -JOBS_ROOT = os.environ.get("NAUTOBOT_JOBS_ROOT", os.path.join(NAUTOBOT_ROOT, "jobs").rstrip("/")) +JOBS_ROOT = os.getenv("NAUTOBOT_JOBS_ROOT", os.path.join(NAUTOBOT_ROOT, "jobs").rstrip("/")) MAINTENANCE_MODE = False MAX_PAGE_SIZE = 1000 @@ -271,7 +271,7 @@ } # The secret key is used to encrypt session keys and salt passwords. -SECRET_KEY = os.environ.get("SECRET_KEY") +SECRET_KEY = os.getenv("SECRET_KEY") # Default overrides ALLOWED_HOSTS = [] diff --git a/nautobot/core/tests/nautobot_config.py b/nautobot/core/tests/nautobot_config.py index 5e9c8b3791..f9b49dc0b0 100644 --- a/nautobot/core/tests/nautobot_config.py +++ b/nautobot/core/tests/nautobot_config.py @@ -17,9 +17,9 @@ "USER": os.getenv("NAUTOBOT_DB_USER", ""), "PASSWORD": os.getenv("NAUTOBOT_DB_PASSWORD", ""), "HOST": os.getenv("NAUTOBOT_DB_HOST", "localhost"), - "PORT": "", - "CONN_MAX_AGE": 300, - "ENGINE": "django.db.backends.postgresql", + "PORT": os.getenv("NAUTOBOT_DB_PORT", ""), + "CONN_MAX_AGE": int(os.getenv("NAUTOBOT_DB_TIMEOUT", 300)), + "ENGINE": os.getenv("NAUTOBOT_DB_ENGINE", "django.db.backends.postgresql"), } } diff --git a/nautobot/extras/tests/test_views.py b/nautobot/extras/tests/test_views.py index b3b7fd206a..dff1423556 100644 --- a/nautobot/extras/tests/test_views.py +++ b/nautobot/extras/tests/test_views.py @@ -802,6 +802,6 @@ def setUpTestData(cls): "slug": "computed_field_four", "label": "Computed Field Four", "template": "{{ obj.name }} is the best Site!", - "fallback_value": "💀", + "fallback_value": ":skull_emoji:", "weight": 100, } diff --git a/tasks.py b/tasks.py index 3187b4aaca..19250e15f2 100644 --- a/tasks.py +++ b/tasks.py @@ -47,8 +47,10 @@ def is_truthy(arg): "python_ver": "3.6", "local": False, "compose_dir": os.path.join(os.path.dirname(__file__), "development/"), - "compose_file": "docker-compose.yml", - "compose_override_file": "docker-compose.dev.yml", + "compose_files": [ + "docker-compose.yml", + "docker-compose.dev.yml", + ], "docker_image_names_main": [ "networktocode/nautobot", "ghcr.io/nautobot/nautobot", @@ -93,11 +95,12 @@ def docker_compose(context, command, **kwargs): command (str): Command string to append to the "docker-compose ..." command, such as "build", "up", etc. **kwargs: Passed through to the context.run() call. """ - compose_file_path = os.path.join(context.nautobot.compose_dir, context.nautobot.compose_file) - compose_command = f'docker-compose --project-name {context.nautobot.project_name} --project-directory "{context.nautobot.compose_dir}" -f "{compose_file_path}"' - compose_override_path = os.path.join(context.nautobot.compose_dir, context.nautobot.compose_override_file) - if os.path.isfile(compose_override_path): - compose_command += f' -f "{compose_override_path}"' + compose_command = f'docker-compose --project-name {context.nautobot.project_name} --project-directory "{context.nautobot.compose_dir}"' + + for compose_file in context.nautobot.compose_files: + compose_file_path = os.path.join(context.nautobot.compose_dir, compose_file) + compose_command += f' -f "{compose_file_path}"' + compose_command += f" {command}" # If `service` was passed as a kwarg, add it to the end. @@ -433,7 +436,7 @@ def unittest( """Run Nautobot unit tests.""" append_arg = " --append" if append else "" - command = f"coverage run{append_arg} --module nautobot.core.cli test {label} --config=nautobot/core/tests/nautobot_config.py" + command = f"coverage run{append_arg} --module nautobot.core.cli --config=nautobot/core/tests/nautobot_config.py test {label}" # booleans if keepdb: command += " --keepdb"