Merge pull request #222 from networktocode/develop-3.0

Merge 3.0 branch into develop

.circleci/config.yml

@@ -0,0 +1,178 @@
+---
+version: 2.1
+aliases:
+  - &linting_and_unit_tests
+    - "linting"
+    - "test"
+  - &linting_test
+    - "linting"
+  - &unit_test
+    - "test"
+executors:
+  linux:
+    docker:
+      - image: "circleci/python:3.7.10"
+orbs:
+  # The python orb contains a set of prepackaged CircleCI configuration you can use repeatedly in your configuration
+  # files
+  # Orb commands and jobs help you with common scripting around a language/tool
+  # so you dont have to copy and paste it everywhere.
+  # See the orb documentation here: https://circleci.com/developer/orbs/orb/circleci/python
+  python: "circleci/python@1.2"
+workflows:
+  TestFlow:  # This is the name of the workflow, feel free to change it to better match your workflow.
+    # Inside the workflow, you define the jobs you want to run.
+    # For more details on extending your workflow, see the configuration docs:
+    # https://circleci.com/docs/2.0/configuration-reference/#workflows
+    jobs:
+      - "linting"
+      - test:
+          requires: *linting_test
+      - "integration_nautobot":
+          requires: *unit_test
+      - "integration_netbox28":
+          requires: *unit_test
+      - "integration_netbox29":
+          requires: *unit_test
+      - "integration_netbox210":
+          requires: *unit_test
+
+jobs:
+  linting:
+    docker:
+      - image: "circleci/python:3.7.10"  # primary container for the build job
+        environment:
+          INVOKE_LOCAL: "True"
+    steps:
+      - "checkout"
+      # - python/install-packages:
+      #     pkg-manager: poetry
+      - run:
+          command: "pip install --upgrade wheel pip"
+          name: "Upgrade Pip"
+      - run:
+          command: "pip install invoke poetry toml"
+          name: "Install pip packages"
+      - run:
+          command: "poetry config virtualenvs.create false"
+          name: "Poetry config virtual envs"
+      - run:
+          command: "poetry config installer.parallel false"
+          name: "Poetry config parallel"
+      - run:
+          command: "poetry install --no-interaction --no-ansi --no-root"
+          name: "Install application"
+      - run:
+          command: "invoke black"
+      - run:
+          command: "invoke bandit"
+      - run:
+          command: "invoke pydocstyle"
+      - run:
+          command: "invoke flake8"
+      - run:
+          command: "invoke yamllint"
+      - run:
+          command: "invoke pylint"
+  test:
+    machine:
+      image: "ubuntu-2004:202104-01"
+    steps:
+      - "checkout"
+      - run:
+          name: "Change to Python 3.7 package on system"
+          command: |
+            pyenv versions
+            pyenv global 3.9.4
+            which pip
+            pip -V
+      - run:
+          name: "Install python packages"
+          command: "pip install --upgrade wheel pip toml invoke"
+      - run:
+          name: "Build Test Image"
+          command: "invoke build-image --nocache"
+      - run:
+          name: "Run Unit Tests"
+          command: "invoke pytest"
+  integration_nautobot:
+    machine:
+      image: "ubuntu-2004:202104-01"
+    steps:
+      - "checkout"
+      - run:
+          name: "Nautobot Test"
+          command: |
+            pyenv versions
+            pyenv global 3.9.4
+            which pip
+            pip -V
+            pip install --upgrade pip wheel
+            pip install invoke toml poetry pynautobot ansible==2.10.7
+            ansible-galaxy collection install networktocode.nautobot
+            poetry config virtualenvs.create false
+            poetry config installer.parallel false
+            poetry install
+            invoke nautobot-integration-tests
+          environment:
+            NAUTOBOT_VERSION: "1.0.1"
+  integration_netbox210:
+    machine:
+      image: "ubuntu-2004:202104-01"
+    steps:
+      - "checkout"
+      - run:
+          name: "NetBox 2.10 Test"
+          command: |
+            pyenv versions
+            pyenv global 3.9.4
+            which pip
+            pip -V
+            pip install invoke toml poetry pynetbox ansible==2.10.7
+            ansible-galaxy collection install git+https://github.com/netbox-community/ansible_modules.git,v3.0.0
+            poetry config virtualenvs.create false
+            poetry config installer.parallel false
+            poetry install
+            invoke integration-tests
+          environment:
+            NETBOX_VERSION: "v2.10"
+  integration_netbox29:
+    machine:
+      image: "ubuntu-2004:202104-01"
+    steps:
+      - "checkout"
+      - run:
+          name: "NetBox 2.9 Test"
+          command: |
+            pyenv versions
+            pyenv global 3.9.4
+            which pip
+            pip -V
+            pip install invoke toml poetry pynetbox ansible==2.10.7
+            ansible-galaxy collection install git+https://github.com/netbox-community/ansible_modules.git,v3.0.0
+            poetry config virtualenvs.create false
+            poetry config installer.parallel false
+            poetry install
+            invoke integration-tests
+          environment:
+            NETBOX_VERSION: "v2.9"
+  integration_netbox28:
+    machine:
+      image: "ubuntu-2004:202104-01"
+    steps:
+      - "checkout"
+      - run:
+          name: "NetBox 2.8 Test"
+          command: |
+            pyenv versions
+            pyenv global 3.9.4
+            which pip
+            pip -V
+            pip install invoke toml poetry pynetbox ansible==2.10.7
+            ansible-galaxy collection install git+https://github.com/netbox-community/ansible_modules.git,v3.0.0
+            poetry config virtualenvs.create false
+            poetry config installer.parallel false
+            poetry install
+            invoke integration-tests
+          environment:
+            NETBOX_VERSION: "v2.8"

.gitignore

@@ -149,3 +149,4 @@ dmypy.json
 .pyre/
 
 *.env
+.vscode

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2020 Network to Code <info@networktocode.com>
+Copyright 2021 Network to Code <opensource@networktocode.com>
 Network to Code, LLC
 
 Licensed under the Apache License, Version 2.0 (the "License");

README.md

@@ -12,6 +12,7 @@ The main use cases for the network importer:
 - [Configuration file](docs/configuration.md)
 - [Supported Features and Architecture](docs/architecture.md)
 - [Extensibility](docs/extensibility.md)
+- [Upgrade procedure](docs/upgrade.md)
 
 ## Questions
 

docs/architecture.md

@@ -1,12 +1,12 @@
 
 # Architecture
 
-Internally the network-importer is leveraging the [diffsync](https://github.com/networktocode/diffsync) library to compare the state of the network and the state of the Source of Truth. The diffsync library is designed to compare the state of 2 **adapters** based on a shared data-models.
+Internally the network-importer is leveraging the [diffsync](https://github.com/networktocode/diffsync) library to compare the state of the network and the state of the Source of Truth. [The diffsync library is designed to compare the state of 2 **adapters** based on a shared data-models](https://blog.networktocode.com/post/intro-to-diffing-and-syncing-data-with-diffsync/).
 
 To communicate to the network devices, the network importer is leveraging [Nornir](https://github.com/nornir-automation/nornir) and incorporate a concept of **drivers** to easily add support for more platforms if needed.
 By default the Network Importer supports 3 main actions to execute against the network devices:
 - `get_config`: Retrieve the running configuration and store it in the `config_directory` folder.
-- `get_neighbors`: Retrieve the list of all neighbors, eiher from LLDP or CDP (based on the configuration)
+- `get_neighbors`: Retrieve the list of all neighbors, either from LLDP or CDP (based on the configuration)
 - `get_vlans`: Retrieve the list of vlans present on the device.
 
 ## Internal Datamodel
@@ -22,13 +22,35 @@ The internal/shared datamodel is defined in the [network_importer/models.py](../
 
 > It's possible to extend the default models and add your own, please check the [extensibility section](extensibility.md) of the documentation
 
-## Adapters
+## Backend, Adapters & Inventory
+
+To operate the Network Importer needs 1 inventory and 2 adapters:
+- An inventory to get the list of devices to analyze and get the minimum information to connect to them (platform, address, cred
+entials ..)
+- One adapter to read the information from the network and one to read/write information to the Source of Truth backend.
+
+Since the inventory is usually leveraging the SOT, the SOT adapter and the inventory are packaged into a **backend**. Both Nautobot and Netbox are supported as backend systems. When a specific backend is selected it will update both the SOT adapter and the inventory.
+With or without leveraging the default backends, it's possible to provide your own adapter or extend one of the default adapters.
+
+> The base adapter for Network Importer is defined in [network_importer/adapters/base.py](../network_importer/adapters/base.py). The main difference with a standard diffsync adapter is that a Network Importer adapter needs to accept a NetworkImporter inventory (based on a Nornir inventory) as parameters at init time (nornir).
 
-To operate the Network Importer needs 2 adapters. 
+### Nautobot API Adapter
 
-An adapter to read the information from the network and one to read/write information to Netbox via its Rest API are provided by default but it's possible to provide your own adapter or extend either or both default adapters.
+The Nautobot API adapter is designed to read the status of a Nautobot server over its Rest API and update Nautobot based on the status of the network.
 
-> The base adapter for Network Importer is defined in [network_importer/adapters/base.py](../network_importer/adapters/base.py). The main difference with a standard diffsync adapter is that a Network Importer adapter needs to accept a nornir inventory as parameters at init time (nornir).
+The table below present the capabilities in term of : Read, Create, Update and Delete supported for each model by the netbox_api adapter.
+
+| Model              | Inherit from | Create | Read   | Update | Delete |
+|--------------------|--------------|--------|--------|--------|--------|
+| NautobotSite       | Site         | No     | Yes    | No     | No     | 
+| NautobotDevice     | Device       | No     | Yes    | No     | No     | 
+| NautobotInterface  | Interface    | Yes    | Yes    | Yes    | Yes    | 
+| NautobotIPAddress  | IPAddress    | Yes    | Yes    | Yes    | Yes    | 
+| NautobotPrefix     | Prefix       | Yes    | Yes    | Yes    | No     | 
+| NautobotVlan       | Vlan         | Yes    | Yes    | Yes    | No     | 
+| NautobotCable      | Cable        | Yes    | Yes    | No     | No     | 
+
+> It's possible to extend the default models and add your own, please check the [extensibility section](extensibility.md) of the documentation
 
 ### NetBox API Adapter
 
@@ -54,7 +76,7 @@ The Network Importer Adapter is designed to read the status of the network prima
 
 ## Drivers
 
-The communicate with the network devices, the network importer is leveraging Nornir and support some drivers per platform to easily support more device type.
+The communicate with the network devices, the network-importer is leveraging Nornir and support some drivers per platform to easily support more device type.
 
 Each driver, should support each of the following actions: 
 - `get_config`: Retrieve the running configuration and store it in the `config_directory` folder.
@@ -72,7 +94,7 @@ By default, 4 drivers are available `default`, `cisco_default`, `juniper_junos`
 | juniper_junos   | juniper_junos  |
 | arista_eos      | arista_eos     |
 
-> The name of the platform must match the name of the slug platform defined in NetBox for a given device
+> The name of the platform must match the name of the slug platform defined in the inventory for a given device
 
 ### Drivers available by default