Remove 3.5 and xenial support (SC-711)

.travis.yml

@@ -75,17 +75,17 @@ matrix:
             - cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc
           script:
             # Ubuntu LTS: Build
-            - ./packages/bddeb -S -d --release xenial
+            - ./packages/bddeb -S -d --release bionic
             - |
                 needs_caching=false
-                if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then
+                if [ -e "$TRAVIS_BUILD_DIR/chroots/bionic-amd64.tar" ]; then
                     # If we have a cached chroot, move it into place
-                    sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64
-                    sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64
+                    sudo mkdir -p /var/lib/schroot/chroots/bionic-amd64
+                    sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/bionic-amd64.tar" -C /var/lib/schroot/chroots/bionic-amd64
                     # Write its configuration
-                    cat > sbuild-xenial-amd64 << EOM
-                [xenial-amd64]
-                description=xenial-amd64
+                    cat > sbuild-bionic-amd64 << EOM
+                [bionic-amd64]
+                description=bionic-amd64
                 groups=sbuild,root,admin
                 root-groups=sbuild,root,admin
                 # Uncomment these lines to allow members of these groups to access
@@ -95,40 +95,40 @@ matrix:
                 type=directory
                 profile=sbuild
                 union-type=overlay
-                directory=/var/lib/schroot/chroots/xenial-amd64
+                directory=/var/lib/schroot/chroots/bionic-amd64
                 EOM
-                    sudo mv sbuild-xenial-amd64 /etc/schroot/chroot.d/
-                    sudo chown root /etc/schroot/chroot.d/sbuild-xenial-amd64
+                    sudo mv sbuild-bionic-amd64 /etc/schroot/chroot.d/
+                    sudo chown root /etc/schroot/chroot.d/sbuild-bionic-amd64
                     # And ensure it's up-to-date.
-                    before_pkgs="$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)"
-                    sudo schroot -c source:xenial-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade"
-                    after_pkgs=$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)
+                    before_pkgs="$(sudo schroot -c source:bionic-amd64 -d / dpkg -l | sha256sum)"
+                    sudo schroot -c source:bionic-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade"
+                    after_pkgs=$(sudo schroot -c source:bionic-amd64 -d / dpkg -l | sha256sum)
                     if [ "$before_pkgs" != "$after_pkgs" ]; then
                         needs_caching=true
                     fi
                 else
                     # Otherwise, create the chroot
-                    sudo -E su $USER -c 'mk-sbuild xenial'
+                    sudo -E su $USER -c 'mk-sbuild bionic'
                     needs_caching=true
                 fi
                 # If there are changes to the schroot (or it's entirely new),
                 # tar up the schroot (to preserve ownership/permissions) and
                 # move it into the cached dir; no need to compress it because
                 # Travis will do that anyway
                 if [ "$needs_caching" = "true" ]; then
-                    sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 .
+                    sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/bionic-amd64.tar" -C /var/lib/schroot/chroots/bionic-amd64 .
                 fi
             # Use sudo to get a new shell where we're in the sbuild group
             # Don't run integration tests when build fails
             - |
-                sudo -E su $USER -c 'DEB_BUILD_OPTIONS=nocheck sbuild --nolog --no-run-lintian --no-run-autopkgtest --verbose --dist=xenial cloud-init_*.dsc' &&
+                sudo -E su $USER -c 'DEB_BUILD_OPTIONS=nocheck sbuild --nolog --no-run-lintian --no-run-autopkgtest --verbose --dist=bionic cloud-init_*.dsc' &&
                   ssh-keygen -P "" -q -f ~/.ssh/id_rsa                                                         &&
                   sg lxd -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls *.deb)" tox -e integration-tests-ci'
-        - python: 3.5
+        - python: 3.6
           env:
-              TOXENV=xenial
-              PYTEST_ADDOPTS=-v  # List all tests run by pytest
-          dist: xenial
+            TOXENV=lowest-supported
+            PYTEST_ADDOPTS=-v  # List all tests run by pytest
+          dist: bionic
         - python: 3.6
           env: TOXENV=flake8
         - python: 3.6
@@ -145,4 +145,3 @@ matrix:
         - python: 3.9
         - python: 3.8
         - python: 3.7
-        - python: 3.5

cloudinit/config/cc_apt_configure.py

@@ -133,7 +133,7 @@
               source1:
                   keyid: 'keyid'
                   keyserver: 'keyserverurl'
-                  source: 'deb [signed-by=$KEY_FILE] http://<url>/ xenial main'
+                  source: 'deb [signed-by=$KEY_FILE] http://<url>/ bionic main'
               source2:
                   source: 'ppa:<ppa-name>'
               source3:

cloudinit/distros/debian.py

@@ -257,7 +257,7 @@ def package_command(self, command, args=None, pkgs=None):
             pkgs = []
 
         e = os.environ.copy()
-        # See: http://manpages.ubuntu.com/manpages/xenial/man7/debconf.7.html
+        # See: http://manpages.ubuntu.com/manpages/bionic/man7/debconf.7.html
         e["DEBIAN_FRONTEND"] = "noninteractive"
 
         wcfg = self.get_option("apt_get_wrapper", APT_GET_WRAPPER)

conftest.py

@@ -65,7 +65,7 @@ def closest_marker_first_arg_or(request, marker_name: str, default):
         return result[0]
 
 
-@pytest.yield_fixture(autouse=True)
+@pytest.fixture(autouse=True)
 def disable_subp_usage(request, fixture_utils):
     """
     Across all (pytest) tests, ensure that subp.subp is not invoked.
@@ -166,7 +166,7 @@ def fixture_utils():
     return _FixtureUtils
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def httpretty():
     """
     Enable HTTPretty for duration of the testcase, resetting before and after.

doc/examples/cloud-config-apt.txt

@@ -254,7 +254,7 @@ apt:
       #
       # Creates a file in /etc/apt/sources.list.d/ for the sources list entry
       # based on the key: "/etc/apt/sources.list.d/curtin-dev-ppa.list"
-      source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main"
+      source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu bionic main"
 
       # 2.2 keyid
       #
@@ -282,13 +282,13 @@ apt:
       # They will be replaced with the default or specified mirrors and the
       # running release.
       # The entry below would be possibly turned into:
-      #   source: deb http://archive.ubuntu.com/ubuntu xenial multiverse
+      #   source: deb http://archive.ubuntu.com/ubuntu bionic multiverse
       source: deb [signed-by=$KEY_FILE] $MIRROR $RELEASE multiverse
       keyid: F430BBA5
 
     my-repo3.list:
       # this would have the same end effect as 'ppa:curtin-dev/test-archive'
-      source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main"
+      source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu bionic main"
       keyid: F430BBA5 # GPG key ID published on the key server
       filename: curtin-dev-ppa.list
 

doc/examples/cloud-config-chef.txt

@@ -4,9 +4,6 @@
 # list of recipes when the instance boots for the first time.
 # Make sure that this file is valid yaml before starting instances.
 # It should be passed as user-data when starting the instance.
-#
-# This example assumes the instance is 16.04 (xenial)
-
 
 # The default is to install from packages.
 
@@ -55,7 +52,7 @@ chef:
 
   # Valid values are 'accept' and 'accept-no-persist'
   chef_license: "accept"
-  
+
   # Valid values are 'gems' and 'packages' and 'omnibus'
   install_type: "packages"
 

doc/rtd/topics/debugging.rst

@@ -88,7 +88,7 @@ To quickly obtain a cloud-init log try using lxc on any ubuntu system:
 
 .. code-block:: shell-session
 
-    $ lxc init ubuntu-daily:xenial x1
+    $ lxc init ubuntu-daily:focal x1
     $ lxc start x1
     $ # Take lxc's cloud-init.log and pipe it to the analyzer
     $ lxc file pull x1/var/log/cloud-init.log - | cloud-init analyze dump -i -
@@ -104,13 +104,13 @@ To quickly analyze a KVM a cloud-init log:
 
 .. code-block:: shell-session
 
-    $ wget https://cloud-images.ubuntu.com/daily/server/xenial/current/xenial-server-cloudimg-amd64.img
+    $ wget https://cloud-images.ubuntu.com/daily/server/focal/current/focal-server-cloudimg-amd64.img
 
 2. Create a snapshot image to preserve the original cloud-image
 
 .. code-block:: shell-session
 
-    $ qemu-img create -b xenial-server-cloudimg-amd64.img -f qcow2 \
+    $ qemu-img create -b focal-server-cloudimg-amd64.img -f qcow2 \
     test-cloudinit.qcow2
 
 3. Create a seed image with metadata using `cloud-localds`

doc/rtd/topics/testing.rst

@@ -54,51 +54,29 @@ Test Layout
 * pytest tests should use bare ``assert`` statements, to take advantage
   of pytest's `assertion introspection`_
 
-  * For ``==`` and other commutative assertions, the expected value
-    should be placed before the value under test:
-    ``assert expected_value == function_under_test()``
-
-
 ``pytest`` Version Gotchas
 --------------------------
 
-As we still support Ubuntu 16.04 (Xenial Xerus), we can only use pytest
-features that are available in v2.8.7.  This is an inexhaustive list of
+As we still support Ubuntu 18.04 (Bionic Beaver), we can only use pytest
+features that are available in v3.3.2.  This is an inexhaustive list of
 ways in which this may catch you out:
 
-* Support for using ``yield`` in ``pytest.fixture`` functions was only
-  introduced in `pytest 3.0`_.  Such functions must instead use the
-  ``pytest.yield_fixture`` decorator.
-
 * Only the following built-in fixtures are available [#fixture-list]_:
 
   * ``cache``
   * ``capfd``
-  * ``caplog`` (provided by ``python3-pytest-catchlog`` on xenial)
+  * ``capfdbinary``
+  * ``caplog``
   * ``capsys``
+  * ``capsysbinary``
+  * ``doctest_namespace``
   * ``monkeypatch``
   * ``pytestconfig``
   * ``record_xml_property``
   * ``recwarn``
   * ``tmpdir_factory``
   * ``tmpdir``
 
-* On xenial, the objects returned by the ``tmpdir`` fixture cannot be
-  used where paths are required; they are rejected as invalid paths.
-  You must instead use their ``.strpath`` attribute.
-
-  * For example, instead of ``util.write_file(tmpdir.join("some_file"),
-    ...)``, you should write
-    ``util.write_file(tmpdir.join("some_file").strpath, ...)``.
-
-* The `pytest.param`_ function cannot be used. It was introduced in
-  pytest 3.1, which means it is not available on xenial.  The more
-  limited mechanism it replaced was removed in pytest 4.0, so is not
-  available in focal or later.  The only available alternatives are to
-  write mark-requiring test instances as completely separate tests,
-  without utilising parameterisation, or to apply the mark to the
-  entire parameterized test (and therefore every test instance).
-
 Mocking and Assertions
 ----------------------
 
@@ -168,9 +146,9 @@ Test Argument Ordering
 .. [#fixture-list] This list of fixtures (with markup) can be
    reproduced by running::
 
-     py.test-3 --fixtures -q | grep "^[^ -]" | grep -v '\(no\|capturelog\)' | sort | sed 's/.*/* ``\0``/'
+     python3 -m pytest  --fixtures -q | grep "^[^ -]" | grep -v 'no tests ran in' | sort | sed 's/ \[session scope\]//g;s/.*/* ``\0``/g'
 
-   in a xenial lxd container with python3-pytest-catchlog installed.
+   in an ubuntu lxd container with python3-pytest installed.
 
 .. _pytest: https://docs.pytest.org/
 .. _pytest fixtures: https://docs.pytest.org/en/latest/fixture.html

tests/integration_tests/bugs/test_gh626.py

@@ -8,7 +8,6 @@
 import yaml
 
 from tests.integration_tests import random_mac_address
-from tests.integration_tests.clouds import ImageSpecification
 from tests.integration_tests.instances import IntegrationInstance
 
 MAC_ADDRESS = random_mac_address()
@@ -29,7 +28,6 @@
     ethernet-wol g"""
 
 
-@pytest.mark.sru_2020_11
 @pytest.mark.lxd_container
 @pytest.mark.lxd_vm
 @pytest.mark.lxd_config_dict(
@@ -39,11 +37,6 @@
     }
 )
 def test_wakeonlan(client: IntegrationInstance):
-    if ImageSpecification.from_os_image().release == "xenial":
-        eni = client.execute("cat /etc/network/interfaces.d/50-cloud-init.cfg")
-        assert eni.endswith(EXPECTED_ENI_END)
-        return
-
     netplan_cfg = client.execute("cat /etc/netplan/50-cloud-init.yaml")
     netplan_yaml = yaml.safe_load(netplan_cfg)
     assert "wakeonlan" in netplan_yaml["network"]["ethernets"]["eth0"]

tests/integration_tests/bugs/test_lp1898997.py

@@ -47,8 +47,6 @@
 @pytest.mark.lxd_vm
 @pytest.mark.lxd_use_exec
 @pytest.mark.not_bionic
-@pytest.mark.not_xenial
-@pytest.mark.sru_2020_11
 @pytest.mark.ubuntu
 class TestInterfaceListingWithOpenvSwitch:
     def test_ovs_member_interfaces_not_excluded(self, client):

tests/integration_tests/conftest.py

@@ -45,17 +45,6 @@
 
 session_start_time = datetime.datetime.now().strftime("%y%m%d%H%M%S")
 
-XENIAL_LXD_VM_EXEC_MSG = """\
-The default xenial images do not support `exec` for LXD VMs.
-
-Specify an image known to work using:
-
-    OS_IMAGE=<image id>::ubuntu::xenial
-
-You can re-run specifically tests that require this by passing `-m
-lxd_use_exec` to pytest.
-"""
-
 
 def pytest_runtest_setup(item):
     """Skip tests on unsupported clouds.
@@ -101,7 +90,7 @@ def disable_subp_usage(request):
     pass
 
 
-@pytest.yield_fixture(scope="session")
+@pytest.fixture(scope="session")
 def session_cloud():
     if integration_settings.PLATFORM not in platforms.keys():
         raise ValueError(
@@ -246,16 +235,6 @@ def _client(request, fixture_utils, session_cloud: IntegrationCloud):
     if lxd_use_exec is not None:
         if not isinstance(session_cloud, _LxdIntegrationCloud):
             pytest.skip("lxd_use_exec requires LXD")
-        if isinstance(session_cloud, LxdVmCloud):
-            image_spec = ImageSpecification.from_os_image()
-            if image_spec.release == image_spec.image_id == "xenial":
-                # Why fail instead of skip?  We expect that skipped tests will
-                # be run in a different one of our usual battery of test runs
-                # (e.g. LXD-only tests are skipped on EC2 but will run in our
-                # normal LXD test runs).  This is not true of this test: it
-                # can't run in our usual xenial LXD VM test run, and it may not
-                # run anywhere else.  A failure flags up this discrepancy.
-                pytest.fail(XENIAL_LXD_VM_EXEC_MSG)
         launch_kwargs["execute_via_ssh"] = False
     local_launch_kwargs = {}
     if lxd_setup is not None:
@@ -276,21 +255,21 @@ def _client(request, fixture_utils, session_cloud: IntegrationCloud):
         _collect_logs(instance, request.node.nodeid, test_failed)
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def client(request, fixture_utils, session_cloud, setup_image):
     """Provide a client that runs for every test."""
     with _client(request, fixture_utils, session_cloud) as client:
         yield client
 
 
-@pytest.yield_fixture(scope="module")
+@pytest.fixture(scope="module")
 def module_client(request, fixture_utils, session_cloud, setup_image):
     """Provide a client that runs once per module."""
     with _client(request, fixture_utils, session_cloud) as client:
         yield client
 
 
-@pytest.yield_fixture(scope="class")
+@pytest.fixture(scope="class")
 def class_client(request, fixture_utils, session_cloud, setup_image):
     """Provide a client that runs once per class."""
     with _client(request, fixture_utils, session_cloud) as client:

tests/integration_tests/datasources/test_lxd_discovery.py

@@ -59,13 +59,9 @@ def test_lxd_datasource_discovery(client: IntegrationInstance):
     )
     if (
         client.settings.PLATFORM == "lxd_vm"
-        and ImageSpecification.from_os_image().release
-        in (
-            "xenial",
-            "bionic",
-        )
+        and ImageSpecification.from_os_image().release == "bionic"
     ):
-        # pycloudlib injects user.vendor_data for lxd_vm on bionic and xenial
+        # pycloudlib injects user.vendor_data for lxd_vm on bionic
         # to start the lxd-agent.
         # https://github.com/canonical/pycloudlib/blob/main/pycloudlib/\
         #    lxd/defaults.py#L13-L27