From 7df4a94ad55f21df69331d0b88700ce6fdb20415 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 01:23:05 +0530 Subject: [PATCH 01/11] Migrate packaging to pyproject.toml (PEP 621) Replace setup.py/setup.cfg with a PEP 621 pyproject.toml using setuptools build backend. Version is read dynamically from cloudbridge.__version__. Coverage config moves into [tool.coverage.run]; flake8 stays in setup.cfg (no pyproject support). Drop stale Py2/old-Py3 classifiers and bdist_wheel universal flag; declare requires-python = ">=3.13" to match tox.ini and CI. Update deploy workflow to use `python -m build`, and pip-cache keys in integration workflows to hash pyproject.toml. --- .github/workflows/deploy.yaml | 6 +- .github/workflows/integration-cloud.yaml | 2 +- .github/workflows/integration.yaml | 2 +- pyproject.toml | 110 ++++++++++++++++++++++ setup.cfg | 11 --- setup.py | 114 ----------------------- tox.ini | 14 ++- 7 files changed, 127 insertions(+), 132 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index b0504433..147c9bbf 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -23,11 +23,11 @@ jobs: python-version: 3.13 - name: Install dependencies run: | - python3 -m pip install --upgrade pip setuptools - python3 -m pip install --upgrade twine wheel + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade build twine - name: Create and check packages run: | - python3 setup.py sdist bdist_wheel + python3 -m build twine check dist/* ls -l dist - name: Publish distribution 📦 to Test PyPI diff --git a/.github/workflows/integration-cloud.yaml b/.github/workflows/integration-cloud.yaml index 74b6caa2..54b77ac1 100644 --- a/.github/workflows/integration-cloud.yaml +++ b/.github/workflows/integration-cloud.yaml @@ -73,7 +73,7 @@ jobs: uses: actions/cache@v5 with: path: ~/.cache/pip - key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('**/setup.py', '**/requirements.txt') }} + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/requirements.txt') }} - name: Install required packages run: pip install tox diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 33751468..0c952b63 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -71,7 +71,7 @@ jobs: uses: actions/cache@v5 with: path: ~/.cache/pip - key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('**/setup.py', '**/requirements.txt') }} + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/requirements.txt') }} - name: Install required packages run: pip install tox diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..b6554734 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,110 @@ +[build-system] +requires = ["setuptools>=77.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "cloudbridge" +description = "A simple layer of abstraction over multiple cloud providers." +readme = "README.rst" +license = "MIT" +license-files = ["LICENSE"] +requires-python = ">=3.13" +authors = [ + { name = "Galaxy and GVL Projects", email = "help@genome.edu.au" }, +] +keywords = ["cloud", "aws", "azure", "gcp", "openstack", "iaas"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ + "six>=1.11", + "tenacity>=6.0", + "deprecation>=2.0.7", + "pyeventsystem<2", +] +dynamic = ["version"] + +[project.urls] +Homepage = "http://cloudbridge.cloudve.org/" +Source = "https://github.com/CloudVE/cloudbridge" +Issues = "https://github.com/CloudVE/cloudbridge/issues" + +[project.optional-dependencies] +aws = [ + "boto3>=1.9.86,<2.0.0", +] +# Install azure>=3.0.0 package to find which of the azure libraries listed +# below are compatible with each other. List individual libraries instead +# of using the azure umbrella package to speed up installation. +# Minimums match SDK generation tested against the model-class +# serialization fixes in cloudbridge/providers/azure/. Older SDKs may +# work but are not covered by integration tests. +azure = [ + "azure-identity>=1.20.0,<2.0.0", + "azure-common>=1.1.28,<2.0.0", + "azure-core>=1.30.0,<2.0.0", + "azure-mgmt-devtestlabs>=9.0.0,<10.0.0", + "azure-mgmt-resource>=23.0.0,<26.0.0", + "azure-mgmt-subscription>=3.0.0,<4.0.0", + "azure-mgmt-compute>=34.0.0,<39.0.0", + "azure-mgmt-network>=28.0.0,<31.0.0", + "azure-mgmt-storage>=22.0.0,<25.0.0", + "azure-storage-blob>=12.20.0,<13.0.0", + "azure-data-tables>=12.4.0,<13.0.0", + "paramiko<6.0.0", +] +gcp = [ + "google-api-python-client>=2.0,<3.0.0", +] +# Minimums match SDK generation tested against the OpenStack +# provider fixes in cloudbridge/providers/openstack/. The previous +# floors were circa-2018 and exposed Nova/Neutron APIs (e.g. the +# add_floating_ip_to_server action) that are gone from any modern +# OpenStack deployment. +openstack = [ + "openstacksdk>=3.0.0,<5.0.0", + "python-novaclient>=17.0.0,<20.0", + "python-swiftclient>=4.0.0,<5.0", + "python-neutronclient>=11.0.0,<13.0", + "python-keystoneclient>=4.0.0,<7.0", +] +full = [ + "cloudbridge[aws,azure,gcp,openstack]", +] +# httpretty is required with/for moto 1.0.0 or AWS tests fail +dev = [ + "cloudbridge[full]", + "tox>=4.0.0", + "pytest", + "moto[ec2,s3]>=5.0.0", + "packaging", + "sphinx>=1.3.1", + "pydevd", + "flake8>=3.3.0", + "flake8-import-order>=0.12", +] + +[tool.setuptools.dynamic] +version = { attr = "cloudbridge.__version__" } + +[tool.setuptools.packages.find] +include = ["cloudbridge*"] +exclude = ["tests*"] + +[tool.coverage.run] +branch = true +source = ["cloudbridge"] +omit = [ + "cloudbridge/interfaces/*", + "cloudbridge/__init__.py", +] +parallel = true diff --git a/setup.cfg b/setup.cfg index 4fc81ca2..c12226c0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,14 +1,3 @@ -[coverage:run] -branch = True -source = cloudbridge -omit = - cloudbridge/interfaces/* - cloudbridge/__init__.py -parallel = True - -[bdist_wheel] -universal = 1 - [flake8] application_import_names = cloudbridge, tests max-line-length = 120 diff --git a/setup.py b/setup.py deleted file mode 100644 index 95591370..00000000 --- a/setup.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -CloudBridge provides a uniform interface to multiple IaaS cloud providers. -""" - -import ast -import os -import re - -from setuptools import find_packages, setup - -# Cannot use "from cloudbridge import get_version" because that would try to -# import the six package which may not be installed yet. -reg = re.compile(r'__version__\s*=\s*(.+)') -with open(os.path.join('cloudbridge', '__init__.py')) as f: - for line in f: - m = reg.match(line) - if m: - version = ast.literal_eval(m.group(1)) - break - -REQS_BASE = [ - 'six>=1.11', - 'tenacity>=6.0', - 'deprecation>=2.0.7', - 'pyeventsystem<2' -] -REQS_AWS = [ - 'boto3>=1.9.86,<2.0.0' -] -# Install azure>=3.0.0 package to find which of the azure libraries listed -# below are compatible with each other. List individual libraries instead -# of using the azure umbrella package to speed up installation. -REQS_AZURE = [ - # Minimums match SDK generation tested against the model-class - # serialization fixes in cloudbridge/providers/azure/. Older SDKs may - # work but are not covered by integration tests. - 'azure-identity>=1.20.0,<2.0.0', - 'azure-common>=1.1.28,<2.0.0', - 'azure-core>=1.30.0,<2.0.0', - 'azure-mgmt-devtestlabs>=9.0.0,<10.0.0', - 'azure-mgmt-resource>=23.0.0,<26.0.0', - 'azure-mgmt-subscription>=3.0.0,<4.0.0', - 'azure-mgmt-compute>=34.0.0,<39.0.0', - 'azure-mgmt-network>=28.0.0,<31.0.0', - 'azure-mgmt-storage>=22.0.0,<25.0.0', - 'azure-storage-blob>=12.20.0,<13.0.0', - 'azure-data-tables>=12.4.0,<13.0.0', - 'paramiko<6.0.0' -] -REQS_GCP = [ - 'google-api-python-client>=2.0,<3.0.0' -] -REQS_OPENSTACK = [ - # Minimums match SDK generation tested against the OpenStack - # provider fixes in cloudbridge/providers/openstack/. The previous - # floors were circa-2018 and exposed Nova/Neutron APIs (e.g. the - # add_floating_ip_to_server action) that are gone from any modern - # OpenStack deployment. - 'openstacksdk>=3.0.0,<5.0.0', - 'python-novaclient>=17.0.0,<20.0', - 'python-swiftclient>=4.0.0,<5.0', - 'python-neutronclient>=11.0.0,<13.0', - 'python-keystoneclient>=4.0.0,<7.0' -] -REQS_FULL = REQS_AWS + REQS_GCP + REQS_OPENSTACK + REQS_AZURE -# httpretty is required with/for moto 1.0.0 or AWS tests fail -REQS_DEV = ([ - 'tox>=4.0.0', - 'pytest', - 'moto[ec2,s3]>=5.0.0', - 'packaging', - 'sphinx>=1.3.1', - 'pydevd', - 'flake8>=3.3.0', - 'flake8-import-order>=0.12'] + REQS_FULL -) - -setup( - name='cloudbridge', - version=version, - description='A simple layer of abstraction over multiple cloud providers.', - long_description=__doc__, - author='Galaxy and GVL Projects', - author_email='help@genome.edu.au', - url='http://cloudbridge.cloudve.org/', - install_requires=REQS_BASE, - extras_require={ - ':python_version<"3.3"': ['ipaddress'], - 'azure': REQS_AZURE, - 'gcp': REQS_GCP, - 'aws': REQS_AWS, - 'openstack': REQS_OPENSTACK, - 'full': REQS_FULL, - 'dev': REQS_DEV - }, - packages=find_packages(), - license='MIT', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: Implementation :: CPython'], - test_suite="tests" -) diff --git a/tox.ini b/tox.ini index 7006f6be..536153b9 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ envlist = {py3.13,pypy}-{aws,azure,gcp,openstack,mock},lint [testenv] -commands = # see setup.cfg for options sent to pytest and coverage +commands = # see pyproject.toml for coverage options; setup.cfg for flake8 coverage run --source=cloudbridge -m pytest -v {posargs:-n 5 tests/} # Combine parallel-mode data files and emit Cobertura XML for upload # by coverallsapp/github-action in CI. Locally this produces @@ -31,6 +31,16 @@ passenv = aws: CB_IMAGE_AWS aws: CB_VM_TYPE_AWS aws: CB_PLACEMENT_AWS + # Standard boto3 credential env vars — set by aws-actions/configure-aws-credentials + # in CI (OIDC) and by `aws configure` locally. Required because tox does not + # forward arbitrary env vars to the test process. + aws: AWS_ACCESS_KEY_ID + aws: AWS_SECRET_ACCESS_KEY + aws: AWS_SESSION_TOKEN + aws: AWS_REGION + aws: AWS_DEFAULT_REGION + # Cloudbridge-specific names, kept for backward compatibility with local dev + # configs that set these. aws: AWS_ACCESS_KEY aws: AWS_SECRET_KEY azure: CB_IMAGE_AZURE @@ -77,5 +87,5 @@ deps = pytest-xdist [testenv:lint] -commands = flake8 cloudbridge tests setup.py +commands = flake8 cloudbridge tests deps = flake8 From 5120e8b095cd594e45f2edd63bb34444921b9c29 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 01:23:33 +0530 Subject: [PATCH 02/11] Drop six dependency Replace six.string_types/text_type/integer_types with str/int; six.u()-on-string-types branches were dead in Py3 and deleted. six.reraise inside an active except becomes a bare raise; the middleware Py2/Py3 fork collapses to `raise cb_ex from e`. Switch gcp/helpers from six.moves.urllib.parse to stdlib urllib.parse, and inline six.ensure_binary. Also remove the configparser try/except shim in base/provider.py that fell back to Py2's SafeConfigParser. --- cloudbridge/base/helpers.py | 19 +++---------------- cloudbridge/base/middleware.py | 16 ++++------------ cloudbridge/base/provider.py | 10 +--------- cloudbridge/base/resources.py | 4 +--- cloudbridge/providers/gcp/helpers.py | 9 +++++---- pyproject.toml | 1 - tests/test_block_store_service.py | 10 ++++------ tests/test_cloud_helpers.py | 4 +--- tests/test_compute_service.py | 6 ++---- tests/test_region_service.py | 4 +--- tests/test_vm_types_service.py | 8 +++----- 11 files changed, 25 insertions(+), 66 deletions(-) diff --git a/cloudbridge/base/helpers.py b/cloudbridge/base/helpers.py index d8389108..ae249521 100644 --- a/cloudbridge/base/helpers.py +++ b/cloudbridge/base/helpers.py @@ -3,7 +3,6 @@ import logging import os import re -import sys from contextlib import contextmanager from cryptography.hazmat.backends import default_backend @@ -12,8 +11,6 @@ from deprecation import deprecated -import six - import cloudbridge from ..interfaces.exceptions import InvalidParamException @@ -50,7 +47,7 @@ def filter_by(prop_name, kwargs, objs): """ prop_val = kwargs.pop(prop_name, None) if prop_val: - if isinstance(prop_val, six.string_types): + if isinstance(prop_val, str): regex = fnmatch.translate(prop_val) results = [o for o in objs if getattr(o, prop_name) @@ -101,12 +98,11 @@ def cleanup_action(cleanup_func): try: yield except Exception: - ex_class, ex_val, ex_traceback = sys.exc_info() try: cleanup_func() except Exception: log.exception("Error during exception cleanup: ") - six.reraise(ex_class, ex_val, ex_traceback) + raise try: cleanup_func() except Exception: @@ -117,11 +113,6 @@ def get_env(varname, default_value=None): """ Return the value of the environment variable or default_value. - This is a helper method that wraps ``os.environ.get`` to ensure type - compatibility across py2 and py3. For py2, any value obtained from an - environment variable, ensure ``unicode`` type and ``str`` for py3. The - casting is done only for string variables. - :type varname: ``str`` :param varname: Name of the environment variable for which to check. @@ -131,11 +122,7 @@ def get_env(varname, default_value=None): :return: Value of the supplied environment if found; value of ``default_value`` otherwise. """ - value = os.environ.get(varname, default_value) - if isinstance(value, six.string_types) and not isinstance( - value, six.text_type): - return six.u(value) - return value + return os.environ.get(varname, default_value) # Alias deprecation decorator, following: diff --git a/cloudbridge/base/middleware.py b/cloudbridge/base/middleware.py index d7671944..7ec339ec 100644 --- a/cloudbridge/base/middleware.py +++ b/cloudbridge/base/middleware.py @@ -1,12 +1,9 @@ import logging -import sys from pyeventsystem.middleware import dispatch as pyevent_dispatch from pyeventsystem.middleware import intercept from pyeventsystem.middleware import observe -import six - from ..interfaces.exceptions import CloudBridgeBaseException log = logging.getLogger(__name__) @@ -46,12 +43,7 @@ def wrap_exception(self, event_args, *args, **kwargs): except Exception as e: if isinstance(e, CloudBridgeBaseException): raise - else: - ex_type, ex_value, traceback = sys.exc_info() - cb_ex = CloudBridgeBaseException( - "CloudBridgeBaseException: {0} from exception type: {1}" - .format(ex_value, ex_type)) - if sys.version_info >= (3, 0): - six.raise_from(cb_ex, e) - else: - six.reraise(CloudBridgeBaseException, cb_ex, traceback) + cb_ex = CloudBridgeBaseException( + "CloudBridgeBaseException: {0} from exception type: {1}" + .format(e, type(e))) + raise cb_ex from e diff --git a/cloudbridge/base/provider.py b/cloudbridge/base/provider.py index 3436f27d..0704fb2a 100644 --- a/cloudbridge/base/provider.py +++ b/cloudbridge/base/provider.py @@ -3,16 +3,11 @@ import functools import logging import os +from configparser import ConfigParser from os.path import expanduser -try: - from configparser import ConfigParser -except ImportError: # Python 2 - from ConfigParser import SafeConfigParser as ConfigParser from pyeventsystem.middleware import SimpleMiddlewareManager -import six - from ..base.middleware import ExceptionWrappingMiddleware from ..interfaces import CloudProvider from ..interfaces.exceptions import ProviderConnectionException @@ -206,7 +201,4 @@ def _get_config_value(self, key, default_value=None): elif (self._config_parser.has_option(self.PROVIDER_ID, key) and self._config_parser.get(self.PROVIDER_ID, key)): value = self._config_parser.get(self.PROVIDER_ID, key) - if isinstance(value, six.string_types) and not isinstance( - value, six.text_type): - return six.u(value) return value diff --git a/cloudbridge/base/resources.py b/cloudbridge/base/resources.py index b09bcdcc..d7759a42 100644 --- a/cloudbridge/base/resources.py +++ b/cloudbridge/base/resources.py @@ -10,8 +10,6 @@ import time import uuid -import six - from cloudbridge.interfaces.exceptions import \ InvalidConfigurationException from cloudbridge.interfaces.exceptions import InvalidLabelException @@ -378,7 +376,7 @@ def _validate_volume_device(self, source=None, is_root=None, raise InvalidConfigurationException( "Source must be a Snapshot, Volume, MachineImage, or None.") if size: - if not isinstance(size, six.integer_types) or not size > 0: + if not isinstance(size, int) or not size > 0: log.exception("InvalidConfigurationException raised: " "size argument must be an integer greater than " "0. Got type %s and value %s.", type(size), size) diff --git a/cloudbridge/providers/gcp/helpers.py b/cloudbridge/providers/gcp/helpers.py index d92dd7bb..57a711f4 100644 --- a/cloudbridge/providers/gcp/helpers.py +++ b/cloudbridge/providers/gcp/helpers.py @@ -3,9 +3,7 @@ import datetime import hashlib import re - -import six -from six.moves.urllib.parse import quote +from urllib.parse import quote from googleapiclient.errors import HttpError @@ -204,7 +202,10 @@ def generate_signed_url(credentials, bucket_name, object_name, # max allowed expiration time is 7 days expiration = 604800 - escaped_object_name = quote(six.ensure_binary(object_name), safe=b'/~') + escaped_object_name = quote( + object_name.encode('utf-8') if isinstance(object_name, str) + else object_name, + safe=b'/~') canonical_uri = '/{}'.format(escaped_object_name) datetime_now = datetime.datetime.utcnow() diff --git a/pyproject.toml b/pyproject.toml index b6554734..964de431 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "six>=1.11", "tenacity>=6.0", "deprecation>=2.0.7", "pyeventsystem<2", diff --git a/tests/test_block_store_service.py b/tests/test_block_store_service.py index df9193a1..74c7ebb8 100644 --- a/tests/test_block_store_service.py +++ b/tests/test_block_store_service.py @@ -1,7 +1,5 @@ import time -import six - from cloudbridge.base import helpers as cb_helpers from cloudbridge.factory import ProviderList from cloudbridge.interfaces import SnapshotState @@ -107,13 +105,13 @@ def test_volume_properties(self): with cb_helpers.cleanup_action(lambda: test_vol.delete()): test_vol.wait_till_ready() self.assertTrue( - isinstance(test_vol.size, six.integer_types) and + isinstance(test_vol.size, int) and test_vol.size >= 0, "Volume.size must be a positive number, but got %s" % test_vol.size) self.assertTrue( test_vol.description is None or - isinstance(test_vol.description, six.string_types), + isinstance(test_vol.description, str), "Volume.description must be None or a string. Got: %s" % test_vol.description) self.assertIsNone(test_vol.source) @@ -207,14 +205,14 @@ def cleanup_snap(snap): with cb_helpers.cleanup_action(lambda: cleanup_snap(test_snap)): test_snap.wait_till_ready() - self.assertTrue(isinstance(test_vol.size, six.integer_types)) + self.assertTrue(isinstance(test_vol.size, int)) self.assertEqual( test_snap.size, test_vol.size, "Snapshot.size must match original volume's size: %s" " but is: %s" % (test_vol.size, test_snap.size)) self.assertTrue( test_vol.description is None or - isinstance(test_vol.description, six.string_types), + isinstance(test_vol.description, str), "Snapshot.description must be None or a string. Got: %s" % test_vol.description) self.assertEqual(test_vol.id, test_snap.volume_id) diff --git a/tests/test_cloud_helpers.py b/tests/test_cloud_helpers.py index 1b6c51e1..8f08553b 100644 --- a/tests/test_cloud_helpers.py +++ b/tests/test_cloud_helpers.py @@ -1,7 +1,5 @@ import itertools -import six - from cloudbridge.base.helpers import get_env from cloudbridge.base.resources import ClientPagedResultList from cloudbridge.base.resources import ServerPagedResultList @@ -84,7 +82,7 @@ def test_type_validation(self): self.provider.config['text_type_check'] = 'test-text' # pylint:disable=protected-access config_value = self.provider._get_config_value('text_type_check', None) - self.assertIsInstance(config_value, six.string_types) + self.assertIsInstance(config_value, str) # pylint:disable=protected-access none_value = self.provider._get_config_value( diff --git a/tests/test_compute_service.py b/tests/test_compute_service.py index b6e1321b..98f25830 100644 --- a/tests/test_compute_service.py +++ b/tests/test_compute_service.py @@ -1,8 +1,6 @@ import datetime import ipaddress -import six - from cloudbridge.base import helpers as cb_helpers from cloudbridge.base.resources import BaseNetwork from cloudbridge.factory import ProviderList @@ -120,7 +118,7 @@ def test_instance_properties(self): "Image id {0} is not equal to the expected id" " {1}".format(test_instance.image_id, image_id)) self.assertIsInstance(test_instance.zone_id, - six.string_types) + str) self.assertEqual( test_instance.image_id, helpers.get_provider_test_data(self.provider, "image")) @@ -158,7 +156,7 @@ def test_instance_properties(self): self._is_valid_ip(ip_address), "Instance must have a valid IP address. Got: %s" % ip_address) self.assertIsInstance(test_instance.vm_type_id, - six.string_types) + str) vm_type = self.provider.compute.vm_types.get( test_instance.vm_type_id) self.assertEqual( diff --git a/tests/test_region_service.py b/tests/test_region_service.py index 75986ea1..5cafdff3 100644 --- a/tests/test_region_service.py +++ b/tests/test_region_service.py @@ -1,5 +1,3 @@ -import six - from cloudbridge.interfaces import Region from tests import helpers @@ -61,7 +59,7 @@ def test_zones(self): self.assertTrue(zone.name) self.assertTrue(zone.region_name is None or isinstance(zone.region_name, - six.string_types)) + str)) if test_zone == zone.name: zone_find_count += 1 # zone info cannot be repeated between regions diff --git a/tests/test_vm_types_service.py b/tests/test_vm_types_service.py index 440c5a30..363d68b6 100644 --- a/tests/test_vm_types_service.py +++ b/tests/test_vm_types_service.py @@ -1,5 +1,3 @@ -import six - from tests import helpers from tests.helpers import ProviderTestBase from tests.helpers import standard_interface_tests as sit @@ -34,12 +32,12 @@ def test_vm_type_properties(self): self.assertTrue( vm_type.family is None or isinstance( vm_type.family, - six.string_types), + str), "VMType family must be None or a" " string but is: {0}".format(vm_type.family)) self.assertTrue( vm_type.vcpus is None or ( - isinstance(vm_type.vcpus, six.integer_types) and + isinstance(vm_type.vcpus, int) and vm_type.vcpus >= 0), "VMType vcpus must be None or a positive integer but is: {0}" .format(vm_type.vcpus)) @@ -58,7 +56,7 @@ def test_vm_type_properties(self): " number") self.assertTrue( isinstance(vm_type.num_ephemeral_disks, - six.integer_types) and + int) and vm_type.num_ephemeral_disks >= 0, "VMType num_ephemeral_disks must be None or a positive" " number") From 0d604fc044720974ac808b93487992afc2528f96 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 11:42:27 +0530 Subject: [PATCH 03/11] Sweep stale Py2.7 / setup.py references from docs and source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update README badge URLs and table header from py3.8 to py3.13 so they match the filenames the integration-cloud workflow writes to the gist. Refresh docs to reflect the modern environment: - install.rst: prerequisite is Python 3.13. - testing.rst: tox envs are py3.13-*; replace the gone `setup.py test` section with pytest equivalents. - release_process.rst: dependencies live in pyproject.toml; build with `python -m build`; CI is GitHub Actions, not Travis; drop six. - provider_development.rst: tox env is py3.13; new provider deps go in pyproject.toml under [project.optional-dependencies]. - troubleshooting.rst: generalize the macOS Python 3.6 cert path. Source cleanups: - openstack/resources.py: drop the urllib.parse/urlparse try/except shim. - gcp/helpers.py: drop "required for Python 2.7" comments (the str() wrappers remain — e.content is bytes, casts are still useful). - tests/__init__.py and tests/test_compute_service.py: drop py27-only invocation guidance and the u"" string-format workaround. --- README.rst | 46 ++++++++++---------- cloudbridge/providers/gcp/helpers.py | 6 --- cloudbridge/providers/openstack/resources.py | 9 +--- docs/topics/install.rst | 3 +- docs/topics/provider_development.rst | 16 ++++--- docs/topics/release_process.rst | 14 +++--- docs/topics/testing.rst | 21 ++++----- docs/topics/troubleshooting.rst | 17 ++++---- tests/__init__.py | 5 +-- tests/test_compute_service.py | 3 +- 10 files changed, 61 insertions(+), 79 deletions(-) diff --git a/README.rst b/README.rst index 51dad75b..37d27188 100644 --- a/README.rst +++ b/README.rst @@ -29,34 +29,34 @@ Build Status Tests :target: https://pypistats.org/packages/cloudbridge :alt: Download stats -.. |aws-py38| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.8_aws.json - :target: https://github.com/CloudVE/cloudbridge/actions/ +.. |aws-py313| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.13_aws.json + :target: https://github.com/CloudVE/cloudbridge/actions/ -.. |azure-py38| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.8_azure.json - :target: https://github.com/CloudVE/cloudbridge/actions/ +.. |azure-py313| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.13_azure.json + :target: https://github.com/CloudVE/cloudbridge/actions/ -.. |gcp-py38| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.8_gcp.json - :target: https://github.com/CloudVE/cloudbridge/actions/ +.. |gcp-py313| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.13_gcp.json + :target: https://github.com/CloudVE/cloudbridge/actions/ + +.. |mock-py313| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.13_mock.json + :target: https://github.com/CloudVE/cloudbridge/actions/ -.. |mock-py38| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.8_mock.json +.. |os-py313| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.13_openstack.json :target: https://github.com/CloudVE/cloudbridge/actions/ -.. |os-py38| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.8_openstack.json - :target: https://github.com/CloudVE/cloudbridge/actions/ - -+---------------------------+----------------+ -| **Provider/Environment** | **Python 3.8** | -+---------------------------+----------------+ -| **Amazon Web Services** | |aws-py38| | -+---------------------------+----------------+ -| **Google Cloud Platform** | |gcp-py38| | -+---------------------------+----------------+ -| **Microsoft Azure** | |azure-py38| | -+---------------------------+----------------+ -| **OpenStack** | |os-py38| | -+---------------------------+----------------+ -| **Mock Provider** | |mock-py38| | -+---------------------------+----------------+ ++---------------------------+-----------------+ +| **Provider/Environment** | **Python 3.13** | ++---------------------------+-----------------+ +| **Amazon Web Services** | |aws-py313| | ++---------------------------+-----------------+ +| **Google Cloud Platform** | |gcp-py313| | ++---------------------------+-----------------+ +| **Microsoft Azure** | |azure-py313| | ++---------------------------+-----------------+ +| **OpenStack** | |os-py313| | ++---------------------------+-----------------+ +| **Mock Provider** | |mock-py313| | ++---------------------------+-----------------+ Installation ~~~~~~~~~~~~ diff --git a/cloudbridge/providers/gcp/helpers.py b/cloudbridge/providers/gcp/helpers.py index 57a711f4..eabf63ad 100644 --- a/cloudbridge/providers/gcp/helpers.py +++ b/cloudbridge/providers/gcp/helpers.py @@ -41,7 +41,6 @@ def __if_fingerprint_differs(e): if isinstance(e, HttpError): expected_message = 'Supplied fingerprint does not match current ' \ 'metadata fingerprint.' - # str wrapper required for Python 2.7 if expected_message in str(e.content): return True return False @@ -155,7 +154,6 @@ def __if_label_fingerprint_differs(e): if isinstance(e, HttpError): expected_message = 'Labels fingerprint either invalid or ' \ 'resource labels have changed' - # str wrapper required for Python 2.7 if expected_message in str(e.content): return True return False @@ -169,10 +167,6 @@ def __if_label_fingerprint_differs(e): def change_label(resource, key, value, res_att, request): resource.assert_valid_resource_label(value) labels = getattr(resource, res_att).get("labels", {}) - # The returned value from above command yields a unicode dict key, which - # cannot be simply cast into a str for py2 so pop the key and re-add it - # The casting needs to be done for all labels, as to support both - # description and label setting labels[key] = str(value) for k in list(labels): labels[str(k)] = str(labels.pop(k)) diff --git a/cloudbridge/providers/openstack/resources.py b/cloudbridge/providers/openstack/resources.py index f4262869..c4948b96 100644 --- a/cloudbridge/providers/openstack/resources.py +++ b/cloudbridge/providers/openstack/resources.py @@ -7,14 +7,9 @@ import os import re -try: - from urllib.parse import urlparse - from urllib.parse import urljoin -except ImportError: # python 2 - from urlparse import urlparse - from urlparse import urljoin - from datetime import datetime +from urllib.parse import urljoin +from urllib.parse import urlparse from keystoneclient.v3.regions import Region diff --git a/docs/topics/install.rst b/docs/topics/install.rst index 62718183..fb595bb7 100644 --- a/docs/topics/install.rst +++ b/docs/topics/install.rst @@ -1,8 +1,7 @@ Installation ============ -**Prerequisites**: CloudBridge runs on Python 2.7 and higher. Python 3 is -recommended. +**Prerequisites**: CloudBridge requires Python 3.13 or higher. We highly recommend installing CloudBridge in a `virtualenv `_. Creating a new virtualenv diff --git a/docs/topics/provider_development.rst b/docs/topics/provider_development.rst index 69f7441c..10833fcf 100644 --- a/docs/topics/provider_development.rst +++ b/docs/topics/provider_development.rst @@ -45,12 +45,12 @@ This only requires that you register the provider's ID in the ``ProviderList``. Add GCP to the ``ProviderList`` class in ``cloudbridge/cloud/factory.py``. -5. Run the test suite. We will get the tests passing on py27 first. +5. Run the test suite. .. code-block:: bash export CB_TEST_PROVIDER=gcp - tox -e py27 + tox -e py3.13 You should see the tests fail with the following message: @@ -195,13 +195,15 @@ tests pass. is up to the implementor, a general design we have followed is to have the cloud connection globally available within the provider. -To add the sdk, we edit CloudBridge's main ``setup.py`` and list the -dependencies. +To add the sdk, we edit CloudBridge's main ``pyproject.toml`` and add the +provider's dependencies under ``[project.optional-dependencies]``. -.. code-block:: python +.. code-block:: toml - gcp_reqs = ['google-api-python-client==1.4.2'] - full_reqs = base_reqs + aws_reqs + openstack_reqs + gcp_reqs + [project.optional-dependencies] + gcp = [ + "google-api-python-client>=2.0,<3.0.0", + ] We will also register the provider in ``cloudbridge/cloud/factory.py``'s provider list. diff --git a/docs/topics/release_process.rst b/docs/topics/release_process.rst index 9a2b59ad..3a2b23e3 100644 --- a/docs/topics/release_process.rst +++ b/docs/topics/release_process.rst @@ -1,28 +1,28 @@ Release Process ~~~~~~~~~~~~~~~ -1. Make sure `all tests pass `_. +1. Make sure all tests pass on the `GitHub Actions workflows + `_. 2. Increment version number in ``cloudbridge/__init__.py`` as per `semver rules `_. -3. Freeze all library dependencies in ``setup.py`` and commit. +3. Freeze all library dependencies in ``pyproject.toml`` and commit. The version numbers can be a range with the upper limit being the latest known working version, and the lowest being the last known working version. In general, our strategy is to make provider sdk libraries fixed within relatively known compatibility ranges, so that we reduce the chances of breakage. If someone uses CloudBridge, presumably, they do not use the SDKs - directly. For all other libraries, especially, general purpose libraries - (e.g. ``six``), our strategy is to make compatibility as broad and - unrestricted as possible. + directly. For all other general purpose libraries, our strategy is to make + compatibility as broad and unrestricted as possible. 4. Add release notes to ``CHANGELOG.rst``. Also add last commit hash to changelog. List of commits can be obtained using ``git shortlog ..HEAD`` 5. Release to PyPi. - (make sure you have run `pip install wheel twine`) + (make sure you have run ``pip install build twine``) First, test release with PyPI staging server as described in: https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/ @@ -32,7 +32,7 @@ Release Process # remove stale files or wheel might package them rm -r build dist - python setup.py sdist bdist_wheel + python -m build twine upload -r pypi dist/cloudbridge-3.0.0* 6. Tag release and make a GitHub release. diff --git a/docs/topics/testing.rst b/docs/topics/testing.rst index 470c6e25..8aca78c7 100644 --- a/docs/topics/testing.rst +++ b/docs/topics/testing.rst @@ -40,10 +40,9 @@ This will run all the tests for all the environments defined in file Specific environment and infrastructure ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you’d like to run the tests on a specific environment only, say Python 2.7, -against a specific infrastructure, say aws, use a command like this: -``tox -e py27-aws``. The available provider names are listed in the -`ProviderList`_ class (e.g., ``aws`` or ``openstack``). +To run the tests against a specific infrastructure, say aws, use a command +like this: ``tox -e py3.13-aws``. The available provider names are listed in +the `ProviderList`_ class (e.g., ``aws`` or ``openstack``). Specific test cases ~~~~~~~~~~~~~~~~~~~~ @@ -51,19 +50,17 @@ You can run a specific test case, as follows: ``tox -- tests/test_image_service.py:CloudImageServiceTestCase.test_create_and_list_imag`` It can also be restricted to a particular environment as follows: -``tox -e "py27-aws" -- tests/test_cloud_factory.py:CloudFactoryTestCase`` +``tox -e "py3.13-aws" -- tests/test_cloud_factory.py:CloudFactoryTestCase`` -See nosetest documentation for other parameters that can be passed in. - -Using unittest directly +Running pytest directly ~~~~~~~~~~~~~~~~~~~~~~~ You can also run the tests against your active virtual environment directly -with ``python setup.py test``. You will need to set the ``CB_TEST_PROVIDER`` +with ``pytest tests/``. You will need to set the ``CB_TEST_PROVIDER`` environment variable prior to running the tests, or they will default to ``CB_TEST_PROVIDER=aws``. -You can also run a specific test case, as follows: -``python setup.py test -s tests.test_cloud_factory.CloudFactoryTestCase`` +To run a specific test case: +``pytest tests/test_cloud_factory.py::CloudFactoryTestCase`` Using a mock provider ~~~~~~~~~~~~~~~~~~~~~ @@ -74,7 +71,7 @@ will simulate AWS resources. You can use ``CB_TEST_PROVIDER=mock`` to run tests against the mock provider only, which will provide faster feedback times. Alternatively you can run the mock tests through tox. -``tox -e "py27-mock"`` +``tox -e "py3.13-mock"`` .. _design goals: https://github.com/CloudVE/cloudbridge/ blob/main/README.rst diff --git a/docs/topics/troubleshooting.rst b/docs/topics/troubleshooting.rst index 2c420899..5b226ffc 100644 --- a/docs/topics/troubleshooting.rst +++ b/docs/topics/troubleshooting.rst @@ -5,18 +5,17 @@ macOS Issues ------------ * If you are getting an error message like so: ``Authentication with cloud provider failed: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:749)`` - then this indicates that you are probably using a newer version of Python on - macOS. Starting with Python 3.6, the Python installer includes its own version - of OpenSSL and it no longer uses the system trusted certificate keychains. + then this indicates that you are probably using a Python distribution on + macOS whose installer ships its own OpenSSL and does not use the system + trusted certificate keychain. - Python 3.6 includes a script that can install a bundle of root certificates - from ``certifi``. To install this bundle execute the following: + The python.org installer includes a script that installs a bundle of root + certificates from ``certifi``. Run the ``Install Certificates.command`` + script bundled with your Python install, for example: .. code-block:: bash - cd /Applications/Python\ 3.6/ - sudo ./Install\ Certificates.command + /Applications/Python\ 3.13/Install\ Certificates.command For more information see `this StackOverflow - answer `_ and the `Python 3.6 - Release Notes `_. + answer `_. diff --git a/tests/__init__.py b/tests/__init__.py index 5e962604..1f699588 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1 @@ -""" -Use ``python setup.py test`` to run these unit tests (alternatively, use -``python -m unittest test``). -""" +"""Run these unit tests with ``pytest tests/`` or via ``tox``.""" diff --git a/tests/test_compute_service.py b/tests/test_compute_service.py index 98f25830..695d6cb8 100644 --- a/tests/test_compute_service.py +++ b/tests/test_compute_service.py @@ -147,8 +147,7 @@ def test_instance_properties(self): ip_address = test_instance.public_ips[0] \ if test_instance.public_ips and test_instance.public_ips[0] \ else ip_private - # Convert to unicode for py27 compatibility with ipaddress() - ip_address = u"{}".format(ip_address) + ip_address = str(ip_address) self.assertIsNotNone( ip_address, "Instance must have either a public IP or a private IP") From a8a2f4551e334af2f69fc86b3f34b922f2c61885 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 12:15:18 +0530 Subject: [PATCH 04/11] Modernize RTD config, docs deps, tox env, and stdlib usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .readthedocs.yaml: add the now-required build.os, build.tools.python, and sphinx.configuration blocks — RTD retired the implicit-defaults build platform in late 2024. docs/conf.py: drop vestigial `import sphinx_rtd_theme` and the html_theme_path call (sphinx_rtd_theme >= 1.0 auto-discovers via entry points); bump copyright through 2026. docs/requirements.txt: raise floors to sphinx>=8 / sphinx_rtd_theme>=3 and drop the comment about a long-resolved sphinx bug. tox.ini: remove the pypy environment from envlist — CI never ran it, and pypy can't realistically build the Azure SDK's C extensions. requirements.txt: drop sshpubkeys (was a moto 1.0.0 workaround; we run moto 5.x now). cloudbridge/__init__.py: replace the custom NullHandler subclass with logging.NullHandler (in stdlib since Py3.1). Add TODO.rst capturing the deferred mechanical sweeps (object base class, super(), f-strings, typing builtins, ruff/mypy, pre-existing flake8 import-order errors, untracked dev artifacts). --- .readthedocs.yaml | 12 +++++++++-- TODO.rst | 45 +++++++++++++++++++++++++++++++++++++++++ cloudbridge/__init__.py | 10 +-------- docs/conf.py | 6 +----- docs/requirements.txt | 5 ++--- requirements.txt | 2 -- tox.ini | 2 +- 7 files changed, 60 insertions(+), 22 deletions(-) create mode 100644 TODO.rst diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 65b390f2..54b53a6e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,5 +1,13 @@ version: 2 +build: + os: ubuntu-24.04 + tools: + python: "3.13" + +sphinx: + configuration: docs/conf.py + python: - install: - - requirements: docs/requirements.txt + install: + - requirements: docs/requirements.txt diff --git a/TODO.rst b/TODO.rst new file mode 100644 index 00000000..9032e3a6 --- /dev/null +++ b/TODO.rst @@ -0,0 +1,45 @@ +Deferred Modernization Work +=========================== + +The packaging migration (setup.py → pyproject.toml, drop six, refresh docs) +landed in commits ``7df4a94``..``HEAD``. The items below were identified +during that sweep but deliberately left out because each is large enough +to warrant its own focused PR. + +Mechanical Python idiom updates +------------------------------- + +Each of these is a near-mechanical refactor with a wide diff. Best done +one-at-a-time so reviewers can read each change as a single transformation. + +* **Drop explicit ``object`` base class.** ``class Foo(object):`` → + ``class Foo:``. No behavior change in Py3. +* **Modernize ``super()`` calls.** ``super(ClassName, self).method(...)`` → + ``super().method(...)``. The arguments are required only in Py2. +* **Adopt f-strings.** ``"x={0}".format(x)`` and ``"x=%s" % x`` → + ``f"x={x}"``. Skip for logging calls — those should keep ``%s`` + formatting so the logger can short-circuit when the level is disabled. +* **Switch typing imports to builtins.** ``List[X]`` / ``Dict[K, V]`` / + ``Optional[X]`` → ``list[X]`` / ``dict[K, V]`` / ``X | None`` once a + Python 3.10+ floor is acceptable (we already require 3.13, so this is + safe today). + +Lint and tooling +---------------- + +* **Fix the ~23 pre-existing flake8 import-order errors.** Run + ``tox -e lint`` to see the list. Mostly ``I100``/``I201``/``I202`` + under ``cloudbridge/providers/azure``, ``gcp``, and ``openstack``. +* **Consider replacing flake8 + flake8-import-order with ruff.** Ruff + reads ``pyproject.toml``, runs ~100× faster, and covers import + ordering (``I``) plus most flake8 plugins out of the box. +* **Consider adding mypy / pyright in CI.** The codebase has no type + hints today; this would be a meaningful uplift, not a one-PR task. + +Repository hygiene +------------------ + +* **Untracked local-dev artifacts at the repo root** — ``azure.txt``, + ``openstack.txt``, ``docs2/``, ``script_test.py``, ``openstack.log``. + Each likely belongs in ``.gitignore`` or in a developer's untracked + workspace; investigate before either committing or deleting. diff --git a/cloudbridge/__init__.py b/cloudbridge/__init__.py index d2911660..4476fef8 100644 --- a/cloudbridge/__init__.py +++ b/cloudbridge/__init__.py @@ -25,14 +25,6 @@ def init_logging(): set_stream_logger(__name__, level=logging.DEBUG) -class NullHandler(logging.Handler): - """A null handler for the logger.""" - - def emit(self, record): - """Don't emit a log.""" - pass - - TRACE = 5 # Lower than debug which is 10 @@ -58,7 +50,7 @@ def trace(self, msg, *args, **kwargs): logging.setLoggerClass(CBLogger) logging.addLevelName(TRACE, "TRACE") log = logging.getLogger('cloudbridge') -log.addHandler(NullHandler()) +log.addHandler(logging.NullHandler()) # Convenience functions to set logging to a particular file or stream # To enable either of these by default within CloudBridge, add the following diff --git a/docs/conf.py b/docs/conf.py index 8b3b265d..293c5145 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,13 +15,12 @@ sys.path.insert(0, os.path.abspath('../')) -import sphinx_rtd_theme import cloudbridge # -- Project information ----------------------------------------------------- project = 'cloudbridge' -copyright = '2021, GVL and Galaxy Projects' +copyright = '2015-2026, GVL and Galaxy Projects' author = 'GVL and Galaxy Projects' # The full version, including alpha/beta/rc tags @@ -57,9 +56,6 @@ # html_theme = 'sphinx_rtd_theme' -# Add any paths that contain custom themes here, relative to this directory. -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". diff --git a/docs/requirements.txt b/docs/requirements.txt index f6e6ad53..da72e08d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,2 @@ -# https://github.yuuza.net/sphinx-doc/sphinx/issues/9727 -sphinx>=4.2.0 -sphinx_rtd_theme>=1.0.0 +sphinx>=8.0 +sphinx_rtd_theme>=3.0 diff --git a/requirements.txt b/requirements.txt index ca101a91..525e3e70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1 @@ -# needed by moto -sshpubkeys -e ".[dev]" diff --git a/tox.ini b/tox.ini index 536153b9..8151da4a 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ # running the tests. [tox] -envlist = {py3.13,pypy}-{aws,azure,gcp,openstack,mock},lint +envlist = py3.13-{aws,azure,gcp,openstack,mock},lint [testenv] commands = # see pyproject.toml for coverage options; setup.cfg for flake8 From 2b8a66f0e860aab36887c5abbd43987a0791abbe Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 12:24:00 +0530 Subject: [PATCH 05/11] Narrow test_crud_instance moto-skip to ==5.2.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skip specifier was `>=5.0.0`, which silently masks the test for every future moto release. Pin to the exact known-broken version (5.2.1) so newer releases re-run the test — we want to discover an upstream fix the first time CI installs a newer moto, not stay quiet forever. --- tests/test_compute_service.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_compute_service.py b/tests/test_compute_service.py index 695d6cb8..3759e0c5 100644 --- a/tests/test_compute_service.py +++ b/tests/test_compute_service.py @@ -35,11 +35,12 @@ def test_storage_services_event_pattern(self): # immediately after RunInstances completes, so the list-after-create # check in standard_interface_tests.check_list fails. A secondary # symptom shows in cleanup, where post-delete state remains - # "deleted" instead of becoming UNKNOWN. Last observed on moto - # 5.2.1. Tighten the specifier when an upstream fix lands. + # "deleted" instead of becoming UNKNOWN. Pinned to the latest moto + # release where this was observed (5.2.1); newer releases re-run the + # test so we notice if it's fixed. Bump the pin if it's still broken. @helpers.skipIfMockMotoVersion( - ">=5.0.0", - "moto 5.x RunInstances/DescribeInstances state-sync bug") + "==5.2.1", + "moto 5.2.1 RunInstances/DescribeInstances state-sync bug") @helpers.skipIfNoService(['compute.instances', 'networking.networks']) def test_crud_instance(self): label = "cb-instcrud-{0}".format(helpers.get_uuid()) From 9b1b94ded4671ad22aceaca75bc9bda688133039 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 13:42:50 +0530 Subject: [PATCH 06/11] Forward AZURE_REGION_NAME and CB_PLACEMENT_AZURE to tox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Azure provider reads AZURE_REGION_NAME for its region and the test framework reads CB_PLACEMENT_AZURE for placement, but neither was plumbed through the workflow's env: block. Setting either as a repo secret had no effect — the secret reached GitHub but never got into the runner process, and so never reached tox or the test code. The provider silently fell back to its hardcoded `eastus` default, overriding whatever the secret said. Add both, gated on `matrix.cloud-provider == 'azure'` to match the secret-scoping pattern the rest of the Azure block already uses. --- .github/workflows/integration-cloud.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration-cloud.yaml b/.github/workflows/integration-cloud.yaml index 54b77ac1..dc92872e 100644 --- a/.github/workflows/integration-cloud.yaml +++ b/.github/workflows/integration-cloud.yaml @@ -100,10 +100,12 @@ jobs: AZURE_SUBSCRIPTION_ID: ${{ matrix.cloud-provider == 'azure' && secrets.AZURE_SUBSCRIPTION_ID || '' }} AZURE_SECRET: ${{ matrix.cloud-provider == 'azure' && secrets.AZURE_SECRET || '' }} AZURE_TENANT: ${{ matrix.cloud-provider == 'azure' && secrets.AZURE_TENANT || '' }} + AZURE_REGION_NAME: ${{ matrix.cloud-provider == 'azure' && secrets.AZURE_REGION_NAME || '' }} AZURE_RESOURCE_GROUP: ${{ matrix.cloud-provider == 'azure' && secrets.AZURE_RESOURCE_GROUP || '' }} AZURE_STORAGE_ACCOUNT: ${{ matrix.cloud-provider == 'azure' && secrets.AZURE_STORAGE_ACCOUNT || '' }} CB_IMAGE_AZURE: ${{ matrix.cloud-provider == 'azure' && secrets.CB_IMAGE_AZURE || '' }} CB_VM_TYPE_AZURE: ${{ matrix.cloud-provider == 'azure' && secrets.CB_VM_TYPE_AZURE || '' }} + CB_PLACEMENT_AZURE: ${{ matrix.cloud-provider == 'azure' && secrets.CB_PLACEMENT_AZURE || '' }} # gcp GCP_SERVICE_CREDS_DICT: ${{ matrix.cloud-provider == 'gcp' && secrets.GCP_SERVICE_CREDS_DICT || '' }} CB_IMAGE_GCP: ${{ matrix.cloud-provider == 'gcp' && secrets.CB_IMAGE_GCP || '' }} From 4138c6d49c78765e2d8b4b6f1cbfbeedb3cc63ba Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 14:30:37 +0530 Subject: [PATCH 07/11] Make OpenStack FIP detach + visibility robust Two related bugs surfaced by test_instance_methods on the CI run for PR #328: * ``remove_floating_ip`` was clearing port_id via ``Connection.network.update_ip(fip, port_id=None)``. Some openstacksdk versions strip ``None`` kwargs from the PUT body, which meant the Neutron-side disassociation never happened and the FIP stayed bound to the instance's port. Switch to a direct neutronclient ``update_floatingip(id, {'floatingip': {'port_id': None}})`` call so the wire payload explicitly carries ``port_id: null``. Also make remove_floating_ip a no-op (instead of an AttributeError) when the FIP id has already been deleted, which happens when a test's nested cleanup_actions detach + delete in sequence. * ``_all_addresses`` was unioning Nova's view of ``server.addresses`` with a live Neutron query. Nova's info_cache lags ~60s and isn't re-queried on server-show, so a FIP detached on the Neutron side still appeared in ``public_ips`` / ``private_ips`` until Nova caught up. Read only **fixed** IPs from Nova (filtering by ``OS-EXT-IPS:type``) and trust Neutron exclusively for floating IPs. --- cloudbridge/providers/openstack/resources.py | 37 ++++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/cloudbridge/providers/openstack/resources.py b/cloudbridge/providers/openstack/resources.py index c4948b96..238ed58a 100644 --- a/cloudbridge/providers/openstack/resources.py +++ b/cloudbridge/providers/openstack/resources.py @@ -328,20 +328,21 @@ def label(self, value): def _all_addresses(self): """All IP addresses associated with this instance. - Combines the addresses Nova reports (via server.addresses / - ``_os_instance.networks``, populated from Nova's info_cache) with - any floating IPs Neutron currently has bound to the instance's - ports. Nova's info_cache is refreshed by a periodic task on a - ~60s cadence and is not re-synced on a plain server-show, so a - FIP attached via the Neutron API (as add_floating_ip does) - otherwise wouldn't show up until the next sync. + Nova's info_cache (which backs ``server.addresses``) is refreshed + by a periodic task on a ~60s cadence and is not re-queried on a + plain server-show. That makes it lag both ways: a FIP just + attached via Neutron won't appear, and a FIP just detached via + Neutron will still appear. So we deliberately read only fixed + IPs from Nova and ask Neutron live for the current floating IPs. """ addrs = set() - for _, network_addrs in self._os_instance.networks.items(): - for address in network_addrs: - addrs.add(address) - # Query Neutron for any floating IPs bound to this instance's - # ports — these may not yet be reflected in Nova's cached view. + for _, addr_list in self._os_instance.addresses.items(): + for entry in addr_list: + if entry.get('OS-EXT-IPS:type') == 'floating': + continue + ip = entry.get('addr') or entry.get('OS-EXT-IPS-MAC:addr') + if ip: + addrs.add(ip) try: for port in self._provider.os_conn.network.ports( device_id=self.id): @@ -519,13 +520,19 @@ def remove_floating_ip(self, floating_ip): Remove a floating IP address from this instance. Same rationale as add_floating_ip; the Nova action endpoint is - gone, so detach by clearing port_id on the Neutron FIP. + gone, so detach by clearing port_id on the Neutron FIP. We go + through neutronclient directly rather than openstacksdk + Connection.network.update_ip(...) because some openstacksdk + versions drop ``None`` kwargs from the PUT body, which leaves + port_id unchanged on the server side. """ log.debug("Removing floating IP adress: %s", floating_ip) fip = (floating_ip if isinstance(floating_ip, OpenStackFloatingIP) else self._get_fip(floating_ip)) - # pylint:disable=protected-access - self._provider.os_conn.network.update_ip(fip._ip, port_id=None) + if fip is None: + return + self._provider.neutron.update_floatingip( + fip.id, {'floatingip': {'port_id': None}}) def add_vm_firewall(self, firewall): """ From 43929ec1969cd04692396fb7d494f349596b6c85 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 14:39:24 +0530 Subject: [PATCH 08/11] Use aws:RequestedRegion to scope the EC2 policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous condition keyed off `ec2:Region`, which AWS only populates in the request context for actions that operate on a region-scoped resource. Service-level `Describe*` calls (e.g. `DescribeAvailabilityZones`) don't have it set, and despite the `StringEqualsIfExists` semantics, IAM evaluated the condition as a non-match — denying the call even though `ec2:*` covered the action. Switch to `aws:RequestedRegion`, which IAM itself populates on every authenticated API call from the endpoint region. Always present, no need for the `IfExists` qualifier. This is AWS's recommended pattern for region-scoping inline policies and works uniformly across services. The role's actual deployed policy is updated by re-running .github/aws/setup.sh against the account; this commit only refreshes the source-of-truth file. --- .github/aws/permissions-policy.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/aws/permissions-policy.json b/.github/aws/permissions-policy.json index 3e085a84..86fb55fe 100644 --- a/.github/aws/permissions-policy.json +++ b/.github/aws/permissions-policy.json @@ -7,8 +7,8 @@ "Action": "ec2:*", "Resource": "*", "Condition": { - "StringEqualsIfExists": { - "ec2:Region": "us-east-1" + "StringEquals": { + "aws:RequestedRegion": "us-east-1" } } }, From c4b42f5c96224f674c4ff8c04539ca8a02b6cf33 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 14:59:16 +0530 Subject: [PATCH 09/11] Run Azure integration tests serially under tox pytest-xdist with -n 5 races against the shared default vnet/subnet: multiple workers call get_or_create_default cold, all find nothing, all try to create cloudbridge-net/cloudbridge-subnet, and Azure ARM rejects all but one with AnotherOperationInProgress or cancels in- flight create LROs. The losing workers' tests then fail at VM creation because they have no usable subnet. Drop the -n 5 default for the azure factor only. Other clouds keep parallel since their tests don't share singleton default resources the same way. --- tox.ini | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8151da4a..77f893b3 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,12 @@ envlist = py3.13-{aws,azure,gcp,openstack,mock},lint [testenv] commands = # see pyproject.toml for coverage options; setup.cfg for flake8 - coverage run --source=cloudbridge -m pytest -v {posargs:-n 5 tests/} + # Azure integration tests race on the shared default vnet/subnet + # under xdist (get_or_create_default has no locking, so two workers + # creating the same named resource conflict on Azure ARM). Run the + # azure factor serially; other providers stay parallel. + !azure: coverage run --source=cloudbridge -m pytest -v {posargs:-n 5 tests/} + azure: coverage run --source=cloudbridge -m pytest -v {posargs:tests/} # Combine parallel-mode data files and emit Cobertura XML for upload # by coverallsapp/github-action in CI. Locally this produces # coverage.xml in the project root, which IDEs can also consume. From 6143ebd218425188168bbab820860a7330c3f005 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 15:25:55 +0530 Subject: [PATCH 10/11] Allow cross-region EC2 reads in the role policy `test_zones` iterates over every AWS region and calls `describe_availability_zones` against each region's endpoint (via cloudbridge.providers.aws.resources.AWSRegion.zones, which spins up a per-region EC2 client). For all calls outside us-east-1, `aws:RequestedRegion` is the target region, so the existing `aws:RequestedRegion == us-east-1` condition rejected them with `UnauthorizedOperation` even though `ec2:*` covered the action. Add a separate `EC2ReadAnyRegion` statement allowing `ec2:Describe*`, `ec2:Get*`, `ec2:List*` without a region condition. Mutations (`RunInstances`, `CreateVpc`, etc.) remain pinned to us-east-1 via the existing `EC2FullAccessUsEast1` statement, so the safety boundary against accidentally provisioning in other regions is preserved. The deployed inline policy is updated by re-running .github/aws/setup.sh; this commit only refreshes the source-of-truth. --- .github/aws/permissions-policy.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/aws/permissions-policy.json b/.github/aws/permissions-policy.json index 86fb55fe..9fc754da 100644 --- a/.github/aws/permissions-policy.json +++ b/.github/aws/permissions-policy.json @@ -1,6 +1,16 @@ { "Version": "2012-10-17", "Statement": [ + { + "Sid": "EC2ReadAnyRegion", + "Effect": "Allow", + "Action": [ + "ec2:Describe*", + "ec2:Get*", + "ec2:List*" + ], + "Resource": "*" + }, { "Sid": "EC2FullAccessUsEast1", "Effect": "Allow", From e4d17fa03567376a854a47e4b7f174a1f2c2e545 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 15 May 2026 16:30:30 +0530 Subject: [PATCH 11/11] Revert "Run Azure integration tests serially under tox" This reverts commit c4b42f5c96224f674c4ff8c04539ca8a02b6cf33. --- tox.ini | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 77f893b3..8151da4a 100644 --- a/tox.ini +++ b/tox.ini @@ -10,12 +10,7 @@ envlist = py3.13-{aws,azure,gcp,openstack,mock},lint [testenv] commands = # see pyproject.toml for coverage options; setup.cfg for flake8 - # Azure integration tests race on the shared default vnet/subnet - # under xdist (get_or_create_default has no locking, so two workers - # creating the same named resource conflict on Azure ARM). Run the - # azure factor serially; other providers stay parallel. - !azure: coverage run --source=cloudbridge -m pytest -v {posargs:-n 5 tests/} - azure: coverage run --source=cloudbridge -m pytest -v {posargs:tests/} + coverage run --source=cloudbridge -m pytest -v {posargs:-n 5 tests/} # Combine parallel-mode data files and emit Cobertura XML for upload # by coverallsapp/github-action in CI. Locally this produces # coverage.xml in the project root, which IDEs can also consume.