Additional fixes for latest dependencies

- Made some small logic tweaks around region validation
- Introduced warning for unit tests if boto3 updated regions are not included in the
  test generated index file. This will also ensure that tests don't break
  should newer AWS regions come on line.
- Updated AWS dependencies and development dependencies
- TODO: Address issues with mkdocs

mkdocs/blog/.authors.yml

@@ -1,4 +1,5 @@
-mikegrima:
-  name: Mike Grima
-  description: Creator of Starfleet
-  avatar: https://github.com/mikegrima.png
+authors:
+  mikegrima:
+    name: Mike Grima
+    description: Creator of Starfleet
+    avatar: https://github.com/mikegrima.png

pyproject.toml

@@ -31,14 +31,14 @@ tests = [
     "black==23.7.0",
     "flake8==6.1.0",
     "pylint==2.17.5",
-    "tox==4.6.4",
+    "tox==4.11.0",
     "moto==4.1.12",
-    "mkdocs==1.5.1",
+    "mkdocs==1.5.2",
     "mkdocstrings[python]==0.22.0",
     "mkdocs-gen-files==0.5.0",
     "mkdocs-literate-nav==0.6.0",
     "mkdocs-section-index==0.3.5",
-    "mkdocs-material==9.2.0b2",
+    "mkdocs-material==9.2.6",
     "cfn-lint>=0.77",
     "checkov>=2.3",
 ]

src/requirements.txt

@@ -6,8 +6,8 @@ retry==0.9.2
 pyjwt==2.8.0
 requests==2.31.0
 cryptography==41.0.3
-botocore==1.31.16
-boto3==1.28.16
+botocore==1.31.39
+boto3==1.28.39
 slack-sdk==3.21.3
 # Deep-diff issue: https://github.com/seperman/deepdiff/issues/415
 deepdiff<6.4.0

src/starfleet/utils/config_schema.py

@@ -10,10 +10,11 @@
 """
 from typing import Any, Dict
 
-import boto3
 from marshmallow import Schema, fields, INCLUDE, validate, validates_schema, ValidationError
 
-aws_regions = set(boto3.session.Session().get_available_regions("ec2"))
+from starfleet.utils.niceties import get_all_regions
+
+aws_regions = get_all_regions()
 
 
 class SecretsManager(Schema):

src/starfleet/utils/niceties.py

@@ -0,0 +1,21 @@
+"""A general set of niceties that Starfleet can use to do things that are nice.
+
+This mostly defines some shortcut code utilities that workers can use for a variety of use cases.
+
+:Module: starfleet.utils.niceties
+:Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info
+:License: See the LICENSE file for details
+:Author: Mike Grima <michael.grima@gemini.com>
+"""
+from typing import Set
+
+import boto3
+
+
+def get_all_regions(service: str = "ec2") -> Set[str]:
+    """
+    This will return all supported AWS regions for the supplied service. By default, this returns the set for EC2.
+
+    This is placed here as a function so that we can easily mock out the values with a static set of values that will persist throughout boto3 updates.
+    """
+    return set(boto3.session.Session().get_available_regions(service))

src/starfleet/worker_ships/base_payload_schemas.py

@@ -9,9 +9,10 @@
 """
 from typing import List, Any, Dict, TypeVar
 
-import boto3
 from marshmallow import Schema, fields, INCLUDE, validates, ValidationError, validate, validates_schema
 
+from starfleet.utils.niceties import get_all_regions
+
 
 class WorkerShipPayloadBaseTemplate(Schema):
     """
@@ -188,7 +189,7 @@ def validate_regions(self, data: Dict[str, Any], **kwargs) -> None:  # pylint: d
         """
         errors = {}
 
-        supported_regions = set(boto3.session.Session().get_available_regions("ec2"))
+        supported_regions = get_all_regions()
 
         # Verify the "ALL" in include regions:
         if "ALL" in data["include_regions"]:

src/starfleet/worker_ships/plugins/account_index_generator/ship.py

@@ -105,10 +105,13 @@ def execute(self, commit: bool = False) -> None:
         else:
             LOGGER.info(f"[🍣] Raw Inventory:\n{json.dumps(account_map, sort_keys=True, indent=4)}")
 
-        # If you need to update the index JSON for unit tests, then run the tests and uncomment the code below. That will generate a new index JSON.
-        # Simply copy and paste this `generatedIndex.json` file to `tests/starfleet_included_plugins/account_index_generator/generatedIndex.json`:
-        # with open("generatedIndex.json", "w") as file:
+        # Instructions for updating the testing account index:
+        # If you need to update the index JSON for unit tests, uncomment the code below and run the tests for this plugin:
+        # with open("generatedIndex.json", "w") as file:  # Uncomment this and the next line
         #     file.write(json.dumps({"accounts": account_map, "generated": datetime.utcnow().replace(tzinfo=None, microsecond=0).isoformat() + "Z"}, indent=4, sort_keys=True))
+        # Then run the tests: `pytest tests/starfleet_included_plugins/account_index_generator`. That will generate a new index JSON.
+        # Simply copy and paste this `generatedIndex.json` file to `tests/starfleet_included_plugins/account_index_generator/generatedIndex.json`
+        # ‼️ Don't forget to re-comment the 2 lines above when done ‼️
 
 
 @click.group()

src/starfleet/worker_ships/plugins/aws_config/schemas.py

@@ -10,14 +10,14 @@
 from enum import Enum
 from typing import Any, Dict
 
-import boto3
 from marshmallow import fields, Schema, validates_schema, validate, ValidationError
 
+from starfleet.utils.niceties import get_all_regions
 from starfleet.worker_ships.base_payload_schemas import BaseAccountRegionPayloadTemplate, AccountsSpecificationSchema, IncludeAccountsSpecificationSchema
 from starfleet.worker_ships.ship_schematics import WorkerShipBaseConfigurationTemplate
 
 
-supported_regions = set(boto3.session.Session().get_available_regions("config"))
+supported_regions = get_all_regions(service="config")
 
 
 class AwsConfigWorkerShipConfigurationTemplate(WorkerShipBaseConfigurationTemplate):

src/starfleet/worker_ships/plugins/github_sync/ship.py

@@ -21,6 +21,7 @@
 from marshmallow import fields, validate
 
 from starfleet.utils.logging import LOGGER
+from starfleet.utils.niceties import get_all_regions
 from starfleet.worker_ships.base_payload_schemas import WorkerShipPayloadBaseTemplate
 from starfleet.worker_ships.cli_utils import StarfleetSingleInvokeCommand
 from starfleet.worker_ships.lambda_utils import worker_lambda
@@ -46,7 +47,7 @@ class GitHubSyncPayloadTemplate(WorkerShipPayloadBaseTemplate):
     github_app_id = fields.String(required=True, data_key="GitHubAppId")
     github_installation_id = fields.String(required=True, data_key="GitHubInstallationId")
     bucket_name = fields.String(required=True, data_key="BucketName")
-    bucket_region = fields.String(required=True, validate=validate.OneOf(boto3.session.Session().get_available_regions("s3")), data_key="BucketRegion")
+    bucket_region = fields.String(required=True, validate=validate.OneOf(get_all_regions(service="s3")), data_key="BucketRegion")
     extract_zip_contents = fields.Boolean(required=True, data_key="ExtractZipContents")
 
     # Optionals

tests/account_index/test_account_index_components.py

@@ -9,6 +9,7 @@
 """
 # pylint: disable=unused-argument
 from typing import Any, Dict
+from unittest import mock
 
 import pytest
 
@@ -58,3 +59,24 @@ def test_loading_invalid_configuration(test_configuration: Dict[str, Any]) -> No
         _ = index_loader.index
 
     assert str(exc.value) == "FakePlugin"
+
+
+def test_boto_outdated_logic(test_configuration: Dict[str, Any]) -> None:
+    """This tests the logic for detecting updated boto3 regions."""
+    from starfleet.utils.niceties import get_all_regions
+    from starfleet.account_index.loader import StarfleetAccountIndexLoader
+    import tests.account_index.testing_plugins
+
+    all_regions = get_all_regions()
+    all_regions.add("some-new-region")
+
+    with mock.patch("tests.account_index.testing_plugins.basic_plugin.get_all_regions", return_value=all_regions):
+        with pytest.warns(UserWarning) as warning:
+            account_indexer = StarfleetAccountIndexLoader()
+            account_indexer._index_ship_path = tests.account_index.testing_plugins.__path__
+            account_indexer._index_ship_prefix = tests.account_index.testing_plugins.__name__ + "."
+            index = account_indexer.index
+
+            assert index.regions_map["some-new-region"] == index.account_ids
+
+        assert "boto3 was updated" in str(warning.list[0].message)

tests/account_index/test_resolvers.py

@@ -10,10 +10,10 @@
 # pylint: disable=unused-argument
 from typing import Any, Dict
 
-import boto3
 import yaml
 
 from starfleet.account_index.schematics import AccountIndexInstance
+from starfleet.utils.niceties import get_all_regions
 
 
 def test_resolve_include_account_specification(test_index: AccountIndexInstance) -> None:
@@ -173,7 +173,7 @@ def test_resolve_worker_templates_disabled_regions(test_index: AccountIndexInsta
     from starfleet.worker_ships.base_payload_schemas import BaseAccountRegionPayloadTemplate
     from starfleet.account_index.resolvers import resolve_worker_template_account_regions
 
-    all_regions = set(boto3.session.Session().get_available_regions("ec2"))
+    all_regions = get_all_regions()
 
     # Disable in 2 accounts:
     test_index.regions_map["ap-east-1"].remove("000000000001")
@@ -193,7 +193,7 @@ def test_resolve_worker_templates_disabled_regions(test_index: AccountIndexInsta
     """
     template = BaseAccountRegionPayloadTemplate().load(yaml.safe_load(payload))
     result = resolve_worker_template_account_regions(template)
-    sans_ap_east_1 = set(boto3.session.Session().get_available_regions("ec2"))
+    sans_ap_east_1 = set(all_regions)
     sans_ap_east_1.remove("ap-east-1")
     assert all_regions - sans_ap_east_1 == {"ap-east-1"}
     assert result == {"000000000001": sans_ap_east_1, "000000000002": sans_ap_east_1, "000000000003": all_regions, "000000000004": all_regions}
@@ -204,7 +204,7 @@ def test_resolve_worker_templates_account_regions(test_index: AccountIndexInstan
     from starfleet.worker_ships.base_payload_schemas import BaseAccountRegionPayloadTemplate
     from starfleet.account_index.resolvers import resolve_worker_template_account_regions
 
-    all_regions = set(boto3.session.Session().get_available_regions("ec2"))
+    all_regions = get_all_regions()
 
     # This will test the "all regions" logic and some account exclusion and org root support:
     payload = """

tests/account_index/testing_plugins/basic_plugin/__init__.py

@@ -8,9 +8,12 @@
 :Author: Mike Grima <michael.grima@gemini.com>
 """
 import json
+import warnings
 from typing import Dict, Set
 
 from starfleet.account_index.plugins.starfleet_default_index import StarfleetDefaultAccountIndex
+from starfleet.utils.niceties import get_all_regions
+
 import tests.starfleet_included_plugins.account_index_generator
 
 
@@ -32,6 +35,34 @@ def __init__(self):  # noqa pylint: disable=super-init-not-called
         with open(path, "r", encoding="utf-8") as file:
             account_dict = json.loads(file.read())["accounts"]
 
+        # NOTE: Boto updates will change the supported regions for accounts. If we detect that the regions in the loaded index file are not the same as what boto3 now supports,
+        # then we will override what is in the JSON with whatever the latest regions supported by boto3 to prevent tests from breaking
+        # However, we will issue a warning if we detect this as it's not ideal to have the loaded tests not use the raw values from the file.
+        all_regions = sorted(get_all_regions())
+        append_regions = []
+
+        for region in all_regions:
+            if region not in account_dict["000000000001"]["Regions"]:
+                append_regions.append(region)
+
+        # If we have found missing regions, then boto3 was updated with new regions that are not present in the account index.
+        if append_regions:
+            corrected_regions = sorted(account_dict["000000000001"]["Regions"] + append_regions)
+
+            warnings.warn(
+                "[⚠️] Note: boto3 was updated and the generated index: `tests/starfleet_included_plugins/account_index_generator/generatedIndex.json` has an outdated list of "
+                "regions. Please update the generatedIndex.json file by following the instructions on this file (the one that emitted this warning)."
+            )
+
+            # Instructions:
+            # To update the generatedIndex.json file, you need to go to: src/starfleet/worker_ships/plugins/account_index_generator/ship.py,
+            # and at the bottom of the `AccountIndexGeneratorShip` class, there is code you need to uncomment (these instructions are there as well). You have to run the unit tests
+            # for the account index generator worker with those lines of code uncommented, and then clone over
+            # that file to 'tests/starfleet_included_plugins/account_index_generator/generatedIndex.json'. Once finished, remember to re-comment out those lines.
+
+            for account in account_dict.values():
+                account["Regions"] = corrected_regions
+
         self._load_inventory(account_dict)