[v2] Implement lazy plugin initialization

.changes/next-release/enhancement-Performance-9167.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "Performance",
+  "description": "Defer loading of built-in plugins until they are actually needed to reduce initialization overhead."
+}

awscli/clidriver.py

@@ -64,11 +64,13 @@
     construct_entry_point_handlers_chain,
 )
 from awscli.formatter import get_formatter
+from awscli.handlers_registry import MAIN_COMMAND_TABLE_OPS
 from awscli.help import (
     OperationHelpCommand,
     ProviderHelpCommand,
     ServiceHelpCommand,
 )
+from awscli.lazy_emitter import LazyInitEmitter
 from awscli.logger import (
     disable_crt_logging,
     enable_crt_logging,
@@ -117,7 +119,10 @@ def create_clidriver(args=None):
         parser = FirstPassGlobalArgParser()
         args, _ = parser.parse_known_args(args)
         debug = args.debug
-    session = botocore.session.Session()
+    lazy_emitter = LazyInitEmitter(
+        main_command_table_ops=MAIN_COMMAND_TABLE_OPS
+    )
+    session = botocore.session.Session(event_hooks=lazy_emitter)
     _set_user_agent_for_session(session)
     load_plugins(
         session.full_config.get('plugins', {}),

awscli/plugin.py

@@ -10,15 +10,19 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import importlib
 import logging
 import os
 import sys
+from functools import singledispatch
 
 from botocore.hooks import HierarchicalEmitter
 
+from awscli.handlers_registry import PLUGIN_REGISTRY
+from awscli.lazy_emitter import LazyInitEmitter
+
 log = logging.getLogger('awscli.plugin')
 
-BUILTIN_PLUGINS = {'__builtin__': 'awscli.handlers'}
 CLI_LEGACY_PLUGIN_PATH = 'cli_legacy_plugin_path'
 
 
@@ -31,21 +35,21 @@ def load_plugins(plugin_mapping, event_hooks=None, include_builtins=True):
 
     :type event_hooks: ``EventHooks``
     :param event_hooks: Event hook emitter.  If one if not provided,
-        an emitter will be created and returned.  Otherwise, the
+        a LazyInitEmitter will be created and returned.  Otherwise, the
         passed in ``event_hooks`` will be used to initialize plugins.
 
     :type include_builtins: bool
-    :param include_builtins: If True, the builtin awscli plugins (specified in
-        ``BUILTIN_PLUGINS``) will be included in the list of plugins to load.
+    :param include_builtins: If True, the built-in plugin registry will be
+        loaded into the emitter.
 
-    :rtype: HierarchicalEmitter
+    :rtype: LazyInitEmitter
     :return: An event emitter object.
 
     """
     if event_hooks is None:
-        event_hooks = HierarchicalEmitter()
+        event_hooks = LazyInitEmitter()
     if include_builtins:
-        _load_plugins(BUILTIN_PLUGINS, event_hooks)
+        _load_registry(event_hooks)
     plugin_path = plugin_mapping.pop(CLI_LEGACY_PLUGIN_PATH, None)
     if plugin_path is not None:
         _add_plugin_path_to_sys_path(plugin_path)
@@ -58,6 +62,32 @@ def load_plugins(plugin_mapping, event_hooks=None, include_builtins=True):
     return event_hooks
 
 
+@singledispatch
+def _load_registry(event_hooks):
+    raise NotImplementedError(
+        f'No _load_registry implementation for '
+        f'{type(event_hooks).__name__}'
+    )
+
+
+@_load_registry.register
+def _(event_hooks: HierarchicalEmitter):
+    seen = set()
+    for event_pattern, entries in PLUGIN_REGISTRY.items():
+        for entry in entries:
+            if entry not in seen:
+                seen.add(entry)
+                module_path, fn_name = entry
+                mod = importlib.import_module(module_path)
+                fn = getattr(mod, fn_name)
+                fn(event_hooks)
+
+
+@_load_registry.register
+def _(event_hooks: LazyInitEmitter):
+    event_hooks.load_registry(PLUGIN_REGISTRY)
+
+
 def _load_plugins(plugin_mapping, event_hooks):
     modules = _import_plugins(plugin_mapping)
     for name, plugin in zip(plugin_mapping.keys(), modules):

exe/pyinstaller/hook-awscli.py

@@ -2,6 +2,8 @@
 
 from PyInstaller.utils import hooks
 
+from awscli.handlers_registry import PLUGIN_REGISTRY
+
 hiddenimports = [
     'docutils',
     'urllib',
@@ -28,6 +30,14 @@
 ) + hooks.collect_submodules('awscli.s3transfer')
 hiddenimports += alias_packages_plugins
 
+# plugin.py uses importlib.import_module at runtime to load customization
+# modules, so PyInstaller cannot discover them statically. Collect all module
+# paths referenced in handlers_registry.py as hidden imports.
+registry_modules = {
+    entry[0] for entries in PLUGIN_REGISTRY.values() for entry in entries
+}
+hiddenimports += registry_modules
+
 
 # Completion model files are only used at build time to generate the
 # ac.index SQLite database. They are not needed at runtime and can be

tests/functional/autocomplete/test_server_index.py

@@ -25,7 +25,7 @@ def setUpClass(cls):
             ],
         )
         driver = clidriver.create_clidriver()
-        driver.session.register(
+        driver.session.get_component('event_emitter').register_last(
             'building-command-table.main', _ddb_only_command_table
         )
         index_generator.generate_index(driver)
@@ -111,7 +111,7 @@ def test_no_errors_when_missing_completion_data(self):
         # This will result in the loader not being able to find any
         # completion data, which allows us to verify the behavior when
         # there's no completion data.
-        driver.session.register(
+        driver.session.get_component('event_emitter').register_last(
             'building-command-table.main', _ddb_only_command_table
         )
         driver.session.register(

tests/functional/autoprompt/test_prompttoolkit.py

@@ -44,7 +44,7 @@ def _generate_index_if_needed(db_connection):
             [indexer.ModelIndexer(db_connection)],
         )
         driver = create_clidriver()
-        driver.session.register(
+        driver.session.get_component('event_emitter').register_last(
             'building-command-table.main', _cloudwatch_only_command_table
         )
         index_generator.generate_index(driver)