support additional indexes from Poetry configuration (relevant also for

#52 to support explicit sources in the dependency config)

src/slap/ext/application/install.py

@@ -11,7 +11,6 @@
 
 from slap.application import Application, Command, option
 from slap.configuration import Configuration
-from slap.install.installer import InstallOptions
 from slap.plugins import ApplicationPlugin
 from slap.project import Project
 
@@ -115,7 +114,7 @@ def handle(self) -> int:
     """
 
     from nr.util.stream import Stream
-    from slap.install.installer import PipInstaller
+    from slap.install.installer import InstallOptions, PipInstaller, get_indexes_for_projects
     from slap.python.dependency import PathDependency, PypiDependency, parse_dependencies
     from slap.python.environment import PythonEnvironment
 
@@ -147,21 +146,23 @@ def handle(self) -> int:
     # Collect the run dependencies to install.
     for project in projects_plus_dependencies:
       assert project.is_python_project, 'Project.is_python_project is deprecated and expected to always be true'
+      deps = project.dependencies()
 
       if not self.option("no-root") and not self.option("link") and not self.option("only-extras") and project.packages():
         # Install the project itself directory unless certain flags turn this behavior off.
         dependencies.append(PathDependency(project.dist_name() or project.id, project.directory))
 
       elif not self.option("only-extras"):
         # Install the run dependencies of the project.
-        dependencies += project.dependencies().run
+        dependencies += deps.run
 
     # Collect dev dependencies and extras from the project.
     for project in projects:
+      deps = project.dependencies()
 
       if (not self.option("no-dev") and not self.option("only-extras")) or 'dev' in install_extras:
         # Install the development dependencies of the project.
-        dependencies += project.dependencies().dev
+        dependencies += deps.dev
 
       # Determine the extras to install for the current project. This changes on development installs because
       # we always consider the ones configured in #InstallConfig.dev_extras, or _all_ extras if the option is
@@ -170,15 +171,15 @@ def handle(self) -> int:
       if not self.option("no-dev"):
         config = self.config[project]
         if config.dev_extras is None:
-          current_project_install_extras.update(project.dependencies().extra.keys())  # Use all extras defined in the project
+          current_project_install_extras.update(deps.extra.keys())  # Use all extras defined in the project
           current_project_install_extras.update(config.extras.keys())  # Use all extras defined in the #InstallConfig
         else:
           current_project_install_extras.update(config.dev_extras)  # Use only the extras explicitly defined in the #InstallConfig
 
       # Append the extra dependencies from the project. We ignore 'dev' here because we already took care of
       # deciding when to install dev dependencies.
       for extra in current_project_install_extras:
-        extra_deps = project.dependencies().extra.get(extra)
+        extra_deps = deps.extra.get(extra)
         if extra_deps is not None:
           discovered_extras.add(extra)
           dependencies += extra_deps
@@ -200,8 +201,12 @@ def handle(self) -> int:
       if not (isinstance(dependency, PypiDependency) and dependency.name in project_names)
     ]
 
+    options = InstallOptions(
+      quiet=self.option("quiet"),
+      indexes=get_indexes_for_projects(projects),
+    )
     installer = PipInstaller(self)
-    status_code = installer.install(dependencies, python_environment, InstallOptions(quiet=self.option("quiet")))
+    status_code = installer.install(dependencies, python_environment, options)
     if status_code != 0:
       return status_code
 

src/slap/ext/project_handlers/poetry.py

@@ -48,17 +48,27 @@ def get_packages(self, project: Project) -> list[Package] | None:
 
   def get_dependencies(self, project: Project) -> Dependencies:
     from slap.python.dependency import PypiDependency, parse_dependencies
+    from slap.install.installer import Indexes
 
     poetry: dict[str, t.Any] = project.pyproject_toml.get('tool', {}).get('poetry', {})
     dependencies = parse_dependencies(poetry.get('dependencies', []))
     python = next((d for d in dependencies if d.name == 'python'), None)
     if python is not None:
       assert isinstance(python, PypiDependency), repr(python)
+
+    # Collect the package indexes from the Poetry config.
+    indexes = Indexes()
+    for source in poetry.get('source', []):
+      if source.get('default', True):
+        indexes.default = source['name']
+      indexes.urls[source['name']] = source['url']
+
     return Dependencies(
-      python.version if python else None,
-      [d for d in dependencies if d.name != 'python'],
-      parse_dependencies(poetry.get('dev-dependencies', [])),
-      {k: parse_dependencies(v) for k, v in poetry.get('extras', {}).items()},
+      python=python.version if python else None,
+      run=[d for d in dependencies if d.name != 'python'],
+      dev=parse_dependencies(poetry.get('dev-dependencies', [])),
+      extra={k: parse_dependencies(v) for k, v in poetry.get('extras', {}).items()},
+      indexes=indexes,
     )
 
   def get_dependency_location_key_sequence(
@@ -71,5 +81,3 @@ def get_dependency_location_key_sequence(
     locator = ['dependencies'] if where == 'run' else ['dev-dependencies'] if where == 'dev' else ['extras', where]
     value: list | dict = {package: str(version_spec)} if where in ('run', 'dev') else [f'{package} {version_spec}']
     return ['tool', 'poetry'] + locator, value
-
-

src/slap/install/installer.py

@@ -15,15 +15,41 @@
 from slap.python.pep508 import filter_dependencies, test_dependency
 
 if t.TYPE_CHECKING:
+  from slap.repository import Repository
+  from slap.project import Project
   from slap.python.dependency import Dependency
   from slap.python.environment import PythonEnvironment
 
 logger = logging.getLogger(__name__)
 
 
+@dataclasses.dataclass
+class Indexes:
+  """ Represents a configuration of PyPI indexes. """
+
+  #: The name of the default index in the #urls mapping.
+  default: str | None = None
+
+  #: A mapping that assigns each key (the name of the index) its index URL.
+  urls: dict[str, str] = dataclasses.field(default_factory=dict)
+
+  def combine_with(self, other: Indexes) -> None:
+    if other.default and self.default and other.default != self.default:
+      logger.warning(
+        'Conflicting default index between projects in repository: %r (current), %r',
+        self.default, other.default
+      )
+    if not self.default:
+      self.default = other.default
+
+    # TODO (@NiklasRosenstein): Warn about conflicting package indexes.
+    self.urls = {**other.urls, **self.urls}
+
+
 @dataclasses.dataclass
 class InstallOptions:
   quiet: bool
+  indexes: Indexes
 
 
 class Installer(abc.ABC):
@@ -58,14 +84,16 @@ def __init__(self,symlink_helper: SymlinkHelper) -> None:
 
   def install(self, dependencies: t.Sequence[Dependency], target: PythonEnvironment, options: InstallOptions) -> int:
 
-    from slap.python.dependency import GitDependency, PathDependency, PypiDependency, UrlDependency
+    from slap.python.dependency import PathDependency, PypiDependency, UrlDependency
 
     # Collect the Pip arguments and the dependencies that need to be installed through other methods.
     supports_hashes = {PypiDependency, UrlDependency}
     unsupported_hashes: dict[type[Dependency], list[Dependency]] = {}
     link_projects: list[Path] = []
     pip_arguments: list[str] = []
+    used_indexes: set[str] = set()
     dependencies = list(dependencies)
+
     while dependencies:
       dependency = dependencies.pop()
 
@@ -87,7 +115,7 @@ def install(self, dependencies: t.Sequence[Dependency], target: PythonEnvironmen
         link_projects.append(dependency.path)
         continue
 
-      elif isinstance(dependency, MultiDependency):
+      if isinstance(dependency, MultiDependency):
         for sub_dependency in dependency:
           # TODO (@NiklasRosenstein): Pass extras from the caller so we can evaluate them here
           if test_dependency(sub_dependency, target.pep508, {}):
@@ -96,6 +124,22 @@ def install(self, dependencies: t.Sequence[Dependency], target: PythonEnvironmen
       else:
         pip_arguments += self.dependency_to_pip_arguments(dependency)
 
+      if isinstance(dependency, PypiDependency) and dependency.source:
+        used_indexes.add(dependency.source)
+
+    # Add the extra index URLs.
+    # TODO (@NiklasRosenstein): Inject credentials for index URLs.
+    # NOTE (@NiklasRosenstein): While the dependency configuration allows you to specify exactly for each
+    #   dependency where it should be fetched from, with the Pip CLI we cannot currently have that level
+    #   of control.
+    try:
+      if options.indexes.default is not None:
+        pip_arguments += ['--index-url', options.indexes.urls[options.indexes.default]]
+      for index_name in used_indexes - {options.indexes.default}:
+        pip_arguments += ['--extra-index-url', options.indexes.urls[index_name]]
+    except KeyError as exc:
+      raise Exception(f'PyPI index {exc} is not configured')
+
     # Construct the Pip command to run.
     pip_command = [target.executable, "-m", "pip", "install"] + pip_arguments
     if options.quiet:
@@ -162,3 +206,12 @@ def dependency_to_pip_arguments(dependency: Dependency) -> list[str]:
 
     assert pip_arguments, dependency
     return pip_arguments
+
+
+def get_indexes_for_projects(projects: t.Sequence[Project]) -> Indexes:
+  """ Combines the indexes configuration from each project into one index. """
+
+  indexes = Indexes()
+  for project in projects:
+    indexes.combine_with(project.dependencies().indexes)
+  return indexes

src/slap/project.py

@@ -13,10 +13,11 @@
 
 if t.TYPE_CHECKING:
   from nr.util.functional import Once
-  from slap.python.dependency import VersionSpec
-  from slap.repository import Repository
+  from slap.install.installer import Indexes
   from slap.plugins import ProjectHandlerPlugin
+  from slap.python.dependency import VersionSpec
   from slap.release import VersionRef
+  from slap.repository import Repository
 
 
 logger = logging.getLogger(__name__)
@@ -28,6 +29,12 @@ class Dependencies:
   run: t.Sequence[Dependency]
   dev: t.Sequence[Dependency]
   extra: t.Mapping[str, t.Sequence[Dependency]]
+  indexes: Indexes = None  # type: ignore  # To avoid having to import the Indexes class globally
+
+  def __post_init__(self) -> None:
+    from slap.install.installer import Indexes
+    if self.indexes is None:
+      self.indexes = Indexes()
 
 
 @dataclasses.dataclass