Use argparse config throughout app (#349)

This PR is a pretty substantial refactor of the entrypoints of pypiserver (`__main__` and `__init__`) to use the argparse-based config added in #339.

- Updated `RunConfig` and `UpdateConfig` classes to have exclusive init kwargs, instead of taking an namespace. This turned out to be much easier when working with the library-style app initialization in `__init__`, both for direct instantiation and via paste config
- Added an `iter_packages()` method to the `RunConfig` to iterate over packages specified by the configuration (note @elfjes, I think that replacing this with e.g. a `backend` reference will be a nice way to tie in #348)
- Added a general-purpose method to map legacy keyword arguments to the `app()` and `paste_app_factory()` functions to updated forms
- Refactored the `paste_app_factory()` to not mutate the incoming dictionary
- Removed all argument-parsing and config-related code from `__main__` and `core`
- Moved `_logwrite` from `__init__` to `__main__`, since that was the only place it was being used after the updates to `core`
- Updated `digest_file` to use `hashlib.new(algo)` instead of `getattr(hashlib, algo)`, because the former supports more algorithms
- Updated `setup.py` to, instead of calling `eval()` on the entirety of `__init__`, to instead just evaluate the line that defines the version
- Assigned the config to a `._pypiserver_config` attribute on the `Bottle` instance to reduce hacky test workarounds
- Fixed the tox config, which I broke in #339 

* Config: add auth & absolute path resolution

* Config: check pkg dirs on config creation

* Instantiate config with kwargs, not namespace

* WIP: still pulling the threads

* Init seems to be working

* tests passing locally, still need to update cache

* Fix tox command

* unused import

* Fix typing

* Be more selective in exec() in setup.py

* Require accurate casing for hash algos

* Remove old comment

* Comments, minor updates and simplifications

* move _logwrite to a more reasonable place

* Update config to work with cache

* Type cachemanager listdir in core

* Update config module docstring, rename method

* Add more comments re: paste config

* Add comments to main, remove unneded check

* Remove commented code

* Use {posargs} instead of [] for clarity in tox

* Add dupe check for kwarg updater

* Remove unused references on app instance

* Fix typo

* Remove redundancy in log level parsing

pypiserver/__init__.py

@@ -1,6 +1,11 @@
-import os
+import functools
+import pathlib
 import re as _re
 import sys
+import typing as t
+
+from pypiserver.bottle import Bottle
+from pypiserver.config import Config, RunConfig, strtobool
 
 version = __version__ = "2.0.0dev1"
 __version_info__ = tuple(_re.split("[.-]", __version__))
@@ -11,203 +16,191 @@
 __uri__ = "https://github.com/pypiserver/pypiserver"
 
 
-class Configuration:
-    """
-    .. see:: config-options: :func:`pypiserver.configure()`
-    """
+identity = lambda x: x
 
-    def __init__(self, **kwds):
-        vars(self).update(kwds)
 
-    def __repr__(self, *args, **kwargs):
-        return f"Configuration(**{vars(self)})"
+def backwards_compat_kwargs(kwargs: dict, warn: bool = True) -> dict:
+    """Return a dict with deprecated kwargs converted to new kwargs.
 
-    def __str__(self, *args, **kwargs):
-        return "Configuration:\n" + "\n".join(
-            f"{k:>20} = {v}" for k, v in sorted(vars(self).items())
+    :param kwargs: the incoming kwargs to convert
+    :param warn: whether to output a warning to stderr if there are deprecated
+        kwargs found in the incoming kwargs
+    """
+    # A mapping of deprecated kwargs to a 2-tuple of their corresponding updated
+    # kwarg and a function to convert the value of the deprecated kwarg to a
+    # value for the new kwarg. `identity` is just a function that returns
+    # whatever it is passed and is used in cases where the only change from
+    # a legacy kwarg is its name.
+    backwards_compat = {
+        "authenticated": ("authenticate", identity),
+        "passwords": ("password_file", identity),
+        # `root` could be a string or an array of strings. Handle both cases,
+        # converting strings to Path instances.
+        "root": (
+            "roots",
+            lambda root: [
+                # Convert strings to absolute Path instances
+                pathlib.Path(r).expanduser().resolve()
+                for r in ([root] if isinstance(root, str) else root)
+            ],
+        ),
+        # `redirect_to_fallback` was changed to `disable_fallback` for clearer
+        # use as a flag to disable the default behavior. Since its behavior
+        # is the opposite, we negate it.
+        "redirect_to_fallback": (
+            "disable_fallback",
+            lambda redirect: not redirect,
+        ),
+        "server": ("server_method", identity),
+        # `welcome_msg` now is just provided as text, so that anyone using
+        # pypiserver as a library doesn't need to worry about distributing
+        # files if they don't need to. If we're still passed an old-style
+        # `welcome_file` argument, we go ahead and resolve it to an absolute
+        # path and read the text.
+        "welcome_file": (
+            "welcome_msg",
+            lambda p: pathlib.Path(p).expanduser().resolve().read_text(),
+        ),
+    }
+    # Warn the user if they're using any deprecated arguments
+    if warn and any(k in backwards_compat for k in kwargs):
+        # Make nice instructions like `Please replace the following:
+        # 'authenticated' with 'authenticate'` and print to stderr.
+        replacement_strs = (
+            f"'{k}' with '{backwards_compat[k][0]}'"
+            for k in filter(lambda k: k in kwargs, backwards_compat)
+        )
+        warn_str = (
+            "You are using deprecated arguments. Please replace the following: \n"
+            f"  {', '.join(replacement_strs)}"
+        )
+        print(warn_str, file=sys.stderr)
+
+    # Create an iterable of 2-tuple to collect into the updated dictionary. Each
+    # item will either be the existing key-value pair from kwargs, or, if the
+    # keyword is a legacy keyword, the new key and potentially adjusted value
+    # for that keyword. Note that depending on the order the argument are
+    # specified, this _could_ mean an updated legacy keyword could override
+    # a new argument if that argument is also specified. However, in that
+    # case, our updated kwargs dictionary would have a different number of
+    # keys compared to our incoming dictionary, so we check for that case
+    # below.
+    rv_iter = (
+        (
+            (k, v)
+            if k not in backwards_compat
+            else (backwards_compat[k][0], backwards_compat[k][1](v))
+        )
+        for k, v in kwargs.items()
+    )
+    updated_kwargs = dict(rv_iter)
+
+    # If our dictionaries have different lengths, we must have gotten duplicate
+    # legacy/modern keys. Figure out which keys were dupes and throw an error.
+    if len(updated_kwargs) != len(kwargs):
+        legacy_to_modern = {k: v[0] for k, v in backwards_compat.items()}
+        dupes = [
+            (k, v)
+            for k, v in legacy_to_modern.items()
+            if k in kwargs and v in kwargs
+        ]
+        raise ValueError(
+            "Keyword arguments for pypiserver app() constructor contained "
+            "duplicate legacy and modern keys. Duplicates are shown below, in "
+            "the form (legacy_key, modern_key):\n"
+            f"{dupes}"
         )
 
-    def update(self, props):
-        d = props if isinstance(props, dict) else vars(props)
-        vars(self).update(d)
-
-
-DEFAULT_SERVER = "auto"
-
-
-def default_config(
-    root=None,
-    host="0.0.0.0",
-    port=8080,
-    server=DEFAULT_SERVER,
-    redirect_to_fallback=True,
-    fallback_url=None,
-    authenticated=["update"],
-    password_file=None,
-    overwrite=False,
-    hash_algo="md5",
-    verbosity=1,
-    log_file=None,
-    log_stream="stderr",
-    log_frmt="%(asctime)s|%(name)s|%(levelname)s|%(thread)d|%(message)s",
-    log_req_frmt="%(bottle.request)s",
-    log_res_frmt="%(status)s",
-    log_err_frmt="%(body)s: %(exception)s \n%(traceback)s",
-    welcome_file=None,
-    cache_control=None,
-    auther=None,
-    VERSION=__version__,
-):
-    """
-    Fetch default-opts with overridden kwds, capable of starting-up pypiserver.
-
-    Does not validate overridden options.
-    Example usage::
-
-        kwds = pypiserver.default_config(<override_kwds> ...)
-        ## More modifications on kwds.
-        pypiserver.app(**kwds)``.
-
-    Kwds correspond to same-named cmd-line opts, with '-' --> '_' substitution.
-    Non standard args are described below:
-
-    :param return_defaults_only:
-            When `True`, returns defaults, otherwise,
-            configures "runtime" attributes and returns also the "packages"
-            found in the roots.
-    :param root:
-            A list of paths, derived from the packages specified on cmd-line.
-            If `None`, defaults to '~/packages'.
-    :param redirect_to_fallback:
-            see :option:`--disable-fallback`
-    :param authenticated:
-            see :option:`--authenticate`
-    :param password_file:
-            see :option:`--passwords`
-    :param log_file:
-            see :option:`--log-file`
-            Not used, passed here for logging it.
-    :param log_frmt:
-            see :option:`--log-frmt`
-            Not used, passed here for logging it.
-    :param callable auther:
-            An API-only options that if it evaluates to a callable,
-            it is invoked to allow access to protected operations
-            (instead of htpaswd mechanism) like that::
-
-                auther(username, password): bool
-
-            When defined, `password_file` is ignored.
-    :param host:
-            see :option:`--interface`
-            Not used, passed here for logging it.
-    :param port:
-            see :option:`--port`
-            Not used, passed here for logging it.
-    :param server:
-            see :option:`--server`
-            Not used, passed here for logging it.
-    :param verbosity:
-            see :option:`-v`
-            Not used, passed here for logging it.
-    :param VERSION:
-            Not used, passed here for logging it.
-
-    :return: a dict of defaults
+    return updated_kwargs
 
-    """
-    return locals()
 
+def app(**kwargs: t.Any) -> Bottle:
+    """Construct a bottle app running pypiserver.
 
-def app(**kwds):
+    :param kwds: Any overrides for defaults. Any property of RunConfig
+        (or its base), defined in `pypiserver.config`, may be overridden.
     """
-    :param dict kwds: Any overrides for defaults, as fetched by
-        :func:`default_config()`. Check the docstring of this function
-        for supported kwds.
-    """
-    from . import core
+    config = Config.default_with_overrides(**backwards_compat_kwargs(kwargs))
+    return app_from_config(config)
+
 
+def app_from_config(config: RunConfig) -> Bottle:
+    """Construct a bottle app from the provided RunConfig."""
+    # The _app module instantiates a Bottle instance directly when it is
+    # imported. That is `_app.app`. We directly mutate some global variables
+    # on the imported `_app` module so that its endpoints will behave as
+    # we expect.
     _app = __import__("_app", globals(), locals(), ["."], 1)
+    # Because we're about to mutate our import, we pop it out of the imported
+    # modules map, so that any future imports do not receive our mutated version
     sys.modules.pop("pypiserver._app", None)
-
-    kwds = default_config(**kwds)
-    config, packages = core.configure(**kwds)
     _app.config = config
-    _app.packages = packages
-    _app.app.module = _app  # HACK for testing.
-
+    # Add a reference to our config on the Bottle app for easy access in testing
+    # and other contexts.
+    _app.app._pypiserver_config = config
     return _app.app
 
 
-def str2bool(s, default):
-    if s is not None and s != "":
-        return s.lower() not in ("no", "off", "0", "false")
-    return default
-
-
-def _str_strip(string):
-    """Provide a generic strip method to pass as a callback."""
-    return string.strip()
-
-
-def paste_app_factory(global_config, **local_conf):
-    """Parse a paste config and return an app."""
-
-    def upd_conf_with_bool_item(conf, attr, sdict):
-        conf[attr] = str2bool(sdict.pop(attr, None), conf[attr])
-
-    def upd_conf_with_str_item(conf, attr, sdict):
-        value = sdict.pop(attr, None)
-        if value is not None:
-            conf[attr] = value
-
-    def upd_conf_with_int_item(conf, attr, sdict):
-        value = sdict.pop(attr, None)
-        if value is not None:
-            conf[attr] = int(value)
-
-    def upd_conf_with_list_item(conf, attr, sdict, sep=" ", parse=_str_strip):
-        values = sdict.pop(attr, None)
-        if values:
-            conf[attr] = list(filter(None, map(parse, values.split(sep))))
-
-    def _make_root(root):
-        root = root.strip()
-        if root.startswith("~"):
-            return os.path.expanduser(root)
-        return root
-
-    c = default_config()
-
-    upd_conf_with_bool_item(c, "overwrite", local_conf)
-    upd_conf_with_bool_item(c, "redirect_to_fallback", local_conf)
-    upd_conf_with_list_item(c, "authenticated", local_conf, sep=" ")
-    upd_conf_with_list_item(c, "root", local_conf, sep="\n", parse=_make_root)
-    upd_conf_with_int_item(c, "verbosity", local_conf)
-    str_items = [
-        "fallback_url",
-        "hash_algo",
-        "log_err_frmt",
-        "log_file",
-        "log_frmt",
-        "log_req_frmt",
-        "log_res_frmt",
-        "password_file",
-        "welcome_file",
-    ]
-    for str_item in str_items:
-        upd_conf_with_str_item(c, str_item, local_conf)
-    # cache_control is undocumented; don't know what type is expected:
-    # upd_conf_with_str_item(c, 'cache_control', local_conf)
-
-    return app(**c)
-
-
-def _logwrite(logger, level, msg):
-    if msg:
-        line_endings = ["\r\n", "\n\r", "\n"]
-        for le in line_endings:
-            if msg.endswith(le):
-                msg = msg[: -len(le)]
-        if msg:
-            logger.log(level, msg)
+T = t.TypeVar("T")
+
+
+def paste_app_factory(_global_config, **local_conf):
+    """Parse a paste config and return an app.
+
+    The paste config is entirely strings, so we need to parse those
+    strings into values usable for the config, if they're present.
+    """
+
+    def to_bool(val: t.Optional[str]) -> t.Optional[bool]:
+        """Convert a string value, if provided, to a bool."""
+        return val if val is None else strtobool(val)
+
+    def to_int(val: t.Optional[str]) -> t.Optional[int]:
+        """Convert a string value, if provided, to an int."""
+        return val if val is None else int(val)
+
+    def to_list(
+        val: t.Optional[str],
+        sep: str = " ",
+        transform: t.Callable[[str], T] = str.strip,
+    ) -> t.Optional[t.List[T]]:
+        """Convert a string value, if provided, to a list.
+
+        :param sep: the separator between items in the string representation
+            of the list
+        :param transform: an optional function to call on each string item of
+            the list
+        """
+        if val is None:
+            return val
+        return list(filter(None, map(transform, val.split(sep))))
+
+    def _make_root(root: str) -> pathlib.Path:
+        """Convert a specified string root into an absolute Path instance."""
+        return pathlib.Path(root.strip()).expanduser().resolve()
+
+    # A map of config keys we expect in the paste config to the appropriate
+    # function to parse the string config value. This map includes both
+    # current and legacy keys.
+    maps = {
+        "cache_control": to_int,
+        "roots": functools.partial(to_list, sep="\n", transform=_make_root),
+        # root is a deprecated argument for roots
+        "root": functools.partial(to_list, sep="\n", transform=_make_root),
+        "disable_fallback": to_bool,
+        # redirect_to_fallback is a deprecated argument for disable_fallback
+        "redirect_to_fallback": to_bool,
+        "overwrite": to_bool,
+        "authenticate": functools.partial(to_list, sep=" "),
+        # authenticated is a deprecated argument for authenticate
+        "authenticated": functools.partial(to_list, sep=" "),
+        "verbosity": to_int,
+    }
+
+    # First, convert values from strings to whatever types we need, or leave
+    # them as strings if there's no mapping function available for them.
+    mapped_conf = {k: maps.get(k, identity)(v) for k, v in local_conf.items()}
+    # Convert any legacy key/value pairs into their modern form.
+    updated_conf = backwards_compat_kwargs(mapped_conf)
+
+    return app(**updated_conf)