Command line args to make local running easier

docs/about.md

@@ -136,6 +136,14 @@ There are 3 types of trigger:
  - **Time** — triggers each day at the given time.
  - **Interval** — triggers every given interval (i.e. 1 hour, 5 minutes)
 
+NOTE: if routemaster is down when a trigger would have fired then it won't fire.
+e.g. If you have a trigger of:
+```
+- time: 18h00m
+```
+Then this condition won't be evaluated if routemaster is down at 6PM. It doesn't persistently track
+that it has triggers upcoming to evaluate, so in this case the condition would be evaluated at
+the next time that routemaster is up at 6PM. 
 
 ### Data feeds
 

docs/hacking.md

@@ -1,5 +1,11 @@
 # Hacking on Routemaster
 
+First, ensure you have a user named `routemaster` in your local postgres, i.e. on your
+local psql execute:
+```
+CREATE USER routemaster;
+```
+
 You'll need to create a database for developing against and for running tests
 against. This can be done by running the `scripts/database/create_databases.sh`
 script. Full details of how the database, models & migrations are handled can

routemaster/app.py

@@ -1,7 +1,7 @@
 """Core App singleton that holds state for the application."""
 import threading
 import contextlib
-from typing import Dict, Optional
+from typing import Dict, Iterable, Optional
 
 from sqlalchemy.orm import Session, sessionmaker
 from sqlalchemy.engine import Engine
@@ -15,6 +15,10 @@
 )
 
 
+def _only_python_logger(logger_configs: Iterable):
+    return [x for x in logger_configs if 'PythonLogger' in x.dotted_path]
+
+
 class App(threading.local):
     """Core application state."""
 
@@ -23,13 +27,19 @@ class App(threading.local):
     logger: BaseLogger
     _current_session: Optional[Session]
     _webhook_runners: Dict[str, WebhookRunner]
+    use_local_urls: bool
+    only_python_logging: bool
 
     def __init__(
         self,
         config: Config,
+        use_local_urls: bool=False,
+        only_python_logging: bool=False,
     ) -> None:
         """Initialisation of the app state."""
         self.config = config
+        self.use_local_urls = use_local_urls
+        self.only_python_logging = only_python_logging
         self.initialise()
 
     def initialise(self):
@@ -40,9 +50,14 @@ def initialise(self):
         environment.
         """
         self._db = initialise_db(self.config.database)
+        logging_plugins = (
+            _only_python_logger(self.config.logging_plugins)
+            if self.only_python_logging
+            else self.config.logging_plugins
+        )
         self.logger = SplitLogger(
             self.config,
-            loggers=register_loggers(self.config),
+            loggers=register_loggers(self.config, logging_plugins),
         )
         self._sessionmaker = sessionmaker(self._db)
         self._current_session = None
@@ -51,7 +66,7 @@ def initialise(self):
         # Webhook runners may choose to persist a session, so we instantiate
         # up-front to ensure we re-use state.
         self._webhook_runners = {
-            x: webhook_runner_for_state_machine(y)
+            x: webhook_runner_for_state_machine(y, self.use_local_urls)
             for x, y in self.config.state_machines.items()
         }
 

routemaster/cli.py

@@ -23,8 +23,20 @@
     type=click.File(encoding='utf-8'),
     required=True,
 )
+@click.option(
+    '--local',
+    help="Override webhook URLs to point to localhost",
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    '--only-python-logging',
+    help="For local testing, only instantiate the basic PythonLogger plugin",
+    default=False,
+    is_flag=True,
+)
 @click.pass_context
-def main(ctx, config_file):
+def main(ctx, config_file, local, only_python_logging):
     """Shared entrypoint configuration."""
     logging.getLogger('schedule').setLevel(logging.CRITICAL)
 
@@ -34,7 +46,11 @@ def main(ctx, config_file):
         logger.exception("Configuration Error")
         click.get_current_context().exit(1)
 
-    ctx.obj = App(config)
+    ctx.obj = App(
+        config,
+        use_local_urls=local,
+        only_python_logging=only_python_logging,
+    )
     _validate_config(ctx.obj)
 
 

routemaster/config/loader.py

@@ -145,6 +145,7 @@ def _load_webhook(yaml: Yaml) -> Webhook:
     return Webhook(
         match=re.compile(yaml['match']),
         headers=yaml['headers'],
+        localport=yaml.get('localport'),
     )
 
 

routemaster/config/model.py

@@ -10,6 +10,7 @@
     Mapping,
     Pattern,
     Iterable,
+    Optional,
     Sequence,
     NamedTuple,
 )
@@ -162,6 +163,7 @@ class Webhook(NamedTuple):
     """Configuration for webdook requests."""
     match: Pattern
     headers: Dict[str, str]
+    localport: Optional[int]
 
 
 class StateMachine(NamedTuple):

routemaster/config/schema.yaml

@@ -34,7 +34,9 @@ properties:
                 type: string
               headers:
                 type: object
-            additionalProperties: false
+            additionalProperties:
+              localport:
+                type: integer
         states:
           title: States
           type: array

routemaster/config/tests/test_loading.py

@@ -93,6 +93,7 @@ def test_realistic_config():
                         headers={
                             'x-api-key': 'Rahfew7eed1ierae0moa2sho3ieB1et3ohhum0Ei',
                         },
+                        localport=None,
                     ),
                 ],
                 states=[
@@ -300,6 +301,7 @@ def test_environment_variables_override_config_file_for_database_config():
                         headers={
                             'x-api-key': 'Rahfew7eed1ierae0moa2sho3ieB1et3ohhum0Ei',
                         },
+                        localport=None,
                     ),
                 ],
                 states=[

routemaster/conftest.py

@@ -71,6 +71,7 @@
                 headers={
                     'x-api-key': 'Rahfew7eed1ierae0moa2sho3ieB1et3ohhum0Ei',
                 },
+                localport=None,
             ),
         ],
         states=[

routemaster/logging/plugins.py

@@ -1,5 +1,6 @@
 """Plugin loading and configuration."""
 import importlib
+from typing import Iterable
 
 from routemaster.config import Config, LoggingPluginConfig
 from routemaster.logging.base import BaseLogger
@@ -9,11 +10,15 @@ class PluginConfigurationException(Exception):
     """Raised to signal an invalid plugin that was loaded."""
 
 
-def register_loggers(config: Config):
+def register_loggers(
+    config: Config,
+    logger_configs: Iterable[LoggingPluginConfig]=None,
+):
     """
     Iterate through all plugins in the config file and instatiate them.
     """
-    return [_import_logger(config, x) for x in config.logging_plugins]
+    logger_configs = logger_configs or config.logging_plugins
+    return [_import_logger(config, x) for x in logger_configs]
 
 
 def _import_logger(

routemaster/webhooks.py

@@ -1,7 +1,8 @@
 """Webhook invocation."""
 
 import enum
-from typing import Any, Dict, Callable, Iterable
+from typing import Any, Dict, Callable, Iterable, Optional
+from urllib.parse import urlparse, urlunparse
 
 import requests
 
@@ -27,11 +28,16 @@ class RequestsWebhookRunner(object):
     Optionally takes a list of webhook configs to modify how requests are made.
     """
 
-    def __init__(self, webhook_configs: Iterable[Webhook]=()) -> None:
+    def __init__(
+        self,
+        webhook_configs: Iterable[Webhook]=(),
+        url_builder: Callable[[str, Optional[int]], str]=(lambda x, y: x),
+    ) -> None:
         # Use a session so that we can take advantage of connection pooling in
         # `urllib3`.
         self.session = requests.Session()
         self.webhook_configs = webhook_configs
+        self.url_builder = url_builder
 
     def __call__(
         self,
@@ -50,7 +56,7 @@ def __call__(
 
         try:
             result = self.session.post(
-                url,
+                self.url_builder(url, self._localport_for_url(url)),
                 data=data,
                 headers=headers,
                 timeout=10,
@@ -73,13 +79,39 @@ def _headers_for_url(self, url: str) -> Dict[str, Any]:
                 headers.update(config.headers)
         return headers
 
+    def _localport_for_url(self, url: str) -> Optional[int]:
+        for config in self.webhook_configs:
+            if config.match.search(url):
+                return config.localport
+        return None
+
+
+def _to_local_scheme_netloc(
+    base_value: tuple,
+    localport: Optional[int],
+) -> tuple:
+    if localport:
+        return ('http', 'localhost:{0}'.format(localport))
+
+    return base_value
+
+
+def _to_local_url(url: str, localport: Optional[int]) -> str:
+    parts = urlparse(url)
+    mapped_url = _to_local_scheme_netloc(parts[:2], localport) + parts[2:6]
+    return urlunparse(mapped_url)
+
 
 def webhook_runner_for_state_machine(
     state_machine: StateMachine,
+    only_use_localhost=False,
 ) -> WebhookRunner:
     """
     Create the webhook runner for a given state machine.
 
     Applies any state machine configuration to the runner.
     """
+    if only_use_localhost:
+        return RequestsWebhookRunner(state_machine.webhooks, _to_local_url)
+
     return RequestsWebhookRunner(state_machine.webhooks)