Move hgmo changeset handling into new module

Move various functions for working with hg.mozilla.org data into their
own module.

Author: mars-f

committelemetry/hgmo.py

@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Functions for interacting with hg.mozilla.org APIs.
+"""
+import logging
+from typing import List
+
+from committelemetry.http import requests_retry_session
+from committelemetry.sentry import client as sentry
+
+log = logging.getLogger(__name__)
+
+
+def changesets_for_pushid(pushid: int, push_json_url: str) -> List[str]:
+    """Return a list of changeset IDs in a repository push.
+
+    Reads data published by the Mozilla hgweb pushlog extension.
+
+    Also see https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/pushlog.html#writing-agents-that-consume-pushlog-data
+
+    Args:
+        pushid: The integer pushlog pushid we want information about.
+        push_json_url: The 'push_json_url' field from a hgpush message.
+            See https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/notifications.html#changegroup-1
+            The pushid in the URL should match the pushid argument to this
+            function.
+
+    Returns:
+        A list of changeset ID strings (40 char hex strings).
+    """
+    log.info(f'processing pushid {pushid}')
+    sentry.extra_context({'pushid': pushid})
+    response = requests_retry_session().get(push_json_url)
+    response.raise_for_status()
+
+    # See https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/pushlog.html#version-2
+    changesets = response.json()['pushes'][str(pushid)]['changesets']
+    log.info(f'got {len(changesets)} changesets for pushid {pushid}')
+    return changesets
+
+
+def fetch_changeset(changeset_id, repo_url):
+    # Example URL: https://hg.mozilla.org/mozilla-central/json-rev/deafa2891c61
+    response = requests_retry_session(
+    ).get(f'{repo_url}/json-rev/{changesetid}')
+    if response.status_code == 404:
+        raise NoSuchChangeset(
+            f'The changeset {changesetid} does not exist in repository {repo_url}'  # yapf:disable
+        )
+    response.raise_for_status()
+    return response.json()
+
+
+def utc_hgwebdate(hgweb_datejson):
+    """Turn a (unixtime, offset) tuple back into a UTC Unix timestamp.
+
+    Pushlog entries are not in UTC, but a tuple of (local-unixtime, utc-offset)
+    created by
+    https://www.mercurial-scm.org/repo/hg/file/8b86acc7aa64/mercurial/utils/dateutil.py#l63.
+    This function reverses the operation that created the tuple.
+
+    Args:
+        hgweb_datejson: A 2-element JSON list of ints.
+            For example: https://hg.mozilla.org/mozilla-central/json-rev/deafa2891c61
+            See https://www.mercurial-scm.org/repo/hg/file/8b86acc7aa64/mercurial/utils/dateutil.py#l63
+            for how this value is created.
+
+    Returns:
+        The UTC Unix time (seconds since the epoch), as an int.
+    """
+    assert len(hgweb_datejson) == 2
+    timestamp, offset = hgweb_datejson
+    return timestamp + offset
+
+
+class Error(Exception):
+    """Generic error class for this module."""
+
+
+class NoSuchChangeset(Error):
+    """Raised if the given changeset ID does not exist in the target system."""

committelemetry/pulse.py

@@ -10,12 +10,11 @@
 import socket
 from contextlib import closing
 from functools import partial
-from typing import List
 
 from kombu import Connection, Exchange, Queue
 
 from committelemetry import config
-from committelemetry.http import requests_retry_session
+from committelemetry.hgmo import changesets_for_pushid
 from committelemetry.telemetry import payload_for_changeset, send_ping
 from committelemetry.sentry import client as sentry
 
@@ -26,34 +25,6 @@ def noop(*args, **kwargs):
     return None
 
 
-def changesets_for_pushid(pushid: int, push_json_url: str) -> List[str]:
-    """Return a list of changeset IDs in a repository push.
-
-    Reads data published by the Mozilla hgweb pushlog extension.
-
-    Also see https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/pushlog.html#writing-agents-that-consume-pushlog-data
-
-    Args:
-        pushid: The integer pushlog pushid we want information about.
-        push_json_url: The 'push_json_url' field from a hgpush message.
-            See https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/notifications.html#changegroup-1
-            The pushid in the URL should match the pushid argument to this
-            function.
-
-    Returns:
-        A list of changeset ID strings (40 char hex strings).
-    """
-    log.info(f'processing pushid {pushid}')
-    sentry.extra_context({'pushid': pushid})
-    response = requests_retry_session().get(push_json_url)
-    response.raise_for_status()
-
-    # See https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/pushlog.html#version-2
-    changesets = response.json()['pushes'][str(pushid)]['changesets']
-    log.info(f'got {len(changesets)} changesets for pushid {pushid}')
-    return changesets
-
-
 def process_push_message(body, message, no_send=False):
     """Process a hg push message from Mozilla Pulse.
 

committelemetry/telemetry.py

@@ -12,7 +12,7 @@
 
 from committelemetry import config
 from committelemetry.classifier import determine_review_system
-from committelemetry.http import requests_retry_session
+from committelemetry.hgmo import fetch_changeset, utc_hgwebdate
 
 log = logging.getLogger(__name__)
 
@@ -49,18 +49,6 @@ def payload_for_changeset(changesetid: str, repo_url: str) -> Dict:
     }
 
 
-def fetch_changeset(changeset_id, repo_url):
-    # Example URL: https://hg.mozilla.org/mozilla-central/json-rev/deafa2891c61
-    response = requests_retry_session(
-    ).get(f'{repo_url}/json-rev/{changesetid}')
-    if response.status_code == 404:
-        raise NoSuchChangeset(
-            f'The changeset {changesetid} does not exist in repository {repo_url}'
-        )
-    response.raise_for_status()
-    return response.json()
-
-
 def send_ping(ping_id, payload):
     """Send an event ping to the Mozilla telemetry service.
 
@@ -87,31 +75,3 @@ def send_ping(ping_id, payload):
     response.raise_for_status()
 
 
-def utc_hgwebdate(hgweb_datejson):
-    """Turn a (unixtime, offset) tuple back into a UTC Unix timestamp.
-
-    Pushlog entries are not in UTC, but a tuple of (local-unixtime, utc-offset)
-    created by
-    https://www.mercurial-scm.org/repo/hg/file/8b86acc7aa64/mercurial/utils/dateutil.py#l63.
-    This function reverses the operation that created the tuple.
-
-    Args:
-        hgweb_datejson: A 2-element JSON list of ints.
-            For example: https://hg.mozilla.org/mozilla-central/json-rev/deafa2891c61
-            See https://www.mercurial-scm.org/repo/hg/file/8b86acc7aa64/mercurial/utils/dateutil.py#l63
-            for how this value is created.
-
-    Returns:
-        The UTC Unix time (seconds since the epoch), as an int.
-    """
-    assert len(hgweb_datejson) == 2
-    timestamp, offset = hgweb_datejson
-    return timestamp + offset
-
-
-class Error(Exception):
-    """Generic error class for this module."""
-
-
-class NoSuchChangeset(Error):
-    """Raised if the given changeset ID does not exist in the target system."""

committelemetry/tool.py

@@ -9,7 +9,8 @@
 
 from committelemetry.pulse import run_pulse_listener
 from committelemetry.pushlog import send_pings_by_pushid
-from .telemetry import NoSuchChangeset, payload_for_changeset
+from .telemetry import payload_for_changeset
+from committelemetry.hgmo import NoSuchChangeset
 
 
 @click.command()