Migrate automation to GitHub action.

.github/actions/data-update/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.8
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY ./src/ .
+
+CMD [ "python", "/usr/src/app/main.py" ]

.github/actions/data-update/action.yml

@@ -0,0 +1,5 @@
+name: 'Data Update'
+description: 'Update contributions & people form the GitHub API.'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/data-update/src/consts.py

@@ -0,0 +1,13 @@
+# Env var names
+ENV_GITHUB_PAT = "GITHUB_PAT"
+ENV_DATA_DIR = "DATA_DIR"
+
+# API and rate limits
+API_REQUEST_DELAY_SEC = 1
+RATE_LIMIT_BUFFER_SEC = 60
+RATE_LIMIT_MAX_AGE_SEC = 30
+MAX_RETRIES = 1
+
+# Files
+CONTRIBUTIONS_FILENAME = "contributions.json"
+PEOPLE_FILENAME = "people.json"

.github/actions/data-update/src/context.py

@@ -0,0 +1,59 @@
+import json
+import os
+from pathlib import Path
+
+import consts
+import log
+
+
+class Context:
+    """
+    Represents an application context containing shared information such as configurations, paths and secrets.
+    """
+
+    def __init__(self, out_dir_path, github_token):
+        """
+        Private constructor.
+        """
+        self._data_dir_path = out_dir_path
+        self._github_token = github_token
+
+    @staticmethod
+    def _read_config_file(main_script_path):
+        file_path = main_script_path.parent.joinpath("config.json")
+        try:
+            with open(file_path) as infile:
+                return json.load(infile)
+        except FileNotFoundError:
+            log.abort_and_exit("CTXT", f"Context init failed, could not to read config file '{file_path}': not found.")
+
+    @staticmethod
+    def _read_env_var(var_name):
+        val = os.getenv(var_name)
+        if val is None:
+            log.abort_and_exit("CTXT", f"Context init failed, required env var '{var_name}' is not set.")
+        return val
+
+    @staticmethod
+    def create():
+        """
+        Read environment variables and create a new Context.
+        :return: newly created Context object
+        """
+        data_dir_path = Path(Context._read_env_var(consts.ENV_DATA_DIR))
+        github_pat = Context._read_env_var(consts.ENV_GITHUB_PAT)
+        return Context(data_dir_path, github_pat)
+
+    def get_github_token(self):
+        """
+        Get the GitHub API token.
+        :return: GitHub API token.
+        """
+        return self._github_token
+
+    def get_data_dir_path(self):
+        """
+        Get the pathlib.Path to the data directory, where data file outputs should go.
+        :return: Path to the data directory
+        """
+        return self._data_dir_path

github/github_api.py → .github/actions/data-update/src/github_api.py

@@ -3,6 +3,7 @@
 import requests
 import re
 
+import consts
 import json_reducer
 import log
 import util
@@ -64,7 +65,7 @@ def _handle_rate_limit(self):
         """
         if self.is_rate_limit_status_stale():
             self.update_rate_limit_status()
-        sleep_duration = self._rate_limit_status["reset_in_sec"] + self._context.get_config("rate_limit_buffer_sec")
+        sleep_duration = self._rate_limit_status["reset_in_sec"] + consts.RATE_LIMIT_BUFFER_SEC
         time.sleep(sleep_duration)
         wakeup_time = util.epoch_to_local_datetime(self._rate_limit_status["reset_at_utc"])
         log.warning("GHUB", f"Rate limit reached - sleeping for {sleep_duration}s until {wakeup_time}.")
@@ -165,7 +166,7 @@ def is_rate_limit_status_stale(self):
         """
         if self._rate_limit_status is None:
             self.update_rate_limit_status()
-        max_age_sec = self._context.get_config("rate_limit_max_age_sec")
+        max_age_sec = consts.RATE_LIMIT_MAX_AGE_SEC
         return (round(time.time()) - self._rate_limit_status["last_update"]) > max_age_sec
 
     def request_rate_limit_status(self, force_update=False, ignore_stale=False):
@@ -246,11 +247,11 @@ def get(self, url, authenticate=True, headers=None, query_params=None, expected_
                 url = f"{url}{key}={value}&"
 
         # If max number of retries is exceeded, abort.
-        if retry > self._context.get_config("max_retries"):
+        if retry > consts.MAX_RETRIES:
             log.abort_and_exit("GHUB", f"Request to {url} with headers {headers} failed after {retry} retries.")
 
         # Sleep before making request to ensure proper delay.
-        time.sleep(self._context.get_config("request_delay_sec"))
+        time.sleep(consts.API_REQUEST_DELAY_SEC)
 
         # Before making a request, check for rate limiting. Wait if necessary.
         if self.is_rate_limited():

github/jobs.py → .github/actions/data-update/src/jobs.py

@@ -1,4 +1,6 @@
 import json
+
+import consts
 import log
 
 
@@ -25,7 +27,7 @@ def _write_to_json_file(self, filename, data):
         :param filename: name of the file (no path), should end in .json
         :param data: JSON-serializable object
         """
-        out_dir_path = self._context.get_workdir_data_dir_path()
+        out_dir_path = self._context.get_data_dir_path()
         filepath = out_dir_path.joinpath(filename)
         try:
             with open(filepath, "w", encoding="utf-8") as outfile:
@@ -71,7 +73,7 @@ def _execute_task(self):
         Override - define tasks of this job.
         """
         repos = self._github_api.collect_org_repos()
-        self._write_to_json_file(self._context.get_config("contributions_filename"), repos)
+        self._write_to_json_file(consts.CONTRIBUTIONS_FILENAME, repos)
 
 
 class JobCollectOrgMembers(Job):
@@ -96,4 +98,4 @@ def _execute_task(self):
         Override - define tasks of this job.
         """
         members = self._github_api.collect_org_members()
-        self._write_to_json_file(self._context.get_config("people_filename"), members)
+        self._write_to_json_file(consts.PEOPLE_FILENAME, members)

github/log.py → .github/actions/data-update/src/log.py

@@ -1,34 +1,10 @@
 import traceback
 import sys
-import os
-from pathlib import Path
 
 import util
 from datetime import datetime
 
 
-_log_file = None
-
-
-def _define_log_file_path(specified_log_dir):
-    """
-    Establish the the correct path to the log file to be used. Does not create the file or check
-    for its existence.
-    :param specified_log_dir: string path to log file directory, or None
-    :return: pathlib.Path to the log file
-    """
-    if specified_log_dir is None:
-        specified_log_dir = os.getcwd()
-    directory = util.ensure_directory(Path(specified_log_dir), Path(os.getcwd()))
-    base_filename = datetime.now().strftime("%y%m%d-%H%M%S")
-    file_path = directory.joinpath(f"{base_filename}.log")
-    suffix_idx = 0
-    while file_path.exists():
-        suffix_idx += 1
-        file_path = directory.joinpath(f"{base_filename}_{suffix_idx}.log")
-    return file_path
-
-
 def _log_message(level, tag, msg, stderr=False):
     """
     Log to console and to file if one is being used.
@@ -40,38 +16,16 @@ def _log_message(level, tag, msg, stderr=False):
     assembled_msg = f"[{timestamp()}] [{level}] [{tag}] {msg}"
     file = sys.stderr if stderr else sys.stdout
     print(assembled_msg, file=file)
-    if _log_file is not None:
-        _log_file.write(assembled_msg)
-        _log_file.write("\n")
-        _log_file.flush()
 
 
 def _terminate(exit_code):
     """
     Terminate the application with the given exit code, performing all necessary pre-shutdown steps.
     :param exit_code: exit code to be used
     """
-    global _log_file
-    if _log_file is not None:
-        _log_file.close()
     exit(exit_code)
 
 
-def open_log_file(specified_log_dir):
-    """
-    Create and open a log file (all future logs will be written to this file). Do not create a file
-    if one is already in use.
-    :param specified_log_dir: string path to log file directory, or None
-    """
-    global _log_file
-    if _log_file is not None:
-        warning("LOGS", "Log file already open.")
-        return
-    file_path = _define_log_file_path(specified_log_dir)
-    _log_file = open(file_path, "w", encoding="utf-8")
-    info("LOGS", f"Logging to file '{file_path}'.")
-
-
 def timestamp():
     """
     Get formatted local date and time.

.github/actions/data-update/src/main.py

@@ -0,0 +1,43 @@
+import os
+
+import log
+from pathlib import Path
+
+import util
+import jobs
+from context import Context
+from github_api import GitHubApi
+
+
+def run_jobs(context, github_api):
+    """
+    Run predefined set of data update jobs.
+    :param context: application context
+    :param github_api: GitHub API wrapper
+    """
+    util.log_rate_limit_status("MAIN", github_api)
+    jobs.JobCollectOrgRepos.initialize(context, github_api).run()
+    # jobs.JobCollectOrgMembers.initialize(context, github_api).run()
+    util.log_rate_limit_status("MAIN", github_api)
+
+
+def main():
+    # Absolute path of this script.
+    script_path = Path(os.path.abspath(__file__))
+
+    # Set up context and API wrappers.
+    context = Context.create()
+    github_api = GitHubApi(context)
+
+    # Run jobs.
+    run_jobs(context, github_api)
+
+    # Terminate.
+    log.terminate_successfully("MAIN")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as ex:
+        log.unhandled_exception_exit("MAIN", ex)

.github/actions/data-update/src/util.py

@@ -0,0 +1,35 @@
+import time
+
+import log
+
+
+def decode_command_output_buffer(buffer):
+    return buffer.decode("utf-8").strip()
+
+
+def get_time_format_pattern():
+    """
+    Return default datetime format string.
+    :return: default datetime format string
+    """
+    return "%Y/%m/%d %H:%M:%S"
+
+
+def epoch_to_local_datetime(epoch_time):
+    """
+    Convert an UTC epoch value to a formatted date and time string in local time.
+    :param epoch_time: epoch seconds value
+    :return: formatted local date and time string
+    """
+    return time.strftime(get_time_format_pattern(), time.localtime(epoch_time))
+
+
+def log_rate_limit_status(tag, github_api):
+    """
+    Log the current rate limit status.
+    :param tag: log location identifier
+    :param github_api: GitHub API wrapper
+    """
+    rl_status = github_api.request_rate_limit_status()
+    log.info(tag,
+             f"{rl_status['remaining']} calls remaining, resets at {epoch_to_local_datetime(rl_status['reset_at_utc'])}.")