Restructure helper

This commit changes the following:
- Use class based approach for projects.
- Automatically read repos from GitHub.
- Use config files in repos.
- Allow link checking, and translation syncing
  (except for in-context).

.gitignore

@@ -1,5 +1,5 @@
-# Ignore directory of cloned repositories
-repositories/
+# Ignore directory of cloned projects
+projects/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

link_checker/__init__.py

@@ -0,0 +1,53 @@
+from linkie import Linkie
+from utils import render_text
+
+
+def check_links(project):
+    """ """
+    linkie_config = project.config.get("broken-link-checker", dict())
+    checker = Linkie(config=linkie_config)
+    result = checker.run()
+    print()
+
+    broken_links = dict()
+    for url, url_data in checker.urls.items():
+        if url_data['broken']:
+            broken_links[url] = url_data
+
+    context = {
+        "broken_links": broken_links,
+        "number_files": checker.file_count,
+    }
+
+    header_text = render_text("link_checker/templates/issue-broken-links-header.txt", context)
+    body_text = render_text("link_checker/templates/issue-broken-links-body.txt", context)
+
+    bot_issues = project.repo.get_issues(creator=project.bot)
+    existing_issue = None
+    for issue in bot_issues:
+        if "broken link" in issue.title:
+            existing_issue = issue
+
+    # If existing issue and no errors, close issue
+    if existing_issue and not result:
+        message = "Closing existing issue, as link checker now detects no broken links."
+        print(message)
+        existing_issue.create_comment(message)
+        existing_issue.edit(state="closed")
+    # Else if existing issue and errors
+    elif existing_issue and result:
+        print("Checking if existing issue matches result.")
+        if header_text == existing_issue.title and body_text == existing_issue.body:
+            print("Existing issue is up to date.")
+        else:
+            message = "Updating issue to match latest broken link checker results."
+            print(message)
+            existing_issue.edit(title=header_text, body=body_text)
+            existing_issue.create_comment(message)
+    # Else if no existing issue and errors, create issue
+    elif not existing_issue and result:
+        project.repo.create_issue(
+            title=header_text,
+            body=body_text,
+            labels=[project.repo.get_label("bug")],
+        )

link_checker/templates/issue-broken-links-body.txt

@@ -0,0 +1,4 @@
+[Linkie](https://github.com/uccser/linkie) was run over this repository, and checked {{ number_files }} file{% if number_files > 1 %}s{% endif %} and detected {{ broken_links|length }} broken link{% if broken_links|length > 1 %}s{% endif %}.
+{% for broken_link, broken_link_data in broken_links.items()|sort %}
+{{ loop.index }}) {{ broken_link }} (error: {{ broken_link_data.status }})
+{%- endfor %}

link_checker/templates/issue-broken-links-header.txt

@@ -0,0 +1 @@
+Fix {{ broken_links|length }} broken link{% if broken_links|length > 1 %}s{% endif %}

requirements.txt

@@ -1 +1,13 @@
+# General
+PyGithub==1.40
+Jinja2==2.10
+PyYAML==4.2b4
+
+# Translations
 requests==2.19.1
+
+# Link Checker
+linkie==1.2.0
+
+# Testing
+flake8==3.5.0

run.py

@@ -0,0 +1,131 @@
+import os
+import base64
+import yaml
+from shutil import rmtree
+from utils import (
+    run_shell,
+    check_envs,
+    display_elapsed_time,
+    get_crowdin_api_key,
+)
+from translation import (
+    update_source_message_file,
+    push_source_files,
+    pull_translations,
+)
+from link_checker import check_links
+import argparse
+import github
+from timeit import default_timer as timer
+
+
+DEFAULT_WORKING_DIRECTORY = os.getcwd()
+PROJECT_DIRECTORY = "projects"
+GITHUB_BOT_EMAIL = "33709036+uccser-bot@users.noreply.github.com"
+GITHUB_BOT_NAME = "UCCSER Bot"
+REQUIRED_ENVIRONMENT_VARIABLES = [
+    ["GITHUB_TOKEN", "OAuth token to use for GitHub API requests"],
+]
+PROJECT_CONFIG_FILE = ".arnold.yaml"
+SEPERATOR_WIDTH = 60
+MAJOR_SEPERATOR = "=" * SEPERATOR_WIDTH
+MINOR_SEPERATOR = "-" * SEPERATOR_WIDTH
+
+
+def setup_git_account():
+    """Set the name and email account of the git account."""
+    run_shell(["git", "config", "user.name", GITHUB_BOT_NAME])
+    run_shell(["git", "config", "user.email", GITHUB_BOT_EMAIL])
+
+
+class Project:
+
+    def __init__(self, config, repo, bot, parent_directory):
+        self.config = config
+        self.repo = repo
+        self.name = repo.name
+        self.bot = bot
+        self.parent_directory = parent_directory
+        self.directory = os.path.join(parent_directory, self.name)
+        self.start_time = timer()
+
+    def display_elapsed_time(self):
+        display_elapsed_time(self.start_time)
+        self.start_time = timer()
+
+    def clone(self):
+        """Clone the repository, deleting any existing installations."""
+        if os.path.isdir(self.directory):
+            print("Existing repository detected! Deleting existing directory...")
+            rmtree(self.repo.name)
+        run_shell(["git", "clone", self.repo.git_url])
+
+    def run(self):
+        if self.config.get("broken-link-checker"):
+            check_links(self)
+            self.display_elapsed_time()
+
+        if self.config.get("translation"):
+            self.crowdin_api_key = get_crowdin_api_key(self.name)
+            update_source_message_file(self)
+            self.display_elapsed_time()
+            push_source_files(self)
+            self.display_elapsed_time()
+            pull_translations(self)
+            self.display_elapsed_time()
+
+
+def main():
+    start_time = timer()
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c",
+        "--skip-clone",
+        help="Skip cloning repositories",
+        action="store_true"
+    )
+    args = parser.parse_args()
+    if args.skip_clone:
+        print("Skip cloning repositories turned on.\n")
+
+    check_envs(REQUIRED_ENVIRONMENT_VARIABLES)
+
+    if not os.path.exists(PROJECT_DIRECTORY):
+        os.makedirs(PROJECT_DIRECTORY)
+    os.chdir(PROJECT_DIRECTORY)
+    directory_of_projects = os.path.abspath(os.getcwd())
+
+    github_env = github.Github(os.environ["GITHUB_TOKEN"])
+    uccser = github_env.get_user("uccser")
+    bot = github_env.get_user("uccser-bot")
+    uccser_repos = uccser.get_repos()
+    for repo in uccser_repos:
+        os.chdir(directory_of_projects)
+        print("{0}\n{1}\n{2}".format(MAJOR_SEPERATOR, repo.full_name, MINOR_SEPERATOR))
+        try:
+            config_file = repo.get_contents(PROJECT_CONFIG_FILE)
+            print("Config file for Arnold detected.")
+        except github.GithubException:
+            config_file = None
+            print("Config file for Arnold not detected.")
+        if config_file:
+            print("Reading Arnold config.")
+            try:
+                config = yaml.load(base64.b64decode(config_file.content).decode("utf-8"))
+            except yaml.YAMLError:
+                print("Error! YAML file invalid.")
+                # TODO: Log issue on repo
+                # TODO: Notify organisation by email or on Slack
+            if config:
+                project = Project(config, repo, bot, directory_of_projects)
+                if not args.skip_clone:
+                    project.clone()
+                os.chdir(project.directory)
+                setup_git_account()
+                project.run()
+        print("{0}\n".format(MAJOR_SEPERATOR))
+    display_elapsed_time(start_time)
+
+
+if __name__ == "__main__":
+    main()

setup.cfg

@@ -0,0 +1,13 @@
+[flake8]
+exclude =
+    # No need to traverse our git or project directories
+    .git,
+    projects,
+
+    # There's no value in checking cache directories
+    __pycache__,
+
+show-source = True
+statistics = True
+count = True
+max-line-length = 119