From b169dc300f738c59e717d444445dfe0c8e6da98a Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 10 Jul 2025 13:38:48 +0100 Subject: [PATCH 1/7] Remove GitHub-Grafana team sync cog --- arthur/apis/github/teams.py | 25 ---- arthur/exts/grafana/github_team_sync.py | 172 ------------------------ 2 files changed, 197 deletions(-) delete mode 100644 arthur/exts/grafana/github_team_sync.py diff --git a/arthur/apis/github/teams.py b/arthur/apis/github/teams.py index 4cf18d4..a8775a3 100644 --- a/arthur/apis/github/teams.py +++ b/arthur/apis/github/teams.py @@ -1,32 +1,7 @@ -import aiohttp - from arthur.config import CONFIG -from arthur.log import logger HEADERS = { "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", "Authorization": f"Bearer {CONFIG.github_token.get_secret_value()}", } -BASE_URL = "https://api.github.com" -MEMBERS_PER_PAGE = 100 - - -class GithubTeamNotFoundError(aiohttp.ClientResponseError): - """Raised when a github team could not be found.""" - - -async def list_team_members(team_slug: str, session: aiohttp.ClientSession) -> list[dict[str, str]]: - """List all Github teams.""" - endpoint = f"{BASE_URL}/orgs/{CONFIG.github_org}/teams/{team_slug}/members" - params = {"per_page": MEMBERS_PER_PAGE} - async with session.get(endpoint, headers=HEADERS, params=params) as response: - response.raise_for_status() - teams_resp = await response.json() - if len(teams_resp) == MEMBERS_PER_PAGE: - logger.warning( - "Max number (%d) of members returned when fetching members of %s. Some members may have been missed.", - MEMBERS_PER_PAGE, - team_slug, - ) - return teams_resp diff --git a/arthur/exts/grafana/github_team_sync.py b/arthur/exts/grafana/github_team_sync.py deleted file mode 100644 index cfc1e77..0000000 --- a/arthur/exts/grafana/github_team_sync.py +++ /dev/null @@ -1,172 +0,0 @@ -import aiohttp -import discord -from discord.ext import commands, tasks - -from arthur.apis import github, grafana -from arthur.bot import KingArthur -from arthur.config import CONFIG -from arthur.log import logger - -from . import MissingMembers, SyncFigures - - -class GrafanaGitHubTeamSync(commands.Cog): - """ - Update Grafana team membership to match Github team membership. - - Grafana team name must match Github team slug exactly. - Use `gh api orgs/{org-name}/teams` to get a list of teams in an org - """ - - def __init__(self, bot: KingArthur) -> None: - self.bot = bot - self.sync_github_grafana_teams.start() - - async def _add_missing_members( - self, - grafana_team_id: int, - github_team_members: set[str], - grafana_team_members: set[str], - all_grafana_users: list[dict], - ) -> MissingMembers: - """ - Adds members to the Grafana team if they're in the Github team and not already present. - - Returns the number of missing members, and the number of members it could actually add. - """ - missing_members = github_team_members - grafana_team_members - added_members = 0 - for grafana_user in all_grafana_users: - if grafana_user["login"] not in missing_members: - continue - if "GitHub" not in grafana_user.get("authLabels", []): - continue - - await grafana.add_user_to_team( - grafana_user["userId"], - grafana_team_id, - self.bot.http_session, - ) - added_members += 1 - return MissingMembers(count=len(missing_members), successfully_added=added_members) - - async def _remove_extra_members( - self, - grafana_team_id: int, - github_team_members: set[str], - grafana_team_members: set[str], - all_grafana_users: list[dict], - ) -> int: - """ - Removes Grafana users from a team if they are not present in the Github team. - - Return how many were removed. - """ - extra_members = grafana_team_members - github_team_members - removed_members = 0 - for grafana_user in all_grafana_users: - if grafana_user["login"] not in extra_members: - continue - await grafana.remove_user_from_team( - grafana_user["userId"], - grafana_team_id, - self.bot.http_session, - ) - removed_members += 1 - return removed_members - - async def _sync_teams(self, team: dict[str, str]) -> SyncFigures: - """ - Ensure members in Github are present in Grafana teams. - - Return the number of members missing from the Grafana team, and the number of members added. - """ - github_team_members = { - member["login"] - for member in await github.list_team_members(team["name"], self.bot.http_session) - } - grafana_team_members = { - member["login"] - for member in await grafana.list_team_members(team["id"], self.bot.http_session) - if member.get("auth_module") == "oauth_github" - } - - all_grafana_users = await grafana.get_all_users(self.bot.http_session) - added_members = await self._add_missing_members( - team["id"], - github_team_members, - grafana_team_members, - all_grafana_users, - ) - removed_members = await self._remove_extra_members( - team["id"], - github_team_members, - grafana_team_members, - all_grafana_users, - ) - - return SyncFigures(added=added_members, removed=removed_members) - - @tasks.loop(hours=12) - async def sync_github_grafana_teams(self, channel: discord.TextChannel | None = None) -> None: - """Update Grafana team membership to match Github team membership.""" - grafana_teams = await grafana.list_teams(self.bot.http_session) - embed = discord.Embed( - title="Sync Stats", - colour=discord.Colour.blue(), - ) - for team in grafana_teams: - logger.debug(f"Processing {team['name']}") - try: - figures = await self._sync_teams(team) - except aiohttp.ClientResponseError as e: - logger.error(e) - if channel: - await channel.send(e) - continue - - lines = [ - f"Missing: {figures.added.count}", - f"Added: {figures.added.successfully_added}", - f"Removed: {figures.removed}", - ] - embed.add_field( - name=team["name"], - value="\n".join(lines), - inline=False, - ) - - if channel: - await channel.send(embed=embed) - - @sync_github_grafana_teams.error - async def on_task_error(self, error: Exception) -> None: - """Ensure task errors are output.""" - logger.error(error) - - @commands.group(name="grafana_github", invoke_without_command=True) - async def grafana_group(self, ctx: commands.Context) -> None: - """Commands for working with grafana API.""" - await ctx.send_help(ctx.command) - - @grafana_group.command(name="sync") - async def sync_teams(self, ctx: commands.Context) -> None: - """Sync Grafana & Github teams now.""" - await self.sync_github_grafana_teams(ctx.channel) - - -async def setup(bot: KingArthur) -> None: - """Add GrafanaGitHubTeamSync cog to bot.""" - if not all( - ( - CONFIG.github_org, - CONFIG.github_token, - CONFIG.grafana_url, - CONFIG.grafana_token, - ) - ): - logger.warning( - "Not loading GrafanaGitHubTeamSync team as a required config entry is missing. See README" - ) - return - await bot.add_cog(GrafanaGitHubTeamSync(bot)) From 947b45d6dc353a0220a271119a857d24adb38b9f Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 10 Jul 2025 13:38:57 +0100 Subject: [PATCH 2/7] Update envvar documentation to reflect removed GitHub-Grafana team sync --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d04e439..85926c3 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@ These environment variables are required to work on the relevant cog. | Environment | Relevant cog | Description | Required/Default | | ------------------------------------- | --------------------- | ------------------------------------------------------------------------- | ------------------------- | | KING_ARTHUR_CLOUDFLARE_TOKEN | Zones | A token for the Cloudflare API used for the Cloudflare commands in Arthur | Required | -| KING_ARTHUR_GITHUB_ORG | GrafanaGitHubTeamSync | The github organisation to fetch teams from | python-discord | -| KING_ARTHUR_GITHUB_TOKEN | GrafanaGitHubTeamSync | The github token used to fetch teams to populate grafana | Required | -| KING_ARTHUR_GRAFANA_URL | GrafanaGitHubTeamSync | The URL to the grafana instance to manage teams | https://grafana.pydis.wtf | -| KING_ARTHUR_GRAFANA_TOKEN | GrafanaGitHubTeamSync | The grafana token used to sync teams with github | Required | +| KING_ARTHUR_GITHUB_ORG | GitHubManagement | The github organisation to fetch teams from | python-discord | +| KING_ARTHUR_GITHUB_TOKEN | GitHubManagement | The github token used to manage the GitHub organisation | Required | +| KING_ARTHUR_GRAFANA_URL | GrafanaLDAPTeamSync | The URL to the grafana instance to manage teams | https://grafana.pydis.wtf | +| KING_ARTHUR_GRAFANA_TOKEN | GrafanaLDAPTeamSync | The grafana token used to sync teams with LDAP | Required | | KING_ARTHUR_YOUTUBE_API_KEY | Motivation | The YouTube API key to fetch missions with | Required | ### LDAP & Directory integrations From 955c19ae3ac9504a246b9647271fa84102c7a477 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 10 Jul 2025 14:14:37 +0100 Subject: [PATCH 3/7] Add new constants --- README.md | 1 + arthur/config.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 85926c3..12de4ef 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ These environment variables are required to work on the relevant cog. | KING_ARTHUR_CLOUDFLARE_TOKEN | Zones | A token for the Cloudflare API used for the Cloudflare commands in Arthur | Required | | KING_ARTHUR_GITHUB_ORG | GitHubManagement | The github organisation to fetch teams from | python-discord | | KING_ARTHUR_GITHUB_TOKEN | GitHubManagement | The github token used to manage the GitHub organisation | Required | +| KING_ARTHUR_GITHUB_TEAM | GitHubManagement | The slug of the GitHub team to add new members to | staff | | KING_ARTHUR_GRAFANA_URL | GrafanaLDAPTeamSync | The URL to the grafana instance to manage teams | https://grafana.pydis.wtf | | KING_ARTHUR_GRAFANA_TOKEN | GrafanaLDAPTeamSync | The grafana token used to sync teams with LDAP | Required | | KING_ARTHUR_YOUTUBE_API_KEY | Motivation | The YouTube API key to fetch missions with | Required | diff --git a/arthur/config.py b/arthur/config.py index 7603cf9..2733ecf 100644 --- a/arthur/config.py +++ b/arthur/config.py @@ -23,9 +23,11 @@ class Config( grafana_token: pydantic.SecretStr | None = None github_token: pydantic.SecretStr | None = None github_org: str = "python-discord" + github_team: str = "staff" devops_role: int = 409416496733880320 helpers_role: int = 267630620367257601 + admins_role: int = 267628507062992896 guild_id: int = 267624335836053506 devops_channel_id: int = 675756741417369640 devops_vc_id: int = 881573757536329758 From 789140344212405e77370de435d8ed03be4efd77 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 10 Jul 2025 14:15:39 +0100 Subject: [PATCH 4/7] Add new APIs for adding team members --- arthur/apis/github/teams.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/arthur/apis/github/teams.py b/arthur/apis/github/teams.py index a8775a3..489da90 100644 --- a/arthur/apis/github/teams.py +++ b/arthur/apis/github/teams.py @@ -1,3 +1,5 @@ +import aiohttp + from arthur.config import CONFIG HEADERS = { @@ -5,3 +7,37 @@ "X-GitHub-Api-Version": "2022-11-28", "Authorization": f"Bearer {CONFIG.github_token.get_secret_value()}", } + +HTTP_404 = 404 +HTTP_403 = 403 +HTTP_422 = 422 + + +class GitHubError(Exception): + """Custom exception for GitHub API errors.""" + + def __init__(self, message: str): + super().__init__(message) + + +async def add_staff_member(username: str) -> None: + """Add a user to the default GitHub team.""" + async with aiohttp.ClientSession() as session: + endpoint = f"https://api.github.com/orgs/{CONFIG.github_org}/teams/{CONFIG.github_team}/memberships/{username}" + async with session.put(endpoint, headers=HEADERS) as response: + try: + response.raise_for_status() + return await response.json() + except aiohttp.ClientResponseError as e: + if e.status == HTTP_404: + msg = f"Team or user not found: {e.message}" + raise GitHubError(msg) + if e.status == HTTP_403: + msg = f"Forbidden: {e.message}" + raise GitHubError(msg) + if e.status == HTTP_422: + msg = "Cannot add organisation as a team member" + raise GitHubError(msg) + + msg = f"Unexpected error: {e.message}" + raise GitHubError(msg) From 2d721775ace9e2ce4c02c1e5c3b922108e6af4f6 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 10 Jul 2025 14:16:05 +0100 Subject: [PATCH 5/7] Add new GitHub extension --- arthur/exts/github/__init__.py | 1 + arthur/exts/github/management.py | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 arthur/exts/github/__init__.py create mode 100644 arthur/exts/github/management.py diff --git a/arthur/exts/github/__init__.py b/arthur/exts/github/__init__.py new file mode 100644 index 0000000..78cc904 --- /dev/null +++ b/arthur/exts/github/__init__.py @@ -0,0 +1 @@ +"""Utilities for managing the GitHub organisation.""" diff --git a/arthur/exts/github/management.py b/arthur/exts/github/management.py new file mode 100644 index 0000000..202dd5b --- /dev/null +++ b/arthur/exts/github/management.py @@ -0,0 +1,42 @@ +"""Commands for managing the GitHub organisation and teams.""" + +from discord.ext.commands import Cog, Context, group + +from arthur.apis.github import GitHubException, add_staff_member +from arthur.bot import KingArthur +from arthur.config import CONFIG + + +class GitHubManagement(Cog): + """Ed is the standard text editor.""" + + def __init__(self, bot: KingArthur) -> None: + self.bot = bot + + async def cog_check(self, ctx: Context) -> bool: + """Check if the user has permission to use this cog.""" + return ( + CONFIG.admins_role in [r.id for r in ctx.author.roles] + or CONFIG.devops_role in [r.id for r in ctx.author.roles] + or await self.bot.is_owner(ctx.author) + ) + + @group(name="github", invoke_without_command=True) + async def github(self, ctx: Context) -> None: + """Group of commands for managing the GitHub organisation.""" + if ctx.invoked_subcommand is None: + await ctx.send_help(ctx.command) + + @github.command(name="add") + async def add_team_member(self, ctx: Context, username: str) -> None: + """Add a user to the default GitHub team.""" + try: + await add_staff_member(username) + await ctx.send(f":white_check_mark: Successfully invited {username} to the staff team.") + except GitHubException as e: + await ctx.send(f":x: Failed to add {username} to the staff team: {e}") + + +async def setup(bot: KingArthur) -> None: + """Add cog to bot.""" + await bot.add_cog(GitHubManagement(bot)) From 2ed8805539da3d91935b4685955d4eb989afaff8 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 10 Jul 2025 14:16:32 +0100 Subject: [PATCH 6/7] Add exemption for new cog to _is_devops --- arthur/bot.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arthur/bot.py b/arthur/bot.py index 86db7f2..c9b572b 100644 --- a/arthur/bot.py +++ b/arthur/bot.py @@ -24,7 +24,7 @@ def __init__(self, *args: list[Any], **kwargs: dict[str, Any]) -> None: super().__init__(*args, **kwargs) self.add_check(self._is_devops) - async def _is_devops(self, ctx: commands.Context | Interaction) -> bool: + async def _is_devops(self, ctx: commands.Context | Interaction) -> bool: # noqa: PLR0911 """Check all commands are executed by authorised personnel.""" u = ctx.user if isinstance(ctx, Interaction) else ctx.author if await arthur.instance.is_owner(u): @@ -38,6 +38,10 @@ async def _is_devops(self, ctx: commands.Context | Interaction) -> bool: if ctx.command.name in {"ed", "rules", "monitor"}: return True + if ctx.command.cog_name == "GitHubManagement": + # Commands in this cog have explicit additional checks. + return True + if not ctx.guild: return False From a78ef1fdaed9a5720db98bb1dcc2ff53bbe73448 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 10 Jul 2025 14:16:44 +0100 Subject: [PATCH 7/7] Update exports for GitHub APIs --- arthur/apis/github/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arthur/apis/github/__init__.py b/arthur/apis/github/__init__.py index 77988a1..11261da 100644 --- a/arthur/apis/github/__init__.py +++ b/arthur/apis/github/__init__.py @@ -1,6 +1,3 @@ -from .teams import GithubTeamNotFoundError, list_team_members +from .teams import GitHubError, add_staff_member -__all__ = ( - "GithubTeamNotFoundError", - "list_team_members", -) +__all__ = ("GitHubError", "add_staff_member")