From 85d4170ae351d8ff550a13aaf2cba4710e883475 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Thu, 21 Nov 2019 13:08:57 +0100 Subject: [PATCH 1/3] Use TaskclusterConfig from official library --- libmozevent/__init__.py | 2 +- libmozevent/taskcluster.py | 131 ------------------------------------- 2 files changed, 1 insertion(+), 132 deletions(-) delete mode 100644 libmozevent/taskcluster.py diff --git a/libmozevent/__init__.py b/libmozevent/__init__.py index 4de9bd5..c0a9388 100644 --- a/libmozevent/__init__.py +++ b/libmozevent/__init__.py @@ -3,6 +3,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from libmozevent.taskcluster import TaskclusterConfig +from taskcluster.helper import TaskclusterConfig taskcluster_config = TaskclusterConfig() diff --git a/libmozevent/taskcluster.py b/libmozevent/taskcluster.py deleted file mode 100644 index 57e4f44..0000000 --- a/libmozevent/taskcluster.py +++ /dev/null @@ -1,131 +0,0 @@ -# -*- coding: utf-8 -*- -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import copy -import os - -import structlog -import taskcluster - -try: - import toml -except ImportError: - import pytoml as toml - -logger = structlog.get_logger(__name__) - - -class TaskclusterConfig(object): - """ - Local configuration used to access Taskcluster service and objects - """ - - def __init__(self, url="https://community-tc.services.mozilla.com"): - self.options = None - self.secrets = None - self.default_url = os.environ.get("TASKCLUSTER_ROOT_URL", url) - - def auth(self, client_id=None, access_token=None): - """ - Build Taskcluster credentials options - Supports, by order of preference: - * directly provided credentials - * credentials from local configuration - * credentials from environment variables - * taskclusterProxy - """ - self.options = {"maxRetries": 12} - - if client_id is None and access_token is None: - # Credentials preference: Use local config from release-services - xdg = os.path.expanduser(os.environ.get("XDG_CONFIG_HOME", "~/.config")) - config = os.path.join(xdg, "please", "config.toml") - try: - assert os.path.exists(config), "No user config available" - data = toml.load(open(config)) - client_id = data["common"]["taskcluster_client_id"] - access_token = data["common"]["taskcluster_access_token"] - assert ( - client_id is not None and access_token is not None - ), "Missing values in user folder" - logger.info("Using taskcluster credentials from local configuration") - except Exception: - # Credentials preference: Use env. variables - client_id = os.environ.get("TASKCLUSTER_CLIENT_ID") - access_token = os.environ.get("TASKCLUSTER_ACCESS_TOKEN") - logger.info("Using taskcluster credentials from environment") - else: - logger.info("Using taskcluster credentials from cli") - - if client_id is not None and access_token is not None: - # Use provided credentials - self.options["credentials"] = { - "clientId": client_id, - "accessToken": access_token, - } - self.options["rootUrl"] = self.default_url - - elif "TASK_ID" in os.environ: - # Load secrets from TC task context - # with taskclusterProxy - # Only works when running in a Taskcluster Task - logger.info("Taskcluster Proxy enabled") - self.options["rootUrl"] = "http://taskcluster" - - else: - logger.info("No Taskcluster authentication.") - self.options["rootUrl"] = self.default_url - - def get_service(self, service_name): - """ - Build a Taskcluster service instance using current authentication - """ - assert self.options is not None, "Not authenticated" - service = getattr(taskcluster, service_name.capitalize(), None) - assert service is not None, "Invalid Taskcluster service {}".format( - service_name - ) - return service(self.options) - - def load_secrets( - self, name, project_name, required=[], existing=dict(), local_secrets=None - ): - """ - Fetch a specific set of secrets by name and verify that the required - secrets exist. - Also supports a local secrets payload (as a dict) to avoid using Taskcluster service - - Merge secrets in the following order (the latter overrides the former): - - `existing` argument - - common secrets, specified under the `common` key in the secrets - object - - project specific secrets, specified under the `project_name` key in - the secrets object - """ - self.secrets = dict() - if existing: - self.secrets = copy.deepcopy(existing) - - if local_secrets is None: - # Use Taskcluster secret service - assert name is not None, "Missing Taskcluster secret name" - secrets_service = self.get_service("secrets") - all_secrets = secrets_service.get(name).get("secret", dict()) - logger.info("Loaded Taskcluster secret", name=name) - else: - # Use local YAML file to avoid using Taskcluster secrets - assert isinstance(local_secrets, dict), "Local secrets must be a dict" - logger.info("Using provided local secrets.") - all_secrets = local_secrets - - secrets_common = all_secrets.get("common", dict()) - self.secrets.update(secrets_common) - - secrets_app = all_secrets.get(project_name, dict()) - self.secrets.update(secrets_app) - - for required_secret in required: - if required_secret not in self.secrets: - raise Exception(f"Missing value {required_secret} in secrets.") From c97b393c37d4a9d943d84933a260d72010f14cf3 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Tue, 3 Dec 2019 10:19:28 +0100 Subject: [PATCH 2/3] monitoring: Do not use local taskcluster config --- libmozevent/__init__.py | 8 -------- libmozevent/monitoring.py | 14 ++++++++------ requirements.txt | 2 +- tests/conftest.py | 12 +++++------- tests/test_monitoring.py | 10 +++++----- 5 files changed, 19 insertions(+), 27 deletions(-) diff --git a/libmozevent/__init__.py b/libmozevent/__init__.py index c0a9388..e69de29 100644 --- a/libmozevent/__init__.py +++ b/libmozevent/__init__.py @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from taskcluster.helper import TaskclusterConfig - -taskcluster_config = TaskclusterConfig() diff --git a/libmozevent/monitoring.py b/libmozevent/monitoring.py index 766b024..af546a5 100644 --- a/libmozevent/monitoring.py +++ b/libmozevent/monitoring.py @@ -3,10 +3,9 @@ from datetime import datetime, timedelta import structlog +from taskcluster.helper import TaskclusterConfig from taskcluster.utils import slugId, stringDate -from libmozevent import taskcluster_config - logger = structlog.get_logger(__name__) GROUP_MD = """ @@ -26,11 +25,14 @@ class Monitoring(object): every X seconds """ - def __init__(self, queue_name, emails, period): - assert isinstance(queue_name, str) - assert isinstance(period, int) + def __init__( + self, + taskcluster_config: TaskclusterConfig, + queue_name: str, + emails: list, + period: int, + ): assert period > 0 - assert isinstance(emails, list) assert len(emails) > 0 self.queue_name = queue_name self.period = period diff --git a/requirements.txt b/requirements.txt index 4a0fee3..f169fd1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ Logbook python-hglib pytoml structlog -taskcluster +taskcluster==24.0.0 diff --git a/tests/conftest.py b/tests/conftest.py index 89d5cfa..5c189b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,9 +15,9 @@ import hglib import pytest import responses +from taskcluster.helper import TaskclusterConfig from taskcluster.utils import stringDate -from libmozevent import taskcluster_config from libmozevent.mercurial import Repository from libmozevent.phabricator import PhabricatorActions @@ -367,10 +367,8 @@ def mock_taskcluster(): """ Mock Tasklcuster authentication """ - taskcluster_config.options = {"rootUrl": "http://taskcluster.test"} + tc = TaskclusterConfig("http://taskcluster.test") + tc.auth() + tc.options["maxRetries"] = 1 - responses.add( - responses.GET, - "https://queue.taskcluster.net/v1/task-group/aGroup/list", - json={"taskGroupId": "aGroup", "tasks": []}, - ) + return tc diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py index 43dba71..418c40f 100644 --- a/tests/test_monitoring.py +++ b/tests/test_monitoring.py @@ -8,7 +8,7 @@ @pytest.mark.asyncio async def test_monitoring(QueueMock, NotifyMock, mock_taskcluster): bus = MessageBus() - monitoring = Monitoring("testqueue", ["pinco@pallino"], 1) + monitoring = Monitoring(mock_taskcluster, "testqueue", ["pinco@pallino"], 1) monitoring.register(bus) await bus.send("testqueue", ("Group1", "Hook1", "Task-invalid")) await bus.send("testqueue", ("Group1", "Hook1", "Task-pending")) @@ -123,7 +123,7 @@ async def test_monitoring(QueueMock, NotifyMock, mock_taskcluster): @pytest.mark.asyncio async def test_report_all_completed(QueueMock, NotifyMock, mock_taskcluster): bus = MessageBus() - monitoring = Monitoring("testqueue", ["pinco@pallino"], 1) + monitoring = Monitoring(mock_taskcluster, "testqueue", ["pinco@pallino"], 1) monitoring.register(bus) await bus.send("testqueue", ("Group1", "Hook1", "Task1-completed")) await bus.send("testqueue", ("Group1", "Hook1", "Task2-completed")) @@ -146,7 +146,7 @@ async def test_monitoring_whiteline_between_failed_and_hook( QueueMock, NotifyMock, mock_taskcluster ): bus = MessageBus() - monitoring = Monitoring("testqueue", ["pinco@pallino"], 1) + monitoring = Monitoring(mock_taskcluster, "testqueue", ["pinco@pallino"], 1) monitoring.register(bus) await bus.send("testqueue", ("Group1", "Hook1", "Task-failed")) await bus.send("testqueue", ("Group1", "Hook2", "Task-failed")) @@ -218,7 +218,7 @@ async def test_monitoring_whiteline_between_failed_and_hook( @pytest.mark.asyncio async def test_monitoring_retry_exceptions(QueueMock, NotifyMock, mock_taskcluster): bus = MessageBus() - monitoring = Monitoring("testqueue", ["pinco@pallino"], 1) + monitoring = Monitoring(mock_taskcluster, "testqueue", ["pinco@pallino"], 1) monitoring.register(bus) await bus.send("testqueue", ("Group1", "Hook1", "Task-exception-retry:2")) await bus.send("testqueue", ("Group1", "Hook2", "Task-exception-retry:0")) @@ -254,7 +254,7 @@ async def test_monitoring_retry_exceptions(QueueMock, NotifyMock, mock_taskclust @pytest.mark.asyncio async def test_monitoring_restartable(QueueMock, IndexMock, mock_taskcluster): bus = MessageBus() - monitoring = Monitoring("testqueue", ["pinco@pallino"], 1) + monitoring = Monitoring(mock_taskcluster, "testqueue", ["pinco@pallino"], 1) monitoring.register(bus) monitoring.index = IndexMock From 3c9851cb6c05f76716192b0efbf8375a23e128ad Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Tue, 3 Dec 2019 10:27:44 +0100 Subject: [PATCH 3/3] Set min version for Taskcluster lib --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f169fd1..e8c1218 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ Logbook python-hglib pytoml structlog -taskcluster==24.0.0 +taskcluster>=24.0.0