diff --git a/.isort.cfg b/.isort.cfg index 9b133c289..9658fe65e 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,6 @@ [settings] known_first_party = code_review_backend,code_review_bot,code_review_tools,code_review_events,conftest -known_third_party = dj_database_url,django,influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,raven,requests,responses,rest_framework,sentry_sdk,setuptools,structlog,taskcluster,toml +known_third_party = dj_database_url,django,influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,raven,requests,responses,rest_framework,sentry_sdk,setuptools,structlog,taskcluster,toml,yaml force_single_line = True default_section=FIRSTPARTY line_length=159 diff --git a/bot/README.md b/bot/README.md index 9f7bb58df..0ae398812 100644 --- a/bot/README.md +++ b/bot/README.md @@ -19,18 +19,18 @@ flake8 pytest ``` -If those tests are OK, you can run the bot locally, by specifying a Taskcluster secret with your configuration, and a task reference to analyze. +If those tests are OK, you can run the bot locally, by using a local configuration file with your Phabricator API token (see details at the end of this README), and a task reference to analyze. ``` export TRY_TASK_ID=XXX export TRY_TASK_GROUP_ID=XXX -code-review-bot --taskcluster-secret=path/to/secret +code-review-bot --configuration=path/to/config.yaml ``` Configuration ------------- -The code review bot is configured through the [Taskcluster secrets service](https://firefox-ci-tc.services.mozilla.com/secrets) +The code review bot is configured through the [Taskcluster secrets service](https://firefox-ci-tc.services.mozilla.com/secrets) or a local YAML configuration file (the latter is preferred for new contributors as it's easier to setup) The following configuration variables are currently supported: @@ -76,38 +76,22 @@ Key `reporter` is `phabricator` Configuration: - * `analyzers` : The analyzers that will be published on Phabricator. Possible values are: mozlint, clang-tidy, clang-format, coverity, infer, coverage. + * `analyzers_skipped` : The analyzers that will **not** be published on Phabricator. This reporter will send detailed information about every **publishable** issue. Example configuration --------------------- -```json -{ - "common": { - "APP_CHANNEL": "staging", - "PAPERTRAIL_HOST": "XXXX.papertrail.net", - "PAPERTRAIL_PORT": 12345, - "PHABRICATOR": { - "url": "https://dev.phabricator.mozilla.com", - "api_key": "deadbeef123456" - } - }, - "code-review-bot": { - "REPORTERS": [ - { - "reporter": "mail", - "emails": [ - "xxx@mozilla.com", - "yyy@mozilla.com" - ] - }, - { - "reporter": "phabricator", - "analyzers": ["clang-tidy", "mozlint"] - } - ] - } -} +```yaml +--- +common: + APP_CHANNEL: development + PHABRICATOR: + url: https://dev.phabricator.mozilla.com + api_key: deadbeef123456 + +code-review-bot: + REPORTERS: + - reporter: phabricator ``` diff --git a/bot/code_review_bot/cli.py b/bot/code_review_bot/cli.py index 4cadf03ec..eea0510f7 100644 --- a/bot/code_review_bot/cli.py +++ b/bot/code_review_bot/cli.py @@ -29,6 +29,12 @@ def parse_cli(): Setup CLI options parser """ parser = argparse.ArgumentParser(description="Mozilla Code Review Bot") + parser.add_argument( + "-c", + "--configuration", + help="Local configuration file replacing Taskcluster secrets", + type=open, + ) parser.add_argument( "--taskcluster-secret", help="Taskcluster Secret path", @@ -55,6 +61,7 @@ def main(): "ZERO_COVERAGE_ENABLED": True, "ALLOWED_PATHS": ["*"], }, + local_source=args.configuration, ) init_logger( diff --git a/tools/code_review_tools/taskcluster.py b/tools/code_review_tools/taskcluster.py index a3f06c873..9512eb2e5 100644 --- a/tools/code_review_tools/taskcluster.py +++ b/tools/code_review_tools/taskcluster.py @@ -11,6 +11,7 @@ import structlog import taskcluster import toml +import yaml logger = structlog.get_logger(__name__) @@ -36,6 +37,9 @@ def auth(self, client_id=None, access_token=None): * taskclusterProxy """ self.options = {"maxRetries": 12} + default_taskcluster_url = os.environ.get( + "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com" + ) if client_id is None and access_token is None: # Credentials preference: Use local config from release-services @@ -64,16 +68,19 @@ def auth(self, client_id=None, access_token=None): "clientId": client_id, "accessToken": access_token, } - self.options["rootUrl"] = os.environ.get( - "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com" - ) + self.options["rootUrl"] = default_taskcluster_url - else: + elif "TASK_ID" in os.environ: # Load secrets from TC task context # with taskclusterProxy + # Only works when running in a Taskcluster Task logger.info("Taskcluster Proxy enabled") self.options["rootUrl"] = "http://taskcluster" + else: + logger.info("No Taskcluster authentication.") + self.options["rootUrl"] = default_taskcluster_url + def get_service(self, service_name): """ Build a Taskcluster service instance using current authentication @@ -85,10 +92,13 @@ def get_service(self, service_name): ) return service(self.options) - def load_secrets(self, name, project_name, required=[], existing=dict()): + def load_secrets( + self, name, project_name, required=[], existing=dict(), local_source=None + ): """ Fetch a specific set of secrets by name and verify that the required secrets exist. + Also supports a local YAML file through local_source (file descriptor) Merge secrets in the following order (the latter overrides the former): - `existing` argument @@ -97,14 +107,20 @@ def load_secrets(self, name, project_name, required=[], existing=dict()): - project specific secrets, specified under the `project_name` key in the secrets object """ - assert name is not None, "Missing Taskcluster secret name" self.secrets = dict() if existing: self.secrets = copy.deepcopy(existing) - secrets_service = self.get_service("secrets") - all_secrets = secrets_service.get(name).get("secret", dict()) - logger.info("Loaded Taskcluster secret", name=name) + if local_source is None: + # Use Taskcluster secret service + assert name is not None, "Missing Taskcluster secret name" + secrets_service = self.get_service("secrets") + all_secrets = secrets_service.get(name).get("secret", dict()) + logger.info("Loaded Taskcluster secret", name=name) + else: + # Use local YAML file to avoid using Taskcluster secrets + logger.info(f"Using local secrets from {local_source.name}") + all_secrets = yaml.safe_load(local_source) secrets_common = all_secrets.get("common", dict()) self.secrets.update(secrets_common) diff --git a/tools/requirements.txt b/tools/requirements.txt index b359e8f62..1925c9394 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,4 +1,5 @@ Logbook==1.5.3 +pyyaml==5.1.2 structlog==19.2.0 taskcluster==22.1.1 toml==0.10.0