diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index d0072ff48366f..09952b00b0a44 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -492,4 +492,4 @@ jobs: run: | pip install slack_sdk - python utils/notification_service.py push + python utils/notification_service_deprecated.py push diff --git a/utils/notification_service_deprecated.py b/utils/notification_service_deprecated.py new file mode 100644 index 0000000000000..b14bff1751921 --- /dev/null +++ b/utils/notification_service_deprecated.py @@ -0,0 +1,217 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Old script for Slack's notification service. Still here as the entire suite has not been moved to the newer implem. + +import os +import re +import sys + +from slack_sdk import WebClient + + +def handle_test_results(test_results): + expressions = test_results.split(" ") + + failed = 0 + success = 0 + + # When the output is short enough, the output is surrounded by = signs: "== OUTPUT ==" + # When it is too long, those signs are not present. + time_spent = expressions[-2] if "=" in expressions[-1] else expressions[-1] + + for i, expression in enumerate(expressions): + if "failed" in expression: + failed += int(expressions[i - 1]) + if "passed" in expression: + success += int(expressions[i - 1]) + + return failed, success, time_spent + + +def format_for_slack(total_results, results, scheduled: bool, title: str): + print(total_results, results) + header = { + "type": "header", + "text": { + "type": "plain_text", + "text": title, + "emoji": True, + }, + } + + if total_results["failed"] > 0: + total = { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": f"*Failures:*\nāŒ {total_results['failed']} failures."}, + {"type": "mrkdwn", "text": f"*Passed:*\nāœ… {total_results['success']} tests passed."}, + ], + } + else: + total = { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": "\nšŸŒž All tests passed."}, + ], + } + + blocks = [header, total] + + if total_results["failed"] > 0: + for key, result in results.items(): + print(key, result) + blocks.append({"type": "header", "text": {"type": "plain_text", "text": key, "emoji": True}}) + blocks.append( + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": f"*Results:*\n{result['failed']} failed, {result['success']} passed.", + }, + {"type": "mrkdwn", "text": f"*Time spent:*\n{result['time_spent']}"}, + ], + } + ) + elif not scheduled: + for key, result in results.items(): + blocks.append( + {"type": "section", "fields": [{"type": "mrkdwn", "text": f"*{key}*\n{result['time_spent']}."}]} + ) + + footer = { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f"", + }, + } + + blocks.append(footer) + + blocks = {"blocks": blocks} + + return blocks + + +if __name__ == "__main__": + arguments = sys.argv[1:] + + if "scheduled" in arguments: + arguments.remove("scheduled") + scheduled = True + else: + scheduled = False + + if scheduled: + # The scheduled run has several artifacts for each job. + file_paths = { + "TF Single GPU": { + "common": "run_all_tests_tf_gpu_test_reports/[].txt", + "pipeline": "run_all_tests_tf_gpu_test_reports/[].txt", + }, + "Torch Single GPU": { + "common": "run_all_tests_torch_gpu_test_reports/[].txt", + "pipeline": "run_all_tests_torch_gpu_test_reports/[].txt", + "examples": "run_all_tests_torch_gpu_test_reports/[].txt", + }, + "TF Multi GPU": { + "common": "run_all_tests_tf_multi_gpu_test_reports/[].txt", + "pipeline": "run_all_tests_tf_multi_gpu_test_reports/[].txt", + }, + "Torch Multi GPU": { + "common": "run_all_tests_torch_multi_gpu_test_reports/[].txt", + "pipeline": "run_all_tests_torch_multi_gpu_test_reports/[].txt", + }, + "Torch Cuda Extensions Single GPU": {"common": "run_tests_torch_cuda_extensions_gpu_test_reports/[].txt"}, + "Torch Cuda Extensions Multi GPU": { + "common": "run_tests_torch_cuda_extensions_multi_gpu_test_reports/[].txt" + }, + } + else: + file_paths = { + "TF Single GPU": {"common": "run_all_tests_tf_gpu_test_reports/[].txt"}, + "Torch Single GPU": {"common": "run_all_tests_torch_gpu_test_reports/[].txt"}, + "TF Multi GPU": {"common": "run_all_tests_tf_multi_gpu_test_reports/[].txt"}, + "Torch Multi GPU": {"common": "run_all_tests_torch_multi_gpu_test_reports/[].txt"}, + "Torch Cuda Extensions Single GPU": {"common": "run_tests_torch_cuda_extensions_gpu_test_reports/[].txt"}, + "Torch Cuda Extensions Multi GPU": { + "common": "run_tests_torch_cuda_extensions_multi_gpu_test_reports/[].txt" + }, + } + + client = WebClient(token=os.environ["CI_SLACK_BOT_TOKEN"]) + + if not scheduled: + channel_id = os.environ["CI_SLACK_CHANNEL_ID"] + elif scheduled and len(arguments): + channel_id = os.environ["CI_SLACK_CHANNEL_ID_PAST_FUTURE"] + else: + channel_id = os.environ["CI_SLACK_CHANNEL_ID_DAILY"] + + if scheduled: + title = "šŸ¤— Results of the scheduled tests." + else: + title = "šŸ¤— Self-push results" + + if len(arguments): + title = f"{arguments} " + title + + try: + results = {} + for job, file_dict in file_paths.items(): + + # Single return value for failed/success across steps of a same job + results[job] = {"failed": 0, "success": 0, "time_spent": "", "failures": ""} + + for key, file_path in file_dict.items(): + try: + with open(file_path.replace("[]", "stats")) as f: + failed, success, time_spent = handle_test_results(f.read()) + results[job]["failed"] += failed + results[job]["success"] += success + results[job]["time_spent"] += time_spent[1:-1] + ", " + with open(file_path.replace("[]", "summary_short")) as f: + for line in f: + if re.search("FAILED", line): + results[job]["failures"] += line + except FileNotFoundError: + print("Artifact was not found, job was probably canceled.") + + # Remove the trailing ", " + results[job]["time_spent"] = results[job]["time_spent"][:-2] + + test_results_keys = ["failed", "success"] + total = {"failed": 0, "success": 0} + for job, job_result in results.items(): + for result_key in test_results_keys: + total[result_key] += job_result[result_key] + + if total["failed"] != 0 or scheduled: + to_be_sent_to_slack = format_for_slack(total, results, scheduled, title) + + result = client.chat_postMessage( + channel=channel_id, + blocks=to_be_sent_to_slack["blocks"], + ) + + for job, job_result in results.items(): + if len(job_result["failures"]): + client.chat_postMessage( + channel=channel_id, text=f"{job}\n{job_result['failures']}", thread_ts=result["ts"] + ) + + except Exception as e: + # Voluntarily catch every exception and send it to Slack. + raise Exception(f"Setup error: no artifacts were found. Error: {e}") from e