diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 1e124c0183..d4e2e060b2 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -359,7 +359,16 @@ jobs: if: matrix.target_platform != 'Desktop' && !cancelled() run: | python scripts/gha/test_lab.py --android_model ${{ needs.prepare_matrix.outputs.android_device }} --android_api ${{ needs.prepare_matrix.outputs.android_api }} --ios_model ${{ needs.prepare_matrix.outputs.ios_device }} --ios_version ${{ needs.prepare_matrix.outputs.ios_version }} --testapp_dir ta --code_platform cpp --key_file scripts/gha-encrypted/gcs_key_file.json - + - name: Prepare results summary artifact + if: ${{ !cancelled() }} + run: | + cp ta/summary.log test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt + - name: Upload results summary artifact + uses: actions/upload-artifact@v2.2.2 + if: ${{ !cancelled() }} + with: + name: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }} + path: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt ### The below allow us to set the failure label and comment early, when the first failure ### in the matrix occurs. It'll be cleaned up in a subsequent job. - name: add failure label @@ -377,6 +386,22 @@ jobs: run: | echo -n "::set-output name=time::" TZ=America/Los_Angeles date + - name: download artifact + uses: actions/download-artifact@v2.0.8 + if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} + with: + # download-artifact doesn't support wildcards, but by default + # will download all artifacts. Sadly this is what we must do. + path: test_results + - name: get summary of test results + id: get-summary + shell: bash + if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} + run: | + mv test_results/test-results-*/test-results-*.txt test_results || true + echo 'SUMMARY_TABLE<> $GITHUB_ENV + python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV - name: add failure status comment uses: phulsechinmay/rewritable-pr-comment@v0.2.1 if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }} @@ -386,6 +411,7 @@ jobs: Requested by @${{github.actor}} on commit ${{github.event.pull_request.head.sha}} Last updated: ${{ steps.get-time.outputs.time }} **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** + ${{ env.SUMMARY_TABLE }} GITHUB_TOKEN: ${{ github.token }} COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} @@ -445,6 +471,29 @@ jobs: run: | echo -n "::set-output name=time::" TZ=America/Los_Angeles date + - uses: actions/checkout@v2 + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install python deps + run: | + python scripts/gha/install_prereqs_desktop.py + + - name: download artifact + uses: actions/download-artifact@v2.0.8 + with: + # download-artifact doesn't support wildcards, but by default + # will download all artifacts. Sadly this is what we must do. + path: test_results + - name: get summary of test results + shell: bash + run: | + mv test_results/test-results-*/test-results-*.txt test_results || true + echo 'SUMMARY_TABLE<> $GITHUB_ENV + python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV - name: add failure status comment uses: phulsechinmay/rewritable-pr-comment@v0.2.1 with: @@ -453,6 +502,7 @@ jobs: Requested by @${{github.actor}} on commit ${{github.event.pull_request.head.sha}} Last updated: ${{ steps.get-time.outputs.time }} **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})** + ${{ env.SUMMARY_TABLE }} GITHUB_TOKEN: ${{ github.token }} COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }} @@ -477,3 +527,32 @@ jobs: label: "${{ env.statusLabelInProgress }}" type: remove + summarize_results: + name: "summarize-results" + needs: [add_failure_label, add_success_label, remove_in_progress_label, tests] + runs-on: ubuntu-latest + if: ${{ !cancelled() }} + steps: + - uses: actions/checkout@v2 + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install python deps + run: | + python scripts/gha/install_prereqs_desktop.py + - name: download artifact + uses: actions/download-artifact@v2.0.8 + with: + path: test_results + - name: Summarize results into GitHub log + run: | + mv test_results/test-results-*/test-results-*.txt test_results || true + python scripts/gha/summarize_test_results.py --dir test_results --github_log + - uses: geekyeggo/delete-artifact@1-glob-support + # Delete all of the test result artifacts. + with: + name: | + test-results-* + failOnError: false + useGlob: true diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py new file mode 100644 index 0000000000..8b4da87283 --- /dev/null +++ b/scripts/gha/summarize_test_results.py @@ -0,0 +1,331 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Summarize integration test results. + +USAGE: + +python summarize_test_results.py --dir [--markdown] + +Example table mode output (will be slightly different with --markdown): + +| Platform | Build failures | Test failures | +|---------------------------|----------------|-----------------| +| iOS (build on iOS) | | auth, firestore | +| Desktop Windows (OpenSSL) | analytics | database | + +python summarize_test_results.py --dir <--text_log | --github_log> + +Example log mode output (will be slightly different with --github_log): + +INTEGRATION TEST FAILURES + +iOS (built on MacOS): + Test failures (2): + - auth + - firestore +Desktop Windows (OpenSSL): + Build failures (1): + - analytics +""" + +from absl import app +from absl import flags +from absl import logging +import glob +import re +import os + +FLAGS = flags.FLAGS + +flags.DEFINE_string( + "dir", None, + "Directory containing test results.", + short_name="d") + +flags.DEFINE_string( + "pattern", "test-results-*.txt", + "File pattern (glob) for test results." + "The '*' part is used to determine the platform.") + +flags.DEFINE_bool( + "include_successful", False, + "Print all logs including successful tests.") + +flags.DEFINE_bool( + "markdown", False, + "Display a Markdown-formatted table.") + +flags.DEFINE_bool( + "github_log", False, + "Display a GitHub log list.") + +flags.DEFINE_bool( + "text_log", False, + "Display a text log list.") + +flags.DEFINE_integer( + "list_max", 5, + "In Markdown mode, collapse lists larger than this size. 0 to disable.") + +CAPITALIZATIONS = { + "macos": "MacOS", + "ubuntu": "Linux", + "windows": "Windows", + "openssl": "(OpenSSL)", + "boringssl": "(BoringSSL)", + "ios": "iOS", + "android": "Android", + "desktop": "Desktop", +} + +PLATFORM_HEADER = "Platform" +BUILD_FAILURES_HEADER = "Build failures" +TEST_FAILURES_HEADER = "Test failures" +SUCCESSFUL_TESTS_HEADER = "Successful tests" + +LOG_HEADER = "INTEGRATION TEST FAILURES" + +# Default list separator for printing in text format. +DEFAULT_LIST_SEPARATOR=", " + +def print_table(log_results, + platform_width = 0, + build_failures_width = 0, + test_failures_width = 0, + successful_width = 0, + space_char = " ", + list_separator = DEFAULT_LIST_SEPARATOR): + """Print out a table in the requested format (text or markdown).""" + # Print table header + output_lines = list() + headers = [ + re.sub(r'\b \b', space_char, PLATFORM_HEADER.ljust(platform_width)), + re.sub(r'\b \b', space_char,BUILD_FAILURES_HEADER.ljust(build_failures_width)), + re.sub(r'\b \b', space_char,TEST_FAILURES_HEADER.ljust(test_failures_width)) + ] + ( + [re.sub(r'\b \b', space_char,SUCCESSFUL_TESTS_HEADER.ljust(successful_width))] + if FLAGS.include_successful else [] + ) + # Print header line. + output_lines.append(("|" + " %s |" * len(headers)) % tuple(headers)) + # Print a |-------|-------|---------| line. + output_lines.append(("|" + "-%s-|" * len(headers)) % + tuple([ re.sub("[^|]","-", header) for header in headers ])) + + # Iterate through platforms and print out table lines. + for platform in sorted(log_results.keys()): + if log_results[platform]["build_failures"] or log_results[platform]["test_failures"] or FLAGS.include_successful: + columns = [ + re.sub(r'\b \b', space_char, platform.ljust(platform_width)), + format_result(log_results[platform]["build_failures"], justify=build_failures_width, list_separator=list_separator), + format_result(log_results[platform]["test_failures"], justify=test_failures_width, list_separator=list_separator), + ] + ( + [format_result(log_results[platform]["successful"], justify=successful_width, list_separator=list_separator)] + if FLAGS.include_successful else [] + ) + output_lines.append(("|" + " %s |" * len(headers)) % tuple(columns)) + + return output_lines + + +def format_result(test_set, list_separator=DEFAULT_LIST_SEPARATOR, justify=0): + """Format a list of test names. + In Markdown mode, this can collapse a large list into a dropdown.""" + list_output = list_separator.join(sorted(test_set)) + if FLAGS.markdown and FLAGS.list_max > 0 and len(test_set) > FLAGS.list_max: + return "
_(%s items)_%s
" % ( + len(test_set), list_output) + else: + return list_output.ljust(justify) + + +def print_text_table(log_results): + """Print out a nicely-formatted text table.""" + # For text formatting, see how wide the strings are so we can + # left-justify each column of the text table. + max_platform = len(PLATFORM_HEADER) + max_build_failures = len(BUILD_FAILURES_HEADER) + max_test_failures = len(TEST_FAILURES_HEADER) + max_sucessful = len(SUCCESSFUL_TESTS_HEADER) + for (platform, results) in log_results.items(): + max_platform = max(max_platform, len(platform)) + max_build_failures = max(max_build_failures, + len(format_result(log_results[platform]["build_failures"]))) + max_test_failures = max(max_test_failures, + len(format_result(log_results[platform]["test_failures"]))) + max_sucessful = max(max_sucessful, + len(format_result(log_results[platform]["successful"]))) + return print_table(log_results, + platform_width=max_platform, + build_failures_width=max_build_failures, + test_failures_width=max_test_failures, + successful_width=max_sucessful) + + +def print_log(log_results): + """Print the results in a text-only log format.""" + output_lines = [] + for platform in sorted(log_results.keys()): + if log_results[platform]["build_failures"] or log_results[platform]["test_failures"] or FLAGS.include_successful: + output_lines.append("") + output_lines.append("%s:" % platform) + if (FLAGS.include_successful and len(log_results[platform]["successful"]) > 0): + output_lines.append(" Successful tests (%d):" % + len(log_results[platform]["successful"])) + for test_name in sorted(log_results[platform]["successful"]): + output_lines.append(" - %s" % test_name) + if (len(log_results[platform]["build_failures"]) > 0): + output_lines.append(" Build failures (%d):" % + len(log_results[platform]["build_failures"])) + for test_name in sorted(log_results[platform]["build_failures"]): + output_lines.append(" - %s" % test_name) + if (len(log_results[platform]["test_failures"]) > 0): + output_lines.append(" Test failures (%d):" % + len(log_results[platform]["test_failures"])) + for test_name in sorted(log_results[platform]["test_failures"]): + output_lines.append(" - %s" % test_name) + return output_lines[1:] # skip first blank line + + +def print_github_log(log_results): + """Print a text log, but replace newlines with %0A and add + the GitHub ::error text.""" + output_lines = [LOG_HEADER, ""] + print_log(log_results) + # "%0A" produces a newline in GitHub workflow logs. + return ["::error ::%s" % "%0A".join(output_lines)] + + +def print_markdown_table(log_results): + """Print a normal table, but with a few changes: + Separate test names by newlines, and replace certain spaces + with HTML non-breaking spaces to prevent aggressive word-wrapping.""" + return print_table(log_results, space_char = " ", list_separator = "
") + + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") + + log_files = glob.glob(os.path.join(FLAGS.dir, FLAGS.pattern)) + + # Replace the "*" in the file glob with a regex capture group, + # so we can report the test name. + log_name_re = re.escape( + os.path.join(FLAGS.dir,FLAGS.pattern)).replace("\\*", "(.*)") + + any_failures = False + log_data = {} + + for log_file in log_files: + # Extract the matrix name for this log. + log_name = re.search(log_name_re, log_file).groups(1)[0] + # Split the matrix name into components. + log_name = re.sub(r'[-_.]+', ' ', log_name).split() + # Remove redundant components. + if "latest" in log_name: log_name.remove("latest") + if "Android" in log_name or "iOS" in log_name: + log_name.remove('openssl') + # Capitalize components in a nice way. + log_name = [ + CAPITALIZATIONS[name.lower()] + if name.lower() in CAPITALIZATIONS + else name + for name in log_name] + if "Android" in log_name or "iOS" in log_name: + # For Android and iOS, highlight the target OS. + log_name[0] = "(built on %s)" % log_name[0] + if FLAGS.markdown: + log_name[1] = "**%s**" % log_name[1] + elif FLAGS.markdown: + # For desktop, highlight the entire platform string. + log_name[0] = "%s**" % log_name[0] + log_name[1] = "**%s" % log_name[1] + # Rejoin matrix name with spaces. + log_name = ' '.join([log_name[1], log_name[0]]+log_name[2:]) + with open(log_file, "r") as log_reader: + log_data[log_name] = log_reader.read() + + log_results = {} + # Go through each log and extract out the build and test failures. + for (platform, log_text) in log_data.items(): + if platform not in log_results: + log_results[platform] = { "build_failures": set(), "test_failures": set(), + "attempted": set(), "successful": set() } + # Get a full list of the products built. + m = re.search(r'TRIED TO BUILD: ([^\n]*)', log_text) + if m: + log_results[platform]["attempted"].update(m.group(1).split(",")) + # Extract build failure lines, which follow "SOME FAILURES OCCURRED:" + m = re.search(r'SOME FAILURES OCCURRED:\n(([\d]+:[^\n]*\n)+)', log_text, re.MULTILINE) + if m: + for build_failure_line in m.group(1).strip("\n").split("\n"): + m2 = re.match(r'[\d]+: ([^,]+)', build_failure_line) + if m2: + product_name = m2.group(1).lower() + if product_name: + log_results[platform]["build_failures"].add(product_name) + any_failures = True + + # Extract test failures, which follow "TESTAPPS EXPERIENCED ERRORS:" + m = re.search(r'TESTAPPS (EXPERIENCED ERRORS|FAILED):\n(([^\n]*\n)+)', log_text, re.MULTILINE) + if m: + for test_failure_line in m.group(2).strip("\n").split("\n"): + # Only get the lines showing paths. + if "/firebase-cpp-sdk/" not in test_failure_line: continue + test_filename = ""; + if "log tail" in test_failure_line: + test_filename = re.match(r'^(.*) log tail', test_failure_line).group(1) + if "lacks logs" in test_failure_line: + test_filename = re.match(r'^(.*) lacks logs', test_failure_line).group(1) + if "it-debug.apk" in test_failure_line: + test_filename = re.match(r'^(.*it-debug\.apk)', test_failure_line).group(1) + if "integration_test.ipa" in test_failure_line: + test_filename = re.match(r'^(.*integration_test\.ipa)', test_failure_line).group(1) + + if test_filename: + m2 = re.search(r'/ta/(firebase)?([^/]+)/iti?/', test_filename, re.IGNORECASE) + if not m2: m2 = re.search(r'/testapps/(firebase)?([^/]+)/integration_test', test_filename, re.IGNORECASE) + if m2: + product_name = m2.group(2).lower() + if product_name: + log_results[platform]["test_failures"].add(product_name) + any_failures = True + + # After processing all the logs, we can determine the successful builds for each platform. + for platform in log_results.keys(): + log_results[platform]["successful"] = log_results[platform]["attempted"].difference( + log_results[platform]["test_failures"].union( + log_results[platform]["build_failures"])) + + if not any_failures and not FLAGS.include_successful: + # No failures occurred, nothing to log. + return(0) + + log_lines = [] + if FLAGS.markdown: + log_lines = print_markdown_table(log_results) + # If outputting Markdown, don't bother justifying the table. + elif FLAGS.github_log: + log_lines = print_github_log(log_results) + elif FLAGS.text_log: + log_lines = print_log(log_results) + else: + log_lines = print_text_table(log_results) + + print("\n".join(log_lines)) + +if __name__ == "__main__": + flags.mark_flag_as_required("dir") + app.run(main)