From 58ae07709ef35d9dfd39e1d9653ff2cfbb3d974a Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 09:24:28 -0700
Subject: [PATCH 01/32] Temporarily disable Windows-Android tests.

---
 .github/workflows/integration_tests.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 1e124c0183..028d2580c6 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -220,6 +220,8 @@ jobs:
             ssl_variant: boringssl
           - target_platform: Android
             ssl_variant: boringssl
+          - os: windows-latest
+            target_platform: Android
 
     steps:
       - uses: actions/checkout@v2

From 3b32605292a3d21bcf8b31c2d30654cef3a874af Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 11:23:51 -0700
Subject: [PATCH 02/32] Add script to summarize test results. Summarize test
 results in failure.

---
 .github/workflows/integration_tests.yml |  37 ++++++
 scripts/gha/summarize_test_results.py   | 169 ++++++++++++++++++++++++
 2 files changed, 206 insertions(+)
 create mode 100644 scripts/gha/summarize_test_results.py

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 028d2580c6..aed3506414 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -362,6 +362,12 @@ jobs:
         run: |
           python scripts/gha/test_lab.py --android_model ${{ needs.prepare_matrix.outputs.android_device }} --android_api ${{ needs.prepare_matrix.outputs.android_api }} --ios_model ${{ needs.prepare_matrix.outputs.ios_device }} --ios_version ${{ needs.prepare_matrix.outputs.ios_version }} --testapp_dir ta --code_platform cpp --key_file scripts/gha-encrypted/gcs_key_file.json
 
+      - name: upload results summary artifact
+        uses: actions/upload-artifact@v2.2.2
+        with:
+          name: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
+          path: ta/summary.log
+
       ### The below allow us to set the failure label and comment early, when the first failure
       ### in the matrix occurs. It'll be cleaned up in a subsequent job.
       - name: add failure label
@@ -379,6 +385,22 @@ jobs:
         run: |
           echo -n "::set-output name=time::"
           TZ=America/Los_Angeles date
+      - name: download artifact
+        uses: actions/download-artifact@v2.0.8
+        if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }}
+        with:
+          # download-artifact doesn't support wildcards, but by default
+          # will download all artifacts. Sadly this is what we must do.
+          path: test_results
+      - name: get summary of test results
+        id: get-summary
+        shell: bash
+        if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }}
+        run: |
+          mv test_results/test-results-*/test-results-*.txt test_results
+          echo -n "::set-output name=table::"
+          python scripts/gha/summarize_test_results.py -d test_results --singleline
+
       - name: add failure status comment
         uses: phulsechinmay/rewritable-pr-comment@v0.2.1
         if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }}
@@ -388,6 +410,7 @@ jobs:
             Requested by @${{github.actor}} on commit ${{github.event.pull_request.head.sha}}
             Last updated: ${{ steps.get-time.outputs.time }}
             **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})**
+            ${{ steps.get-summary.outputs.table }}
           GITHUB_TOKEN: ${{ github.token }}
           COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }}
 
@@ -447,6 +470,19 @@ jobs:
         run: |
           echo -n "::set-output name=time::"
           TZ=America/Los_Angeles date
+      - name: download artifact
+        uses: actions/download-artifact@v2.0.8
+        with:
+          # download-artifact doesn't support wildcards, but by default
+          # will download all artifacts. Sadly this is what we must do.
+          path: test_results
+      - name: get summary of test results
+        id: get-summary
+        shell: bash
+        run: |
+          mv test_results/test-results-*/test-results-*.txt test_results
+          echo -n "::set-output name=table::"
+          python scripts/gha/summarize_test_results.py -d test_results --singleline
       - name: add failure status comment
         uses: phulsechinmay/rewritable-pr-comment@v0.2.1
         with:
@@ -455,6 +491,7 @@ jobs:
             Requested by @${{github.actor}} on commit ${{github.event.pull_request.head.sha}}
             Last updated: ${{ steps.get-time.outputs.time }}
             **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})**
+            ${{ steps.get-summary.outputs.table }}
           GITHUB_TOKEN: ${{ github.token }}
           COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }}
 
diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
new file mode 100644
index 0000000000..a52f24a275
--- /dev/null
+++ b/scripts/gha/summarize_test_results.py
@@ -0,0 +1,169 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Summarize integration test results.
+
+USAGE:
+
+python summarize_test_results.py -d <directory>
+
+Example output:
+
+| Platform                | Build failures | Test failures   |
+| ----------------------- | -------------- | --------------- |
+| MacOS iOS               |                | auth, firestore |
+| Windows Desktop OpenSSL | analytics      | database        |
+"""
+
+from absl import app
+from absl import flags
+from absl import logging
+import glob
+import re
+import os
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string(
+    "dir", None,
+    "Directory containing test results.",
+    short_name="d")
+
+flags.DEFINE_string(
+    "pattern", "test-results-*.txt",
+    "File pattern (glob) for test results.")
+
+flags.DEFINE_bool(
+    "full", False,
+    "Print a full table, including successful tests.")
+
+flags.DEFINE_bool(
+    "singleline", False,
+    "Output a single line, with \n for newlines.")
+
+CAPITALIZATIONS = {
+    "macos": "MacOS",
+    "ubuntu": "Ubuntu",
+    "windows": "Windows",
+    "openssl": "OpenSSL",
+    "boringssl": "BoringSSL",
+    "ios": "iOS",
+    "android": "Android",
+}
+
+PLATFORM_HEADER = "Platform"
+BUILD_FAILURES_HEADER = "Build failures"
+TEST_FAILURES_HEADER = "Test failures"
+
+def main(argv):
+  if len(argv) > 1:
+      raise app.UsageError("Too many command-line arguments.")
+
+  log_files = glob.glob(os.path.join(FLAGS.dir, FLAGS.pattern))
+
+  # Replace the "*" in the file glob with a regex capture group,
+  # so we can report the test name.
+  log_name_re = re.escape(
+      os.path.join(FLAGS.dir,FLAGS.pattern)).replace("\\*", "(.*)")
+
+  log_data = {}
+
+  for log_file in log_files:
+      # Extract the matrix name for this log.
+      log_name = re.search(log_name_re, log_file).groups(1)[0]
+      # Split the matrix name into components.
+      log_name = re.sub(r'[-_.]+', ' ', log_name).split()
+      # Remove redundant components.
+      if "Android" in log_name or "iOS" in log_name:
+          log_name.remove('openssl')
+      log_name.remove('latest')
+      # Capitalize components in a nice way.
+      log_name = [
+          CAPITALIZATIONS[name.lower()]
+          if name.lower() in CAPITALIZATIONS
+          else name.capitalize()
+          for name in log_name]
+      # Rejoin matrix name with spaces.
+      log_name = ' '.join(log_name)
+      with open(log_file, "r") as log_reader:
+          log_data[log_name] = log_reader.read()
+
+  log_results = {}
+  # Go through each log and extract out the build and test failures.
+  for (platform, log_text) in log_data.items():
+      log_results[platform] = { "build_failures": set(), "test_failures": set() }
+      # Extract build failure lines, which follow "SOME FAILURES OCCURRED:"
+      m = re.search(r'SOME FAILURES OCCURRED:\n(([\d+]:[^\n]*\n)+)', log_text, re.MULTILINE)
+      if m:
+        for build_failure_line in m.group(1).strip("\n").split("\n"):
+          m2 = re.match(r'[\d]+: ([^,]+)', build_failure_line)
+          if m2:
+            product_name = m2.group(1).lower()
+            if product_name:
+              log_results[platform]["build_failures"].add(product_name)
+      # Extract test failures, which follow "TESTAPPS EXPERIENCED ERRORS:"
+      m = re.search(r'TESTAPPS EXPERIENCED ERRORS:\n(([^\n]*\n)+)', log_text, re.MULTILINE)
+      if m:
+        for test_failure_line in m.group(1).strip("\n").split("\n"):
+          # Only get the lines showing paths.
+          if "/firebase-cpp-sdk/" not in test_failure_line: continue
+          test_filename = "";
+          if "log tail" in test_failure_line:
+            test_filename = re.match(r'^(.*) log tail', test_failure_line).group(1)
+          if "lacks logs" in test_failure_line:
+            test_filename = re.match(r'^(.*) lacks logs', test_failure_line).group(1)
+          if test_filename:
+            m2 = re.search(r'/ta/(firebase)?([^/]+)/iti?/', test_filename, re.IGNORECASE)
+            if not m2: m2 = re.search(r'/testapps/(firebase)?([^/]+)/integration_test', test_filename, re.IGNORECASE)
+            if m2:
+              product_name = m2.group(2).lower()
+              if product_name:
+                log_results[platform]["test_failures"].add(product_name)
+          
+
+  
+  # For text formatting, see how wide the strings are.
+  max_platform = len(PLATFORM_HEADER)
+  max_build_failures = len(BUILD_FAILURES_HEADER)
+  max_test_failures = len(TEST_FAILURES_HEADER)
+  for (platform, results) in log_results.items():
+    build_failures = ", ".join(sorted(log_results[platform]["build_failures"]))
+    test_failures = ", ".join(sorted(log_results[platform]["test_failures"]))
+    max_platform = max(max_platform, len(platform))
+    max_build_failures = max(max_build_failures, len(build_failures))
+    max_test_failures = max(max_test_failures, len(test_failures))
+
+  output_lines = list()
+  output_lines.append("| %s | %s | %s |" % (
+    PLATFORM_HEADER.ljust(max_platform),
+    BUILD_FAILURES_HEADER.ljust(max_build_failures),
+    TEST_FAILURES_HEADER.ljust(max_test_failures)))
+  output_lines.append("|-%s-|-%s-|-%s-|" % (
+    "".ljust(max_platform, "-"),
+    "".ljust(max_build_failures, "-"),
+    "".ljust(max_test_failures, "-")))
+
+  for platform in sorted(log_results.keys()):
+    if log_results[platform]["build_failures"] or log_results[platform]["test_failures"]:
+      platform_str = platform.ljust(max_platform)
+      build_failures = ", ".join(sorted(log_results[platform]["build_failures"])).ljust(max_build_failures)
+      test_failures = ", ".join(sorted(log_results[platform]["test_failures"])).ljust(max_test_failures)
+      output_lines.append("| %s | %s | %s |" % (platform_str, build_failures, test_failures))
+
+  output_delim = "\\n" if FLAGS.singleline else "\n"
+  print(output_delim.join(output_lines))
+
+if __name__ == "__main__":
+  flags.mark_flag_as_required("dir")
+  app.run(main)

From 6acd2df82a52da8c87cf0fe852452d6317c8e9cf Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 12:24:40 -0700
Subject: [PATCH 03/32] Upload artifact even if failed. Re-add Windows Android.

---
 .github/workflows/integration_tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index aed3506414..734f8dda2a 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -220,8 +220,6 @@ jobs:
             ssl_variant: boringssl
           - target_platform: Android
             ssl_variant: boringssl
-          - os: windows-latest
-            target_platform: Android
 
     steps:
       - uses: actions/checkout@v2
@@ -364,6 +362,7 @@ jobs:
 
       - name: upload results summary artifact
         uses: actions/upload-artifact@v2.2.2
+        if: !cancelled()
         with:
           name: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
           path: ta/summary.log

From 427eb3fe5cfc89e05a43fb399a723e3302457536 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 12:58:59 -0700
Subject: [PATCH 04/32] Fix if: statement

---
 .github/workflows/integration_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 734f8dda2a..93c108394b 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -362,7 +362,7 @@ jobs:
 
       - name: upload results summary artifact
         uses: actions/upload-artifact@v2.2.2
-        if: !cancelled()
+        if: ${{ !cancelled() }}
         with:
           name: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
           path: ta/summary.log

From 25dff9be9e53af87cecda88763033cb23d07f908 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 13:36:35 -0700
Subject: [PATCH 05/32] Add non-breaking spaces to clean up wrapping.

---
 .github/workflows/integration_tests.yml |  4 ++--
 scripts/gha/summarize_test_results.py   | 15 ++++++++++-----
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 93c108394b..fee42e6cd0 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -398,7 +398,7 @@ jobs:
         run: |
           mv test_results/test-results-*/test-results-*.txt test_results
           echo -n "::set-output name=table::"
-          python scripts/gha/summarize_test_results.py -d test_results --singleline
+          python scripts/gha/summarize_test_results.py --dir test_results --singleline --markdown
 
       - name: add failure status comment
         uses: phulsechinmay/rewritable-pr-comment@v0.2.1
@@ -481,7 +481,7 @@ jobs:
         run: |
           mv test_results/test-results-*/test-results-*.txt test_results
           echo -n "::set-output name=table::"
-          python scripts/gha/summarize_test_results.py -d test_results --singleline
+          python scripts/gha/summarize_test_results.py --dir test_results --singleline --markdown
       - name: add failure status comment
         uses: phulsechinmay/rewritable-pr-comment@v0.2.1
         with:
diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index a52f24a275..bc8eb47d37 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -52,6 +52,10 @@
     "singleline", False,
     "Output a single line, with \n for newlines.")
 
+flags.DEFINE_bool(
+    "markdown", False,
+    "Display a Markdown-formatted table.")
+
 CAPITALIZATIONS = {
     "macos": "MacOS",
     "ubuntu": "Ubuntu",
@@ -92,7 +96,7 @@ def main(argv):
       log_name = [
           CAPITALIZATIONS[name.lower()]
           if name.lower() in CAPITALIZATIONS
-          else name.capitalize()
+          else name
           for name in log_name]
       # Rejoin matrix name with spaces.
       log_name = ' '.join(log_name)
@@ -144,11 +148,12 @@ def main(argv):
     max_build_failures = max(max_build_failures, len(build_failures))
     max_test_failures = max(max_test_failures, len(test_failures))
 
+  space_char = "&nbsp;" if FLAGS.markdown else " ";
   output_lines = list()
   output_lines.append("| %s | %s | %s |" % (
-    PLATFORM_HEADER.ljust(max_platform),
-    BUILD_FAILURES_HEADER.ljust(max_build_failures),
-    TEST_FAILURES_HEADER.ljust(max_test_failures)))
+    re.sub(r'\b \b', space_char, PLATFORM_HEADER.ljust(max_platform)),
+    re.sub(r'\b \b', space_char,BUILD_FAILURES_HEADER.ljust(max_build_failures)),
+    re.sub(r'\b \b', space_char,TEST_FAILURES_HEADER.ljust(max_test_failures))))
   output_lines.append("|-%s-|-%s-|-%s-|" % (
     "".ljust(max_platform, "-"),
     "".ljust(max_build_failures, "-"),
@@ -156,7 +161,7 @@ def main(argv):
 
   for platform in sorted(log_results.keys()):
     if log_results[platform]["build_failures"] or log_results[platform]["test_failures"]:
-      platform_str = platform.ljust(max_platform)
+      platform_str = re.sub(r'\b \b', space_char, platform.ljust(max_platform))
       build_failures = ", ".join(sorted(log_results[platform]["build_failures"])).ljust(max_build_failures)
       test_failures = ", ".join(sorted(log_results[platform]["test_failures"])).ljust(max_test_failures)
       output_lines.append("| %s | %s | %s |" % (platform_str, build_failures, test_failures))

From 2947a13e5f38b34e654a0d608eabb25b03eb6e92 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 13:56:04 -0700
Subject: [PATCH 06/32] Add more Markdown niceness to the test results.

---
 .github/workflows/integration_tests.yml |  2 +-
 scripts/gha/summarize_test_results.py   | 47 ++++++++++++++++++-------
 2 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index fee42e6cd0..61df9410d9 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -396,7 +396,7 @@ jobs:
         shell: bash
         if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }}
         run: |
-          mv test_results/test-results-*/test-results-*.txt test_results
+          mv test_results/test-results-*/test-results-*.txt test_results || true
           echo -n "::set-output name=table::"
           python scripts/gha/summarize_test_results.py --dir test_results --singleline --markdown
 
diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index bc8eb47d37..cdc91d3988 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -56,6 +56,10 @@
     "markdown", False,
     "Display a Markdown-formatted table.")
 
+flags.DEFINE_integer(
+    "list_max", 5,
+    "In Markdown mode, collapse lists larger than this size. 0 to disable.")
+
 CAPITALIZATIONS = {
     "macos": "MacOS",
     "ubuntu": "Ubuntu",
@@ -137,18 +141,26 @@ def main(argv):
           
 
   
-  # For text formatting, see how wide the strings are.
-  max_platform = len(PLATFORM_HEADER)
-  max_build_failures = len(BUILD_FAILURES_HEADER)
-  max_test_failures = len(TEST_FAILURES_HEADER)
-  for (platform, results) in log_results.items():
-    build_failures = ", ".join(sorted(log_results[platform]["build_failures"]))
-    test_failures = ", ".join(sorted(log_results[platform]["test_failures"]))
-    max_platform = max(max_platform, len(platform))
-    max_build_failures = max(max_build_failures, len(build_failures))
-    max_test_failures = max(max_test_failures, len(test_failures))
-
-  space_char = "&nbsp;" if FLAGS.markdown else " ";
+  if FLAGS.markdown:
+    # If outputting Markdown, don't bother justifying the table.
+    max_platform = 0
+    max_build_failures = 0
+    max_test_failures = 0
+    space_char = "&nbsp;"
+  else:
+    # For text formatting, see how wide the strings are so we can
+    # justify the text table.
+    max_platform = len(PLATFORM_HEADER)
+    max_build_failures = len(BUILD_FAILURES_HEADER)
+    max_test_failures = len(TEST_FAILURES_HEADER)
+    for (platform, results) in log_results.items():
+      build_failures = ", ".join(sorted(log_results[platform]["build_failures"]))
+      test_failures = ", ".join(sorted(log_results[platform]["test_failures"]))
+      max_platform = max(max_platform, len(platform))
+      max_build_failures = max(max_build_failures, len(build_failures))
+      max_test_failures = max(max_test_failures, len(test_failures))
+      space_char = " "
+
   output_lines = list()
   output_lines.append("| %s | %s | %s |" % (
     re.sub(r'\b \b', space_char, PLATFORM_HEADER.ljust(max_platform)),
@@ -164,6 +176,17 @@ def main(argv):
       platform_str = re.sub(r'\b \b', space_char, platform.ljust(max_platform))
       build_failures = ", ".join(sorted(log_results[platform]["build_failures"])).ljust(max_build_failures)
       test_failures = ", ".join(sorted(log_results[platform]["test_failures"])).ljust(max_test_failures)
+      if FLAGS.markdown:
+        # If there are more than N failures, collapse the results.
+        if FLAGS.list_max and len(log_results[platform]["build_failures"]) > FLAGS.list_max:
+          build_failures = "<details><summary>_%s items_</summary>%s</details>" % (
+            len(log_results[platform]["build_failures"]), build_failures)
+        if FLAGS.list_max and len(log_results[platform]["test_failures"]) > FLAGS.list_max:
+          test_failures = "<details><summary>_%s items_</summary>%s</details>" % (
+            len(log_results[platform]["test_failures"]), test_failures)
+      else:
+        build_failures = ", ".join(sorted(log_results[platform]["build_failures"])).ljust(max_build_failures)
+        test_failures = ", ".join(sorted(log_results[platform]["test_failures"])).ljust(max_test_failures)
       output_lines.append("| %s | %s | %s |" % (platform_str, build_failures, test_failures))
 
   output_delim = "\\n" if FLAGS.singleline else "\n"

From 272730892507322f4658796c6de3530cb49bead4 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 13:57:57 -0700
Subject: [PATCH 07/32] Add debug output.

---
 .github/workflows/integration_tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 61df9410d9..41a06d5c2d 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -396,6 +396,7 @@ jobs:
         shell: bash
         if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }}
         run: |
+          ls -lR test_results
           mv test_results/test-results-*/test-results-*.txt test_results || true
           echo -n "::set-output name=table::"
           python scripts/gha/summarize_test_results.py --dir test_results --singleline --markdown

From 8f04396dc9369fafd573f41faf73989405035bc4 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 14:05:32 -0700
Subject: [PATCH 08/32] don't error out if there is only one failure artifact.

---
 .github/workflows/integration_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 41a06d5c2d..19861cf33f 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -480,7 +480,7 @@ jobs:
         id: get-summary
         shell: bash
         run: |
-          mv test_results/test-results-*/test-results-*.txt test_results
+          mv test_results/test-results-*/test-results-*.txt test_results || true
           echo -n "::set-output name=table::"
           python scripts/gha/summarize_test_results.py --dir test_results --singleline --markdown
       - name: add failure status comment

From 599f6fb75a754309fabdcf4552adcbea3d162b44 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 14:19:51 -0700
Subject: [PATCH 09/32] Add tracking of attempted and successful products (not
 printed) and fix artifact upload.

---
 .github/workflows/integration_tests.yml |  5 +++--
 scripts/gha/summarize_test_results.py   | 16 ++++++++++++----
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 19861cf33f..54facd65ea 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -359,13 +359,14 @@ jobs:
         if: matrix.target_platform != 'Desktop' && !cancelled()
         run: |
           python scripts/gha/test_lab.py --android_model ${{ needs.prepare_matrix.outputs.android_device }} --android_api ${{ needs.prepare_matrix.outputs.android_api }} --ios_model ${{ needs.prepare_matrix.outputs.ios_device }} --ios_version ${{ needs.prepare_matrix.outputs.ios_version }} --testapp_dir ta --code_platform cpp --key_file scripts/gha-encrypted/gcs_key_file.json
+          cp ta/summary.log test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
 
       - name: upload results summary artifact
         uses: actions/upload-artifact@v2.2.2
         if: ${{ !cancelled() }}
         with:
-          name: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
-          path: ta/summary.log
+          name: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}
+          path: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
 
       ### The below allow us to set the failure label and comment early, when the first failure
       ### in the matrix occurs. It'll be cleaned up in a subsequent job.
diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index cdc91d3988..c0090b0fbc 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -110,7 +110,12 @@ def main(argv):
   log_results = {}
   # Go through each log and extract out the build and test failures.
   for (platform, log_text) in log_data.items():
-      log_results[platform] = { "build_failures": set(), "test_failures": set() }
+      log_results[platform] = { "build_failures": set(), "test_failures": set(),
+                                "all": set(), "successful": set() }
+      # Get a full list of the products built.
+      m = re.search(r'TRIED TO BUILD: ([^\n]*)', log_text)
+      if m:
+        log_results[platform]["all"].update(m.group(1).split(","))
       # Extract build failure lines, which follow "SOME FAILURES OCCURRED:"
       m = re.search(r'SOME FAILURES OCCURRED:\n(([\d+]:[^\n]*\n)+)', log_text, re.MULTILINE)
       if m:
@@ -138,8 +143,11 @@ def main(argv):
               product_name = m2.group(2).lower()
               if product_name:
                 log_results[platform]["test_failures"].add(product_name)
-          
 
+  for platform in log_results.keys():
+    log_results[platform]["successful"] = log_results[platform]["all"].difference(
+      log_results[platform]["test_failures"].union(
+        log_results[platform]["build_failures"]))
   
   if FLAGS.markdown:
     # If outputting Markdown, don't bother justifying the table.
@@ -179,10 +187,10 @@ def main(argv):
       if FLAGS.markdown:
         # If there are more than N failures, collapse the results.
         if FLAGS.list_max and len(log_results[platform]["build_failures"]) > FLAGS.list_max:
-          build_failures = "<details><summary>_%s items_</summary>%s</details>" % (
+          build_failures = "<details><summary>_(%s items)_</summary>%s</details>" % (
             len(log_results[platform]["build_failures"]), build_failures)
         if FLAGS.list_max and len(log_results[platform]["test_failures"]) > FLAGS.list_max:
-          test_failures = "<details><summary>_%s items_</summary>%s</details>" % (
+          test_failures = "<details><summary>_(%s items)_</summary>%s</details>" % (
             len(log_results[platform]["test_failures"]), test_failures)
       else:
         build_failures = ", ".join(sorted(log_results[platform]["build_failures"])).ljust(max_build_failures)

From 3c61d506f2f2dd16d1cdb8a68016f6086db2b505 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 14:26:12 -0700
Subject: [PATCH 10/32] Clean up output slightly.

---
 scripts/gha/summarize_test_results.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index c0090b0fbc..70b37ad73e 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -64,8 +64,8 @@
     "macos": "MacOS",
     "ubuntu": "Ubuntu",
     "windows": "Windows",
-    "openssl": "OpenSSL",
-    "boringssl": "BoringSSL",
+    "openssl": "(OpenSSL)",
+    "boringssl": "(BoringSSL)",
     "ios": "iOS",
     "android": "Android",
 }
@@ -111,11 +111,11 @@ def main(argv):
   # Go through each log and extract out the build and test failures.
   for (platform, log_text) in log_data.items():
       log_results[platform] = { "build_failures": set(), "test_failures": set(),
-                                "all": set(), "successful": set() }
+                                "attempted": set(), "successful": set() }
       # Get a full list of the products built.
       m = re.search(r'TRIED TO BUILD: ([^\n]*)', log_text)
       if m:
-        log_results[platform]["all"].update(m.group(1).split(","))
+        log_results[platform]["attempted"].update(m.group(1).split(","))
       # Extract build failure lines, which follow "SOME FAILURES OCCURRED:"
       m = re.search(r'SOME FAILURES OCCURRED:\n(([\d+]:[^\n]*\n)+)', log_text, re.MULTILINE)
       if m:
@@ -145,7 +145,7 @@ def main(argv):
                 log_results[platform]["test_failures"].add(product_name)
 
   for platform in log_results.keys():
-    log_results[platform]["successful"] = log_results[platform]["all"].difference(
+    log_results[platform]["successful"] = log_results[platform]["attempted"].difference(
       log_results[platform]["test_failures"].union(
         log_results[platform]["build_failures"]))
   
@@ -154,6 +154,8 @@ def main(argv):
     max_platform = 0
     max_build_failures = 0
     max_test_failures = 0
+    # Certain spaces are replaced with HTML non-breaking spaces, to prevent
+    # aggressive word-wrapping from messing up the formatting.
     space_char = "&nbsp;"
   else:
     # For text formatting, see how wide the strings are so we can
@@ -169,6 +171,7 @@ def main(argv):
       max_test_failures = max(max_test_failures, len(test_failures))
       space_char = " "
 
+  # Output a table (text or markdown) of failure platforms & tests.
   output_lines = list()
   output_lines.append("| %s | %s | %s |" % (
     re.sub(r'\b \b', space_char, PLATFORM_HEADER.ljust(max_platform)),

From 28c086177fa39e05b40a714664cf5d5545f19f49 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 15:00:46 -0700
Subject: [PATCH 11/32] Use multi-line environment variable instead of output.

---
 .github/workflows/integration_tests.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 54facd65ea..2ee9df4126 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -397,11 +397,11 @@ jobs:
         shell: bash
         if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }}
         run: |
-          ls -lR test_results
+        run: |
           mv test_results/test-results-*/test-results-*.txt test_results || true
-          echo -n "::set-output name=table::"
-          python scripts/gha/summarize_test_results.py --dir test_results --singleline --markdown
-
+          echo 'SUMMARY_TABLE<<EOF' >> $GITHUB_ENV
+          python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
       - name: add failure status comment
         uses: phulsechinmay/rewritable-pr-comment@v0.2.1
         if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }}
@@ -411,7 +411,7 @@ jobs:
             Requested by @${{github.actor}} on commit ${{github.event.pull_request.head.sha}}
             Last updated: ${{ steps.get-time.outputs.time }}
             **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})**
-            ${{ steps.get-summary.outputs.table }}
+            ${{ env.SUMMARY_TABLE }}
           GITHUB_TOKEN: ${{ github.token }}
           COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }}
 
@@ -478,12 +478,12 @@ jobs:
           # will download all artifacts. Sadly this is what we must do.
           path: test_results
       - name: get summary of test results
-        id: get-summary
         shell: bash
         run: |
           mv test_results/test-results-*/test-results-*.txt test_results || true
-          echo -n "::set-output name=table::"
-          python scripts/gha/summarize_test_results.py --dir test_results --singleline --markdown
+          echo 'SUMMARY_TABLE<<EOF' >> $GITHUB_ENV
+          python scripts/gha/summarize_test_results.py --dir test_results --markdown >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
       - name: add failure status comment
         uses: phulsechinmay/rewritable-pr-comment@v0.2.1
         with:
@@ -492,7 +492,7 @@ jobs:
             Requested by @${{github.actor}} on commit ${{github.event.pull_request.head.sha}}
             Last updated: ${{ steps.get-time.outputs.time }}
             **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})**
-            ${{ steps.get-summary.outputs.table }}
+            ${{ env.SUMMARY_TABLE }}
           GITHUB_TOKEN: ${{ github.token }}
           COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }}
 

From e76ffe7a4fddbc9dfa7b8206cb64f4eb791f75a4 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 15:02:39 -0700
Subject: [PATCH 12/32] Fix syntax issue.

---
 .github/workflows/integration_tests.yml | 1 -
 scripts/gha/summarize_test_results.py   | 7 +------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 2ee9df4126..257d95a0ff 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -396,7 +396,6 @@ jobs:
         id: get-summary
         shell: bash
         if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }}
-        run: |
         run: |
           mv test_results/test-results-*/test-results-*.txt test_results || true
           echo 'SUMMARY_TABLE<<EOF' >> $GITHUB_ENV
diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index 70b37ad73e..46d106ec85 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -48,10 +48,6 @@
     "full", False,
     "Print a full table, including successful tests.")
 
-flags.DEFINE_bool(
-    "singleline", False,
-    "Output a single line, with \n for newlines.")
-
 flags.DEFINE_bool(
     "markdown", False,
     "Display a Markdown-formatted table.")
@@ -200,8 +196,7 @@ def main(argv):
         test_failures = ", ".join(sorted(log_results[platform]["test_failures"])).ljust(max_test_failures)
       output_lines.append("| %s | %s | %s |" % (platform_str, build_failures, test_failures))
 
-  output_delim = "\\n" if FLAGS.singleline else "\n"
-  print(output_delim.join(output_lines))
+  print("\n".join(output_lines))
 
 if __name__ == "__main__":
   flags.mark_flag_as_required("dir")

From 068320d33e6add5aad08148650d1503cb27cf0ed Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 15:05:31 -0700
Subject: [PATCH 13/32] Fix handling of 10th+ build error.

---
 scripts/gha/summarize_test_results.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index 46d106ec85..1a278e4f4a 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -113,7 +113,7 @@ def main(argv):
       if m:
         log_results[platform]["attempted"].update(m.group(1).split(","))
       # Extract build failure lines, which follow "SOME FAILURES OCCURRED:"
-      m = re.search(r'SOME FAILURES OCCURRED:\n(([\d+]:[^\n]*\n)+)', log_text, re.MULTILINE)
+      m = re.search(r'SOME FAILURES OCCURRED:\n(([\d]+:[^\n]*\n)+)', log_text, re.MULTILINE)
       if m:
         for build_failure_line in m.group(1).strip("\n").split("\n"):
           m2 = re.match(r'[\d]+: ([^,]+)', build_failure_line)

From 82008dbb8a50bfe2a1e072ef66a0e84735541869 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 15:52:55 -0700
Subject: [PATCH 14/32] Use newline to separate test names. Add debugging info
 for github context.

---
 .github/workflows/integration_tests.yml |  3 ++-
 scripts/gha/summarize_test_results.py   | 13 ++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 257d95a0ff..80bd0cabf9 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -99,6 +99,7 @@ jobs:
         elif [[ "${{ github.event.label.name }}" == "${{ env.triggerLabelQuick }}" ]]; then
           echo "::set-output name=requested_tests::auto"
         fi
+        echo ${{ toJson(github) }}          
     ### Add the in-progress label and remove any previous success/fail labels.
     - name: add in-progress label
       uses: buildsville/add-remove-label@v1
@@ -366,7 +367,7 @@ jobs:
         if: ${{ !cancelled() }}
         with:
           name: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}
-          path: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
+          path: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}-.txt
 
       ### The below allow us to set the failure label and comment early, when the first failure
       ### in the matrix occurs. It'll be cleaned up in a subsequent job.
diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index 1a278e4f4a..f4e1b7cfc2 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -153,6 +153,7 @@ def main(argv):
     # Certain spaces are replaced with HTML non-breaking spaces, to prevent
     # aggressive word-wrapping from messing up the formatting.
     space_char = "&nbsp;"
+    list_seperator = "<br/>"
   else:
     # For text formatting, see how wide the strings are so we can
     # justify the text table.
@@ -160,8 +161,9 @@ def main(argv):
     max_build_failures = len(BUILD_FAILURES_HEADER)
     max_test_failures = len(TEST_FAILURES_HEADER)
     for (platform, results) in log_results.items():
-      build_failures = ", ".join(sorted(log_results[platform]["build_failures"]))
-      test_failures = ", ".join(sorted(log_results[platform]["test_failures"]))
+      list_seperator = ", "
+      build_failures = list_seperator.join(sorted(log_results[platform]["build_failures"]))
+      test_failures = list_seperator.join(sorted(log_results[platform]["test_failures"]))
       max_platform = max(max_platform, len(platform))
       max_build_failures = max(max_build_failures, len(build_failures))
       max_test_failures = max(max_test_failures, len(test_failures))
@@ -181,8 +183,8 @@ def main(argv):
   for platform in sorted(log_results.keys()):
     if log_results[platform]["build_failures"] or log_results[platform]["test_failures"]:
       platform_str = re.sub(r'\b \b', space_char, platform.ljust(max_platform))
-      build_failures = ", ".join(sorted(log_results[platform]["build_failures"])).ljust(max_build_failures)
-      test_failures = ", ".join(sorted(log_results[platform]["test_failures"])).ljust(max_test_failures)
+      build_failures = list_seperator.join(sorted(log_results[platform]["build_failures"])).ljust(max_build_failures)
+      test_failures = list_seperator.join(sorted(log_results[platform]["test_failures"])).ljust(max_test_failures)
       if FLAGS.markdown:
         # If there are more than N failures, collapse the results.
         if FLAGS.list_max and len(log_results[platform]["build_failures"]) > FLAGS.list_max:
@@ -191,9 +193,6 @@ def main(argv):
         if FLAGS.list_max and len(log_results[platform]["test_failures"]) > FLAGS.list_max:
           test_failures = "<details><summary>_(%s items)_</summary>%s</details>" % (
             len(log_results[platform]["test_failures"]), test_failures)
-      else:
-        build_failures = ", ".join(sorted(log_results[platform]["build_failures"])).ljust(max_build_failures)
-        test_failures = ", ".join(sorted(log_results[platform]["test_failures"])).ljust(max_test_failures)
       output_lines.append("| %s | %s | %s |" % (platform_str, build_failures, test_failures))
 
   print("\n".join(output_lines))

From bc080b2713763657da258b442551b762975e29e7 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 15:57:44 -0700
Subject: [PATCH 15/32] Remove debug info.

---
 .github/workflows/integration_tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 80bd0cabf9..41d4014af6 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -99,7 +99,6 @@ jobs:
         elif [[ "${{ github.event.label.name }}" == "${{ env.triggerLabelQuick }}" ]]; then
           echo "::set-output name=requested_tests::auto"
         fi
-        echo ${{ toJson(github) }}          
     ### Add the in-progress label and remove any previous success/fail labels.
     - name: add in-progress label
       uses: buildsville/add-remove-label@v1

From 822e62eded699f1e31566014b7bec866b1ccabe6 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 16:04:07 -0700
Subject: [PATCH 16/32] Try printing job context.

---
 .github/workflows/integration_tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 41d4014af6..4b3e9fb194 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -99,6 +99,7 @@ jobs:
         elif [[ "${{ github.event.label.name }}" == "${{ env.triggerLabelQuick }}" ]]; then
           echo "::set-output name=requested_tests::auto"
         fi
+        echo '${{ toJson(jobs[github.job]) }}'
     ### Add the in-progress label and remove any previous success/fail labels.
     - name: add in-progress label
       uses: buildsville/add-remove-label@v1

From 4e32fd4ad0747cd3afb424667aab8930c1e89aa8 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 16:06:07 -0700
Subject: [PATCH 17/32] Fix syntax.

---
 .github/workflows/integration_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 4b3e9fb194..83c2a9ecaf 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -99,7 +99,7 @@ jobs:
         elif [[ "${{ github.event.label.name }}" == "${{ env.triggerLabelQuick }}" ]]; then
           echo "::set-output name=requested_tests::auto"
         fi
-        echo '${{ toJson(jobs[github.job]) }}'
+        echo '${{ toJson(job) }}'
     ### Add the in-progress label and remove any previous success/fail labels.
     - name: add in-progress label
       uses: buildsville/add-remove-label@v1

From 8e938284b950ba3ec3a2c6e24e95a8858a9aa3c7 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 16:09:38 -0700
Subject: [PATCH 18/32] Remove extra context debugging.

---
 .github/workflows/integration_tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 83c2a9ecaf..41d4014af6 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -99,7 +99,6 @@ jobs:
         elif [[ "${{ github.event.label.name }}" == "${{ env.triggerLabelQuick }}" ]]; then
           echo "::set-output name=requested_tests::auto"
         fi
-        echo '${{ toJson(job) }}'
     ### Add the in-progress label and remove any previous success/fail labels.
     - name: add in-progress label
       uses: buildsville/add-remove-label@v1

From 9ae4ad461379032b17a20cff67ba7bdb78dad6b0 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 16:11:40 -0700
Subject: [PATCH 19/32] Handle the case where there are no errors - print
 nothing.

---
 scripts/gha/summarize_test_results.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index f4e1b7cfc2..b000c3fae9 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -81,6 +81,7 @@ def main(argv):
   log_name_re = re.escape(
       os.path.join(FLAGS.dir,FLAGS.pattern)).replace("\\*", "(.*)")
 
+  any_failures = False
   log_data = {}
 
   for log_file in log_files:
@@ -121,6 +122,8 @@ def main(argv):
             product_name = m2.group(1).lower()
             if product_name:
               log_results[platform]["build_failures"].add(product_name)
+              any_failures = True
+
       # Extract test failures, which follow "TESTAPPS EXPERIENCED ERRORS:"
       m = re.search(r'TESTAPPS EXPERIENCED ERRORS:\n(([^\n]*\n)+)', log_text, re.MULTILINE)
       if m:
@@ -139,12 +142,17 @@ def main(argv):
               product_name = m2.group(2).lower()
               if product_name:
                 log_results[platform]["test_failures"].add(product_name)
+                any_failures = True
 
   for platform in log_results.keys():
     log_results[platform]["successful"] = log_results[platform]["attempted"].difference(
       log_results[platform]["test_failures"].union(
         log_results[platform]["build_failures"]))
-  
+
+  if not any_failures:
+    # No failures occurred, nothing to log.
+    return(0)
+
   if FLAGS.markdown:
     # If outputting Markdown, don't bother justifying the table.
     max_platform = 0
@@ -160,6 +168,7 @@ def main(argv):
     max_platform = len(PLATFORM_HEADER)
     max_build_failures = len(BUILD_FAILURES_HEADER)
     max_test_failures = len(TEST_FAILURES_HEADER)
+    space_char = " "
     for (platform, results) in log_results.items():
       list_seperator = ", "
       build_failures = list_seperator.join(sorted(log_results[platform]["build_failures"]))
@@ -167,7 +176,6 @@ def main(argv):
       max_platform = max(max_platform, len(platform))
       max_build_failures = max(max_build_failures, len(build_failures))
       max_test_failures = max(max_test_failures, len(test_failures))
-      space_char = " "
 
   # Output a table (text or markdown) of failure platforms & tests.
   output_lines = list()

From 20b92f24de0c1f92de7aa6096c41aa8aa16c8bdb Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 16:29:30 -0700
Subject: [PATCH 20/32] Fix output logs, and format the platform names better.

---
 .github/workflows/integration_tests.yml |  2 +-
 scripts/gha/summarize_test_results.py   | 16 +++++++++++++---
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 41d4014af6..257d95a0ff 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -366,7 +366,7 @@ jobs:
         if: ${{ !cancelled() }}
         with:
           name: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}
-          path: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}-.txt
+          path: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
 
       ### The below allow us to set the failure label and comment early, when the first failure
       ### in the matrix occurs. It'll be cleaned up in a subsequent job.
diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index b000c3fae9..5a428e3de6 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -58,12 +58,13 @@
 
 CAPITALIZATIONS = {
     "macos": "MacOS",
-    "ubuntu": "Ubuntu",
+    "ubuntu": "Linux",
     "windows": "Windows",
     "openssl": "(OpenSSL)",
     "boringssl": "(BoringSSL)",
     "ios": "iOS",
     "android": "Android",
+    "desktop": "Desktop",
 }
 
 PLATFORM_HEADER = "Platform"
@@ -90,17 +91,26 @@ def main(argv):
       # Split the matrix name into components.
       log_name = re.sub(r'[-_.]+', ' ', log_name).split()
       # Remove redundant components.
+      if "latest" in log_name: log_name.remove("latest")
       if "Android" in log_name or "iOS" in log_name:
           log_name.remove('openssl')
-      log_name.remove('latest')
       # Capitalize components in a nice way.
       log_name = [
           CAPITALIZATIONS[name.lower()]
           if name.lower() in CAPITALIZATIONS
           else name
           for name in log_name]
+      if FLAGS.markdown:
+        if "Android" in log_name or "iOS" in log_name:
+          # For Android and iOS, highlight the target OS.
+            log_name[0] = "(built on %s)" % log_name[0]
+            log_name[1] = "**%s**" % log_name[1]
+        else:
+          # For desktop, highlight the entire platform string.
+          log_name[0] = "%s**" % log_name[0]
+          log_name[1] = "**%s" % log_name[1]
       # Rejoin matrix name with spaces.
-      log_name = ' '.join(log_name)
+      log_name = ' '.join([log_name[1], log_name[0]]+log_name[2:])
       with open(log_file, "r") as log_reader:
           log_data[log_name] = log_reader.read()
 

From a265bd0a05843d33c9a31db0394c6b6b4198f1aa Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 16:33:22 -0700
Subject: [PATCH 21/32] Clean up non-Markdown results.

---
 scripts/gha/summarize_test_results.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index 5a428e3de6..2cda9a8914 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -100,15 +100,15 @@ def main(argv):
           if name.lower() in CAPITALIZATIONS
           else name
           for name in log_name]
-      if FLAGS.markdown:
-        if "Android" in log_name or "iOS" in log_name:
-          # For Android and iOS, highlight the target OS.
-            log_name[0] = "(built on %s)" % log_name[0]
-            log_name[1] = "**%s**" % log_name[1]
-        else:
-          # For desktop, highlight the entire platform string.
-          log_name[0] = "%s**" % log_name[0]
-          log_name[1] = "**%s" % log_name[1]
+      if "Android" in log_name or "iOS" in log_name:
+        # For Android and iOS, highlight the target OS.
+        log_name[0] = "(built on %s)" % log_name[0]
+        if FLAGS.markdown:
+          log_name[1] = "**%s**" % log_name[1]
+      elif FLAGS.markdown:
+        # For desktop, highlight the entire platform string.
+        log_name[0] = "%s**" % log_name[0]
+        log_name[1] = "**%s" % log_name[1]
       # Rejoin matrix name with spaces.
       log_name = ' '.join([log_name[1], log_name[0]]+log_name[2:])
       with open(log_file, "r") as log_reader:

From 5d6ae1acee52b6648e047c4d62f55ce6dc985d4b Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 16:43:28 -0700
Subject: [PATCH 22/32] Add an overall summary step at the end, which logs to
 GitHub log.

---
 .github/workflows/integration_tests.yml | 14 ++++++++++++++
 scripts/gha/summarize_test_results.py   | 11 ++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 257d95a0ff..eadd186186 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -516,3 +516,17 @@ jobs:
           label: "${{ env.statusLabelInProgress }}"
           type: remove
 
+  summarize_results:
+    name: "summarize results"
+    needs: [tests]
+    runs-on: ubuntu-latest
+    if: ${{ !cancelled() }}
+    steps:
+      - name: download artifact
+        uses: actions/download-artifact@v2.0.8
+        with:
+          path: test_results
+      - name: Summarize results into GitHub log.
+        run: |
+          mv test_results/test-results-*/test-results-*.txt test_results || true
+          python scripts/gha/summarize_test_results.py --dir test_results --github_log
diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index 2cda9a8914..f98ad97f51 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -52,6 +52,10 @@
     "markdown", False,
     "Display a Markdown-formatted table.")
 
+flags.DEFINE_bool(
+    "github_log", False,
+    "Display a GitHub log formatted table.")
+
 flags.DEFINE_integer(
     "list_max", 5,
     "In Markdown mode, collapse lists larger than this size. 0 to disable.")
@@ -71,6 +75,8 @@
 BUILD_FAILURES_HEADER = "Build failures"
 TEST_FAILURES_HEADER = "Test failures"
 
+LOG_HEADER = "INTEGRATION TEST FAILURES"
+
 def main(argv):
   if len(argv) > 1:
       raise app.UsageError("Too many command-line arguments.")
@@ -213,7 +219,10 @@ def main(argv):
             len(log_results[platform]["test_failures"]), test_failures)
       output_lines.append("| %s | %s | %s |" % (platform_str, build_failures, test_failures))
 
-  print("\n".join(output_lines))
+  if FLAGS.github_log:
+    print("::error ::%s%%0A%s%%0A%%0A%s" % (LOG_HEADER, "-".ljust(len(LOG_HEADER), "-"), "%0A".join(output_lines)))
+  else:
+    print("\n".join(output_lines))
 
 if __name__ == "__main__":
   flags.mark_flag_as_required("dir")

From 66b3f724f7b8771fd846559b1a264b265bad16b0 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 16:46:26 -0700
Subject: [PATCH 23/32] Add comment.

---
 scripts/gha/summarize_test_results.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index f98ad97f51..205a5a9c36 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -220,6 +220,7 @@ def main(argv):
       output_lines.append("| %s | %s | %s |" % (platform_str, build_failures, test_failures))
 
   if FLAGS.github_log:
+    # "%0A" produces a newline in GitHub workflow logs.
     print("::error ::%s%%0A%s%%0A%%0A%s" % (LOG_HEADER, "-".ljust(len(LOG_HEADER), "-"), "%0A".join(output_lines)))
   else:
     print("\n".join(output_lines))

From a524f2f51b79e6b37356568cb296dd9eb4980c8c Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 19:08:21 -0700
Subject: [PATCH 24/32] Fix some small issues with test summary logging.

---
 .github/workflows/integration_tests.yml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index eadd186186..37a0ebffc5 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -359,15 +359,16 @@ jobs:
         if: matrix.target_platform != 'Desktop' && !cancelled()
         run: |
           python scripts/gha/test_lab.py --android_model ${{ needs.prepare_matrix.outputs.android_device }} --android_api ${{ needs.prepare_matrix.outputs.android_api }} --ios_model ${{ needs.prepare_matrix.outputs.ios_device }} --ios_version ${{ needs.prepare_matrix.outputs.ios_version }} --testapp_dir ta --code_platform cpp --key_file scripts/gha-encrypted/gcs_key_file.json
+      - name: Prepare results summary artifact
+        if: !cancelled()
+        run: |
           cp ta/summary.log test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
-
-      - name: upload results summary artifact
+      - name: Upload results summary artifact
         uses: actions/upload-artifact@v2.2.2
         if: ${{ !cancelled() }}
         with:
           name: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}
           path: test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
-
       ### The below allow us to set the failure label and comment early, when the first failure
       ### in the matrix occurs. It'll be cleaned up in a subsequent job.
       - name: add failure label
@@ -470,6 +471,7 @@ jobs:
         run: |
           echo -n "::set-output name=time::"
           TZ=America/Los_Angeles date
+      - uses: actions/checkout@v2
       - name: download artifact
         uses: actions/download-artifact@v2.0.8
         with:
@@ -518,10 +520,11 @@ jobs:
 
   summarize_results:
     name: "summarize results"
-    needs: [tests]
+    needs: [add_failure_label, tests]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }}
     steps:
+      - uses: actions/checkout@v2
       - name: download artifact
         uses: actions/download-artifact@v2.0.8
         with:

From b6acded71d17e8fc9045139086a140d833ac93b4 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 19:10:55 -0700
Subject: [PATCH 25/32] Fix workflow issue

---
 .github/workflows/integration_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 37a0ebffc5..213cf55226 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -360,7 +360,7 @@ jobs:
         run: |
           python scripts/gha/test_lab.py --android_model ${{ needs.prepare_matrix.outputs.android_device }} --android_api ${{ needs.prepare_matrix.outputs.android_api }} --ios_model ${{ needs.prepare_matrix.outputs.ios_device }} --ios_version ${{ needs.prepare_matrix.outputs.ios_version }} --testapp_dir ta --code_platform cpp --key_file scripts/gha-encrypted/gcs_key_file.json
       - name: Prepare results summary artifact
-        if: !cancelled()
+        if: ${{ !cancelled() }}
         run: |
           cp ta/summary.log test-results-${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}.txt
       - name: Upload results summary artifact

From 6082841c0351378b1a80c8dea54511a71d48d3d9 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 19:23:36 -0700
Subject: [PATCH 26/32] Add extra deps.

---
 .github/workflows/integration_tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 213cf55226..3b0f30f791 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -519,8 +519,8 @@ jobs:
           type: remove
 
   summarize_results:
-    name: "summarize results"
-    needs: [add_failure_label, tests]
+    name: "summarize-results"
+    needs: [add_failure_label, add_success_label, remove_in_progress_label, tests]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }}
     steps:
@@ -529,7 +529,7 @@ jobs:
         uses: actions/download-artifact@v2.0.8
         with:
           path: test_results
-      - name: Summarize results into GitHub log.
+      - name: Summarize results into GitHub log
         run: |
           mv test_results/test-results-*/test-results-*.txt test_results || true
           python scripts/gha/summarize_test_results.py --dir test_results --github_log

From f6991063f87f25e0148fddb23289cc7f642a8bb3 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 20:43:40 -0700
Subject: [PATCH 27/32] Set up absl-py in the summarizing jobs.

---
 .github/workflows/integration_tests.yml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 3b0f30f791..a46ba14771 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -472,6 +472,15 @@ jobs:
           echo -n "::set-output name=time::"
           TZ=America/Los_Angeles date
       - uses: actions/checkout@v2
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+
+      - name: Install python deps
+        run: |
+          python scripts/gha/install_prereqs_desktop.py
+
       - name: download artifact
         uses: actions/download-artifact@v2.0.8
         with:
@@ -525,6 +534,15 @@ jobs:
     if: ${{ !cancelled() }}
     steps:
       - uses: actions/checkout@v2
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+
+      - name: Install python deps
+        run: |
+          python scripts/gha/install_prereqs_desktop.py
+
       - name: download artifact
         uses: actions/download-artifact@v2.0.8
         with:

From 897645a63404db149d59dbcfa2a4af3d580f7fb6 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Thu, 25 Mar 2021 21:07:41 -0700
Subject: [PATCH 28/32] Add matching for Android test failure.

---
 scripts/gha/summarize_test_results.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index 205a5a9c36..f2ad2b6e0b 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -141,9 +141,9 @@ def main(argv):
               any_failures = True
 
       # Extract test failures, which follow "TESTAPPS EXPERIENCED ERRORS:"
-      m = re.search(r'TESTAPPS EXPERIENCED ERRORS:\n(([^\n]*\n)+)', log_text, re.MULTILINE)
+      m = re.search(r'TESTAPPS (EXPERIENCED ERRORS|FAILED):\n(([^\n]*\n)+)', log_text, re.MULTILINE)
       if m:
-        for test_failure_line in m.group(1).strip("\n").split("\n"):
+        for test_failure_line in m.group(2).strip("\n").split("\n"):
           # Only get the lines showing paths.
           if "/firebase-cpp-sdk/" not in test_failure_line: continue
           test_filename = "";
@@ -151,6 +151,11 @@ def main(argv):
             test_filename = re.match(r'^(.*) log tail', test_failure_line).group(1)
           if "lacks logs" in test_failure_line:
             test_filename = re.match(r'^(.*) lacks logs', test_failure_line).group(1)
+          if "it-debug.apk" in test_failure_line:
+            test_filename = re.match(r'^(.*it-debug\.apk)', test_failure_line).group(1)
+          if "integration_test.ipa" in test_failure_line:
+            test_filename = re.match(r'^(.*integration_test\.ipa)', test_failure_line).group(1)
+
           if test_filename:
             m2 = re.search(r'/ta/(firebase)?([^/]+)/iti?/', test_filename, re.IGNORECASE)
             if not m2: m2 = re.search(r'/testapps/(firebase)?([^/]+)/integration_test', test_filename, re.IGNORECASE)

From d525cc51abdbdbc445c53889ad93fffc5a4b8a49 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Fri, 26 Mar 2021 00:01:04 -0700
Subject: [PATCH 29/32] Clean up text-log table.

---
 scripts/gha/summarize_test_results.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index f2ad2b6e0b..7e310b321a 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -226,7 +226,8 @@ def main(argv):
 
   if FLAGS.github_log:
     # "%0A" produces a newline in GitHub workflow logs.
-    print("::error ::%s%%0A%s%%0A%%0A%s" % (LOG_HEADER, "-".ljust(len(LOG_HEADER), "-"), "%0A".join(output_lines)))
+    output_lines = [output_lines[1]] + output_lines + [output_lines[1]]
+    print("::error ::%s%%0A%%0A%s" % (LOG_HEADER, "%0A".join(output_lines).replace(" ", "&nbsp;")))
   else:
     print("\n".join(output_lines))
 

From 26ecd7758276bad44459cd6ab535cade023ca08f Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Fri, 26 Mar 2021 13:56:18 -0700
Subject: [PATCH 30/32] Refactor summarize test results script.

---
 scripts/gha/summarize_test_results.py | 261 +++++++++++++++++---------
 1 file changed, 169 insertions(+), 92 deletions(-)

diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index 7e310b321a..e16c8f80ca 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -45,8 +45,8 @@
     "File pattern (glob) for test results.")
 
 flags.DEFINE_bool(
-    "full", False,
-    "Print a full table, including successful tests.")
+    "include_successful", False,
+    "Print all logs including successful tests.")
 
 flags.DEFINE_bool(
     "markdown", False,
@@ -54,7 +54,11 @@
 
 flags.DEFINE_bool(
     "github_log", False,
-    "Display a GitHub log formatted table.")
+    "Display a GitHub log list.")
+
+flags.DEFINE_bool(
+    "text_log", False,
+    "Display a text log list.")
 
 flags.DEFINE_integer(
     "list_max", 5,
@@ -74,9 +78,124 @@
 PLATFORM_HEADER = "Platform"
 BUILD_FAILURES_HEADER = "Build failures"
 TEST_FAILURES_HEADER = "Test failures"
+SUCCESSFUL_TESTS_HEADER = "Successful tests"
 
 LOG_HEADER = "INTEGRATION TEST FAILURES"
 
+# Default list separator for printing in text format.
+DEFAULT_LIST_SEPARATOR=", "
+
+def print_table(log_results,
+            platform_width = 0,
+            build_failures_width = 0,
+            test_failures_width = 0,
+            successful_width = 0,
+            space_char = " ",
+            list_separator = DEFAULT_LIST_SEPARATOR):
+  """Print out a table in the specified format."""
+  # Print table header
+  output_lines = list()
+  headers = [
+    re.sub(r'\b \b', space_char, PLATFORM_HEADER.ljust(platform_width)),
+    re.sub(r'\b \b', space_char,BUILD_FAILURES_HEADER.ljust(build_failures_width)),
+    re.sub(r'\b \b', space_char,TEST_FAILURES_HEADER.ljust(test_failures_width))
+  ] + (
+    [re.sub(r'\b \b', space_char,SUCCESSFUL_TESTS_HEADER.ljust(successful_width))]
+    if FLAGS.include_successful else []
+  )
+  # Print header line.
+  output_lines.append(("|" + " %s |" * len(headers)) % tuple(headers))
+  # Print a |-------|-------|---------| line.
+  output_lines.append(("|" + "-%s-|" * len(headers)) % 
+                      tuple([ re.sub("[^|]","-", header) for header in headers ]))
+
+  # Iterate through platforms and print out table lines.
+  for platform in sorted(log_results.keys()):
+    if log_results[platform]["build_failures"] or log_results[platform]["test_failures"] or FLAGS.include_successful:
+      columns = [
+        re.sub(r'\b \b', space_char, platform.ljust(platform_width)),
+        format_result(log_results[platform]["build_failures"], justify=build_failures_width, list_separator=list_separator),
+        format_result(log_results[platform]["test_failures"], justify=test_failures_width, list_separator=list_separator),
+      ] + (
+        [format_result(log_results[platform]["successful"], justify=successful_width, list_separator=list_separator)]
+        if FLAGS.include_successful else []
+      )
+      output_lines.append(("|" + " %s |" * len(headers)) % tuple(columns))
+
+  return output_lines
+
+
+def format_result(test_set, list_separator=DEFAULT_LIST_SEPARATOR, justify=0):
+  """Format a list of tests."""
+  list_output = list_separator.join(sorted(test_set))
+  if FLAGS.markdown and FLAGS.list_max > 0 and len(test_set) > FLAGS.list_max:
+      return "<details><summary>_(%s items)_</summary>%s</details>" % (
+        len(test_set), list_output)
+  else:
+    return list_output.ljust(justify)
+
+
+def print_text_table(log_results):
+  """Print out a nicely-formatted text table."""
+  # For text formatting, see how wide the strings are so we can
+  # justify the text table.
+  max_platform = len(PLATFORM_HEADER)
+  max_build_failures = len(BUILD_FAILURES_HEADER)
+  max_test_failures = len(TEST_FAILURES_HEADER)
+  max_sucessful = len(SUCCESSFUL_TESTS_HEADER)
+  for (platform, results) in log_results.items():
+    max_platform = max(max_platform, len(platform))
+    max_build_failures = max(max_build_failures,
+                             len(format_result(log_results[platform]["build_failures"])))
+    max_test_failures = max(max_test_failures,
+                            len(format_result(log_results[platform]["test_failures"])))
+    max_sucessful = max(max_sucessful,
+                        len(format_result(log_results[platform]["successful"])))
+  return print_table(log_results,
+                     platform_width=max_platform,
+                     build_failures_width=max_build_failures,
+                     test_failures_width=max_test_failures,
+                     successful_width=max_sucessful)
+
+
+def print_log(log_results):
+  """Print the results in a text-only log format."""
+  output_lines = []
+  for platform in sorted(log_results.keys()):
+    if log_results[platform]["build_failures"] or log_results[platform]["test_failures"] or FLAGS.include_successful:
+      output_lines.append("")
+      output_lines.append("%s:" % platform)
+      if (FLAGS.include_successful and len(log_results[platform]["successful"]) > 0):
+        output_lines.append("  Successful tests (%d):" %
+                            len(log_results[platform]["successful"]))
+        for test_name in sorted(log_results[platform]["successful"]):
+          output_lines.append("  - %s" % test_name)
+      if (len(log_results[platform]["build_failures"]) > 0):
+        output_lines.append("  Build failures (%d):" %
+                            len(log_results[platform]["build_failures"]))
+        for test_name in sorted(log_results[platform]["build_failures"]):
+          output_lines.append("  - %s" % test_name)
+      if (len(log_results[platform]["test_failures"]) > 0):
+        output_lines.append("  Test failures (%d):" %
+                            len(log_results[platform]["test_failures"]))
+        for test_name in sorted(log_results[platform]["test_failures"]):
+          output_lines.append("  - %s" % test_name)
+  return output_lines[1:]  # skip first blank line
+
+
+def print_github_log(log_results):
+  output_lines = [LOG_HEADER, ""] + print_log(log_results)
+  # "%0A" produces a newline in GitHub workflow logs.
+  return ["::error ::%s" % "%0A".join(output_lines)]
+
+
+def print_markdown_table(log_results):
+  # Print a normal table, but with a few changes:
+  # Separate test names by newlines, and replace certain spaces
+  # with HTML non-breaking spaces to prevent aggressive word-wrapping.
+  return print_table(log_results, space_char = "&nbsp;", list_separator = "<br/>")
+
+
 def main(argv):
   if len(argv) > 1:
       raise app.UsageError("Too many command-line arguments.")
@@ -123,113 +242,71 @@ def main(argv):
   log_results = {}
   # Go through each log and extract out the build and test failures.
   for (platform, log_text) in log_data.items():
+    if platform not in log_results:
       log_results[platform] = { "build_failures": set(), "test_failures": set(),
                                 "attempted": set(), "successful": set() }
-      # Get a full list of the products built.
-      m = re.search(r'TRIED TO BUILD: ([^\n]*)', log_text)
-      if m:
-        log_results[platform]["attempted"].update(m.group(1).split(","))
-      # Extract build failure lines, which follow "SOME FAILURES OCCURRED:"
-      m = re.search(r'SOME FAILURES OCCURRED:\n(([\d]+:[^\n]*\n)+)', log_text, re.MULTILINE)
-      if m:
-        for build_failure_line in m.group(1).strip("\n").split("\n"):
-          m2 = re.match(r'[\d]+: ([^,]+)', build_failure_line)
+    # Get a full list of the products built.
+    m = re.search(r'TRIED TO BUILD: ([^\n]*)', log_text)
+    if m:
+      log_results[platform]["attempted"].update(m.group(1).split(","))
+    # Extract build failure lines, which follow "SOME FAILURES OCCURRED:"
+    m = re.search(r'SOME FAILURES OCCURRED:\n(([\d]+:[^\n]*\n)+)', log_text, re.MULTILINE)
+    if m:
+      for build_failure_line in m.group(1).strip("\n").split("\n"):
+        m2 = re.match(r'[\d]+: ([^,]+)', build_failure_line)
+        if m2:
+          product_name = m2.group(1).lower()
+          if product_name:
+            log_results[platform]["build_failures"].add(product_name)
+            any_failures = True
+
+    # Extract test failures, which follow "TESTAPPS EXPERIENCED ERRORS:"
+    m = re.search(r'TESTAPPS (EXPERIENCED ERRORS|FAILED):\n(([^\n]*\n)+)', log_text, re.MULTILINE)
+    if m:
+      for test_failure_line in m.group(2).strip("\n").split("\n"):
+        # Only get the lines showing paths.
+        if "/firebase-cpp-sdk/" not in test_failure_line: continue
+        test_filename = "";
+        if "log tail" in test_failure_line:
+          test_filename = re.match(r'^(.*) log tail', test_failure_line).group(1)
+        if "lacks logs" in test_failure_line:
+          test_filename = re.match(r'^(.*) lacks logs', test_failure_line).group(1)
+        if "it-debug.apk" in test_failure_line:
+          test_filename = re.match(r'^(.*it-debug\.apk)', test_failure_line).group(1)
+        if "integration_test.ipa" in test_failure_line:
+          test_filename = re.match(r'^(.*integration_test\.ipa)', test_failure_line).group(1)
+
+        if test_filename:
+          m2 = re.search(r'/ta/(firebase)?([^/]+)/iti?/', test_filename, re.IGNORECASE)
+          if not m2: m2 = re.search(r'/testapps/(firebase)?([^/]+)/integration_test', test_filename, re.IGNORECASE)
           if m2:
-            product_name = m2.group(1).lower()
+            product_name = m2.group(2).lower()
             if product_name:
-              log_results[platform]["build_failures"].add(product_name)
+              log_results[platform]["test_failures"].add(product_name)
               any_failures = True
 
-      # Extract test failures, which follow "TESTAPPS EXPERIENCED ERRORS:"
-      m = re.search(r'TESTAPPS (EXPERIENCED ERRORS|FAILED):\n(([^\n]*\n)+)', log_text, re.MULTILINE)
-      if m:
-        for test_failure_line in m.group(2).strip("\n").split("\n"):
-          # Only get the lines showing paths.
-          if "/firebase-cpp-sdk/" not in test_failure_line: continue
-          test_filename = "";
-          if "log tail" in test_failure_line:
-            test_filename = re.match(r'^(.*) log tail', test_failure_line).group(1)
-          if "lacks logs" in test_failure_line:
-            test_filename = re.match(r'^(.*) lacks logs', test_failure_line).group(1)
-          if "it-debug.apk" in test_failure_line:
-            test_filename = re.match(r'^(.*it-debug\.apk)', test_failure_line).group(1)
-          if "integration_test.ipa" in test_failure_line:
-            test_filename = re.match(r'^(.*integration_test\.ipa)', test_failure_line).group(1)
-
-          if test_filename:
-            m2 = re.search(r'/ta/(firebase)?([^/]+)/iti?/', test_filename, re.IGNORECASE)
-            if not m2: m2 = re.search(r'/testapps/(firebase)?([^/]+)/integration_test', test_filename, re.IGNORECASE)
-            if m2:
-              product_name = m2.group(2).lower()
-              if product_name:
-                log_results[platform]["test_failures"].add(product_name)
-                any_failures = True
-
+  # After processing all the logs, we can determine the successful builds for each platform.
   for platform in log_results.keys():
     log_results[platform]["successful"] = log_results[platform]["attempted"].difference(
       log_results[platform]["test_failures"].union(
         log_results[platform]["build_failures"]))
 
-  if not any_failures:
+  if not any_failures and not FLAGS.include_successful:
     # No failures occurred, nothing to log.
     return(0)
 
+  log_lines = []
   if FLAGS.markdown:
+    log_lines = print_markdown_table(log_results)
     # If outputting Markdown, don't bother justifying the table.
-    max_platform = 0
-    max_build_failures = 0
-    max_test_failures = 0
-    # Certain spaces are replaced with HTML non-breaking spaces, to prevent
-    # aggressive word-wrapping from messing up the formatting.
-    space_char = "&nbsp;"
-    list_seperator = "<br/>"
+  elif FLAGS.github_log:
+    log_lines = print_github_log(log_results)
+  elif FLAGS.text_log:
+    log_lines = print_log(log_results)
   else:
-    # For text formatting, see how wide the strings are so we can
-    # justify the text table.
-    max_platform = len(PLATFORM_HEADER)
-    max_build_failures = len(BUILD_FAILURES_HEADER)
-    max_test_failures = len(TEST_FAILURES_HEADER)
-    space_char = " "
-    for (platform, results) in log_results.items():
-      list_seperator = ", "
-      build_failures = list_seperator.join(sorted(log_results[platform]["build_failures"]))
-      test_failures = list_seperator.join(sorted(log_results[platform]["test_failures"]))
-      max_platform = max(max_platform, len(platform))
-      max_build_failures = max(max_build_failures, len(build_failures))
-      max_test_failures = max(max_test_failures, len(test_failures))
-
-  # Output a table (text or markdown) of failure platforms & tests.
-  output_lines = list()
-  output_lines.append("| %s | %s | %s |" % (
-    re.sub(r'\b \b', space_char, PLATFORM_HEADER.ljust(max_platform)),
-    re.sub(r'\b \b', space_char,BUILD_FAILURES_HEADER.ljust(max_build_failures)),
-    re.sub(r'\b \b', space_char,TEST_FAILURES_HEADER.ljust(max_test_failures))))
-  output_lines.append("|-%s-|-%s-|-%s-|" % (
-    "".ljust(max_platform, "-"),
-    "".ljust(max_build_failures, "-"),
-    "".ljust(max_test_failures, "-")))
+    log_lines = print_text_table(log_results)
 
-  for platform in sorted(log_results.keys()):
-    if log_results[platform]["build_failures"] or log_results[platform]["test_failures"]:
-      platform_str = re.sub(r'\b \b', space_char, platform.ljust(max_platform))
-      build_failures = list_seperator.join(sorted(log_results[platform]["build_failures"])).ljust(max_build_failures)
-      test_failures = list_seperator.join(sorted(log_results[platform]["test_failures"])).ljust(max_test_failures)
-      if FLAGS.markdown:
-        # If there are more than N failures, collapse the results.
-        if FLAGS.list_max and len(log_results[platform]["build_failures"]) > FLAGS.list_max:
-          build_failures = "<details><summary>_(%s items)_</summary>%s</details>" % (
-            len(log_results[platform]["build_failures"]), build_failures)
-        if FLAGS.list_max and len(log_results[platform]["test_failures"]) > FLAGS.list_max:
-          test_failures = "<details><summary>_(%s items)_</summary>%s</details>" % (
-            len(log_results[platform]["test_failures"]), test_failures)
-      output_lines.append("| %s | %s | %s |" % (platform_str, build_failures, test_failures))
-
-  if FLAGS.github_log:
-    # "%0A" produces a newline in GitHub workflow logs.
-    output_lines = [output_lines[1]] + output_lines + [output_lines[1]]
-    print("::error ::%s%%0A%%0A%s" % (LOG_HEADER, "%0A".join(output_lines).replace(" ", "&nbsp;")))
-  else:
-    print("\n".join(output_lines))
+  print("\n".join(log_lines))
 
 if __name__ == "__main__":
   flags.mark_flag_as_required("dir")

From 2b423945107dfce438aabcab0968227f16f2d3b4 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Fri, 26 Mar 2021 16:44:21 -0700
Subject: [PATCH 31/32] Delete artifacts at end of test run.

---
 .github/workflows/integration_tests.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index a46ba14771..d4e2e060b2 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -538,11 +538,9 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: '3.7'
-
       - name: Install python deps
         run: |
           python scripts/gha/install_prereqs_desktop.py
-
       - name: download artifact
         uses: actions/download-artifact@v2.0.8
         with:
@@ -551,3 +549,10 @@ jobs:
         run: |
           mv test_results/test-results-*/test-results-*.txt test_results || true
           python scripts/gha/summarize_test_results.py --dir test_results --github_log
+      - uses: geekyeggo/delete-artifact@1-glob-support
+        # Delete all of the test result artifacts.
+        with:
+          name: |
+            test-results-*
+          failOnError: false
+          useGlob: true

From 4f88c675bb4c3619f6ee56c353fa1b3fe5796ffa Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Mon, 29 Mar 2021 10:30:18 -0700
Subject: [PATCH 32/32] Improve python script comments and documentation.

---
 scripts/gha/summarize_test_results.py | 46 +++++++++++++++++++--------
 1 file changed, 32 insertions(+), 14 deletions(-)

diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
index e16c8f80ca..8b4da87283 100644
--- a/scripts/gha/summarize_test_results.py
+++ b/scripts/gha/summarize_test_results.py
@@ -16,14 +16,28 @@
 
 USAGE:
 
-python summarize_test_results.py -d <directory>
+python summarize_test_results.py --dir <directory> [--markdown]
 
-Example output:
+Example table mode output (will be slightly different with --markdown):
 
-| Platform                | Build failures | Test failures   |
-| ----------------------- | -------------- | --------------- |
-| MacOS iOS               |                | auth, firestore |
-| Windows Desktop OpenSSL | analytics      | database        |
+| Platform                  | Build failures | Test failures   |
+|---------------------------|----------------|-----------------|
+| iOS (build on iOS)        |                | auth, firestore |
+| Desktop Windows (OpenSSL) | analytics      | database        |
+
+python summarize_test_results.py --dir <directory> <--text_log | --github_log>
+
+Example log mode output (will be slightly different with --github_log):
+
+INTEGRATION TEST FAILURES
+
+iOS (built on MacOS):
+  Test failures (2):
+  - auth
+  - firestore
+Desktop Windows (OpenSSL):
+  Build failures (1):
+  - analytics
 """
 
 from absl import app
@@ -42,7 +56,8 @@
 
 flags.DEFINE_string(
     "pattern", "test-results-*.txt",
-    "File pattern (glob) for test results.")
+    "File pattern (glob) for test results."
+    "The '*' part is used to determine the platform.")
 
 flags.DEFINE_bool(
     "include_successful", False,
@@ -92,7 +107,7 @@ def print_table(log_results,
             successful_width = 0,
             space_char = " ",
             list_separator = DEFAULT_LIST_SEPARATOR):
-  """Print out a table in the specified format."""
+  """Print out a table in the requested format (text or markdown)."""
   # Print table header
   output_lines = list()
   headers = [
@@ -106,7 +121,7 @@ def print_table(log_results,
   # Print header line.
   output_lines.append(("|" + " %s |" * len(headers)) % tuple(headers))
   # Print a |-------|-------|---------| line.
-  output_lines.append(("|" + "-%s-|" * len(headers)) % 
+  output_lines.append(("|" + "-%s-|" * len(headers)) %
                       tuple([ re.sub("[^|]","-", header) for header in headers ]))
 
   # Iterate through platforms and print out table lines.
@@ -126,7 +141,8 @@ def print_table(log_results,
 
 
 def format_result(test_set, list_separator=DEFAULT_LIST_SEPARATOR, justify=0):
-  """Format a list of tests."""
+  """Format a list of test names.
+  In Markdown mode, this can collapse a large list into a dropdown."""
   list_output = list_separator.join(sorted(test_set))
   if FLAGS.markdown and FLAGS.list_max > 0 and len(test_set) > FLAGS.list_max:
       return "<details><summary>_(%s items)_</summary>%s</details>" % (
@@ -138,7 +154,7 @@ def format_result(test_set, list_separator=DEFAULT_LIST_SEPARATOR, justify=0):
 def print_text_table(log_results):
   """Print out a nicely-formatted text table."""
   # For text formatting, see how wide the strings are so we can
-  # justify the text table.
+  # left-justify each column of the text table.
   max_platform = len(PLATFORM_HEADER)
   max_build_failures = len(BUILD_FAILURES_HEADER)
   max_test_failures = len(TEST_FAILURES_HEADER)
@@ -184,15 +200,17 @@ def print_log(log_results):
 
 
 def print_github_log(log_results):
+  """Print a text log, but replace newlines with %0A and add
+  the GitHub ::error text."""
   output_lines = [LOG_HEADER, ""] + print_log(log_results)
   # "%0A" produces a newline in GitHub workflow logs.
   return ["::error ::%s" % "%0A".join(output_lines)]
 
 
 def print_markdown_table(log_results):
-  # Print a normal table, but with a few changes:
-  # Separate test names by newlines, and replace certain spaces
-  # with HTML non-breaking spaces to prevent aggressive word-wrapping.
+  """Print a normal table, but with a few changes:
+  Separate test names by newlines, and replace certain spaces
+  with HTML non-breaking spaces to prevent aggressive word-wrapping."""
   return print_table(log_results, space_char = "&nbsp;", list_separator = "<br/>")