From effb38713b360df3e5d9bd1154042092e4efb882 Mon Sep 17 00:00:00 2001
From: Fabian Brandstetter <21087362+FaBrand@users.noreply.github.com>
Date: Thu, 27 May 2021 11:38:04 +0200
Subject: [PATCH] Create a failure test rule

When testing for analysis-phase-failures in rules this rule provides
otherwise necessary boilerplate that simply allows to test for correct
Error messages
---
 docs/BUILD                            |   6 +
 docs/analysis_failure_test_doc.md     |  54 ++++++++
 rules/BUILD                           |   6 +
 rules/analysis_failure_test.bzl       |  64 +++++++++
 tests/BUILD                           |  16 +++
 tests/analysis_failure_test_test.sh   | 188 ++++++++++++++++++++++++++
 tests/analysis_failure_test_tests.bzl |  45 ++++++
 7 files changed, 379 insertions(+)
 create mode 100644 docs/analysis_failure_test_doc.md
 create mode 100644 rules/analysis_failure_test.bzl
 create mode 100755 tests/analysis_failure_test_test.sh
 create mode 100644 tests/analysis_failure_test_tests.bzl

diff --git a/docs/BUILD b/docs/BUILD
index 4d31cd10..87949f3f 100644
--- a/docs/BUILD
+++ b/docs/BUILD
@@ -8,6 +8,12 @@ stardoc_with_diff_test(
     out_label = "//docs:analysis_test_doc.md",
 )
 
+stardoc_with_diff_test(
+    name = "analysis_failure_test",
+    bzl_library_target = "//rules:analysis_failure_test",
+    out_label = "//docs:analysis_failure_test_doc.md",
+)
+
 stardoc_with_diff_test(
     name = "build_test",
     bzl_library_target = "//rules:build_test",
diff --git a/docs/analysis_failure_test_doc.md b/docs/analysis_failure_test_doc.md
new file mode 100644
index 00000000..fe736ec9
--- /dev/null
+++ b/docs/analysis_failure_test_doc.md
@@ -0,0 +1,54 @@
+<!-- Generated with Stardoc: http://skydoc.bazel.build -->
+
+A test verifying that another target fails to analyse as part of a `bazel test`
+
+This analysistest is mostly aimed at rule authors that want to assert certain error conditions.
+If the target under test does not fail the analysis phase, the test will evaluate to FAILED.
+If the given error_message is not contained in the otherwise printed ERROR message, the test evaluates to FAILED.
+If the given error_message is contained in the otherwise printed ERROR message, the test evaluates to PASSED.
+
+NOTE:
+Adding the `manual` tag  to the target-under-test is recommended.
+It prevents analysis failure of that target if `bazel test //...` is used.
+
+Typical usage:
+```
+load("@bazel_skylib//rules:analysis_failure_test.bzl", "analysis_failure_test")
+
+rule_with_analysis_failure(
+    name = "unit",
+    tags = ["manual"],
+)
+
+
+analysis_failure_test(
+    name = "analysis_fails_with_error",
+    target_under_test = ":unit",
+    error_message = _EXPECTED_ERROR_MESSAGE,
+)
+```
+
+Args:
+  target_under_test: The target that is expected to cause an anlysis failure
+  error_message: The asserted error message in the (normally printed) ERROR.
+
+<a id="analysis_failure_test"></a>
+
+## analysis_failure_test
+
+<pre>
+analysis_failure_test(<a href="#analysis_failure_test-name">name</a>, <a href="#analysis_failure_test-error_message">error_message</a>, <a href="#analysis_failure_test-target_under_test">target_under_test</a>)
+</pre>
+
+
+
+**ATTRIBUTES**
+
+
+| Name  | Description | Type | Mandatory | Default |
+| :------------- | :------------- | :------------- | :------------- | :------------- |
+| <a id="analysis_failure_test-name"></a>name |  A unique name for this target.   | <a href="https://bazel.build/concepts/labels#target-names">Name</a> | required |  |
+| <a id="analysis_failure_test-error_message"></a>error_message |  The test asserts that the given string is contained in the error message of the target under test.   | String | required |  |
+| <a id="analysis_failure_test-target_under_test"></a>target_under_test |  -   | <a href="https://bazel.build/concepts/labels">Label</a> | required |  |
+
+
diff --git a/rules/BUILD b/rules/BUILD
index 3a06554c..166d2bc8 100644
--- a/rules/BUILD
+++ b/rules/BUILD
@@ -9,6 +9,12 @@ bzl_library(
     srcs = ["analysis_test.bzl"],
 )
 
+bzl_library(
+    name = "analysis_failure_test",
+    srcs = ["analysis_failure_test.bzl"],
+    deps = ["//lib:unittest"],
+)
+
 bzl_library(
     name = "build_test",
     srcs = ["build_test.bzl"],
diff --git a/rules/analysis_failure_test.bzl b/rules/analysis_failure_test.bzl
new file mode 100644
index 00000000..9690a7af
--- /dev/null
+++ b/rules/analysis_failure_test.bzl
@@ -0,0 +1,64 @@
+# Copyright 2021 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A test verifying that another target fails to analyse as part of a `bazel test`
+
+This analysistest is mostly aimed at rule authors that want to assert certain error conditions.
+If the target under test does not fail the analysis phase, the test will evaluate to FAILED.
+If the given error_message is not contained in the otherwise printed ERROR message, the test evaluates to FAILED.
+If the given error_message is contained in the otherwise printed ERROR message, the test evaluates to PASSED.
+
+NOTE:
+Adding the `manual` tag  to the target-under-test is recommended.
+It prevents analysis failure of that target if `bazel test //...` is used.
+
+Typical usage:
+```
+load("@bazel_skylib//rules:analysis_failure_test.bzl", "analysis_failure_test")
+
+rule_with_analysis_failure(
+    name = "unit",
+    tags = ["manual"],
+)
+
+
+analysis_failure_test(
+    name = "analysis_fails_with_error",
+    target_under_test = ":unit",
+    error_message = _EXPECTED_ERROR_MESSAGE,
+)
+```
+
+Args:
+  target_under_test: The target that is expected to cause an anlysis failure
+  error_message: The asserted error message in the (normally printed) ERROR."""
+
+load("//lib:unittest.bzl", "analysistest", "asserts")
+
+def _analysis_failure_test_impl(ctx):
+    """Implementation function for analysis_failure_test. """
+    env = analysistest.begin(ctx)
+    asserts.expect_failure(env, expected_failure_msg = ctx.attr.error_message)
+    return analysistest.end(env)
+
+analysis_failure_test = analysistest.make(
+    _analysis_failure_test_impl,
+    expect_failure = True,
+    attrs = {
+        "error_message": attr.string(
+            mandatory = True,
+            doc = "The test asserts that the given string is contained in the error message of the target under test.",
+        ),
+    },
+)
diff --git a/tests/BUILD b/tests/BUILD
index bbab077a..72e352c9 100644
--- a/tests/BUILD
+++ b/tests/BUILD
@@ -1,4 +1,5 @@
 load("//:bzl_library.bzl", "bzl_library")
+load(":analysis_failure_test_tests.bzl", "analysis_failure_test_test_suite")
 load(":build_test_tests.bzl", "build_test_test_suite")
 load(":collections_tests.bzl", "collections_test_suite")
 load(":dicts_tests.bzl", "dicts_test_suite")
@@ -46,6 +47,8 @@ unittest_passing_tests_suite()
 
 versions_test_suite()
 
+analysis_failure_test_test_suite()
+
 bzl_library(
     name = "unittest_tests_bzl",
     srcs = ["unittest_tests.bzl"],
@@ -81,6 +84,19 @@ sh_test(
     tags = ["local"],
 )
 
+sh_test(
+    name = "analysis_failure_test_e2e_test",
+    srcs = ["analysis_failure_test_test.sh"],
+    data = [
+        ":unittest.bash",
+        "//lib:unittest",
+        "//rules:analysis_failure_test.bzl",
+        "//toolchains/unittest:test_deps",
+        "@bazel_tools//tools/bash/runfiles",
+    ],
+    tags = ["local"],
+)
+
 sh_test(
     name = "common_settings_e2e_test",
     srcs = ["common_settings_test.sh"],
diff --git a/tests/analysis_failure_test_test.sh b/tests/analysis_failure_test_test.sh
new file mode 100755
index 00000000..b25a972f
--- /dev/null
+++ b/tests/analysis_failure_test_test.sh
@@ -0,0 +1,188 @@
+#!/bin/bash
+
+# Copyright 2021 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# End to end tests for analysis_failure_test.bzl.
+#
+# End to end tests of analysis_failure_test.bzl cover verification that
+# analysis_failure_test tests succeed when their underlying test targets fail analysis with
+# a given error message.
+
+# --- begin runfiles.bash initialization ---
+set -euo pipefail
+if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  if [[ -f "$0.runfiles_manifest" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
+  elif [[ -f "$0.runfiles/MANIFEST" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
+  elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+    export RUNFILES_DIR="$0.runfiles"
+  fi
+fi
+if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
+elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
+            "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
+else
+  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
+  exit 1
+fi
+# --- end runfiles.bash initialization ---
+
+source "$(rlocation $TEST_WORKSPACE/tests/unittest.bash)" \
+  || { echo "Could not source $TEST_WORKSPACE/tests/unittest.bash" >&2; exit 1; }
+
+function create_pkg() {
+  local -r pkg="$1"
+  mkdir -p "$pkg"
+  cd "$pkg"
+
+  cat > WORKSPACE <<EOF
+workspace(name = 'bazel_skylib')
+
+load("//lib:unittest.bzl", "register_unittest_toolchains")
+
+register_unittest_toolchains()
+EOF
+
+  mkdir -p rules
+  cat > rules/BUILD <<EOF
+exports_files(["*.bzl"])
+EOF
+  ln -sf "$(rlocation bazel_skylib/rules/analysis_failure_test.bzl)" rules/analysis_failure_test.bzl
+
+  mkdir -p lib
+  cat > lib/BUILD <<EOF
+exports_files(["*.bzl"])
+EOF
+  ln -sf "$(rlocation bazel_skylib/lib/unittest.bzl)" lib/unittest.bzl
+  ln -sf "$(rlocation bazel_skylib/lib/types.bzl)" lib/types.bzl
+  ln -sf "$(rlocation bazel_skylib/lib/partial.bzl)" lib/partial.bzl
+  ln -sf "$(rlocation bazel_skylib/lib/new_sets.bzl)" lib/new_sets.bzl
+  ln -sf "$(rlocation bazel_skylib/lib/dicts.bzl)" lib/dicts.bzl
+
+  mkdir -p toolchains/unittest
+  ln -sf "$(rlocation bazel_skylib/toolchains/unittest/BUILD)" toolchains/unittest/BUILD
+
+  mkdir -p fakerules
+  cat > fakerules/rules.bzl <<EOF
+load("//rules:analysis_failure_test.bzl", "analysis_failure_test")
+
+def _fake_rule_impl(ctx):
+    fail("This rule fails at analysis phase")
+
+fake_rule = rule(
+    implementation = _fake_rule_impl,
+)
+
+def _fake_depending_rule_impl(ctx):
+    return []
+
+fake_depending_rule = rule(
+    implementation = _fake_depending_rule_impl,
+    attrs = {"deps" : attr.label_list()},
+)
+EOF
+
+  cat > fakerules/BUILD <<EOF
+exports_files(["*.bzl"])
+EOF
+
+  mkdir -p testdir
+  cat > testdir/BUILD <<EOF
+load("//rules:analysis_failure_test.bzl", "analysis_failure_test")
+load("//fakerules:rules.bzl", "fake_rule", "fake_depending_rule")
+
+fake_rule(name = "target_fails")
+
+fake_depending_rule(
+    name = "dep_fails",
+    deps = [":target_fails"],
+)
+
+fake_depending_rule(
+    name = "rule_that_does_not_fail",
+    deps = [],
+)
+
+analysis_failure_test(
+    name = "direct_target_fails",
+    target_under_test = ":target_fails",
+    error_message = "This rule fails at analysis phase",
+)
+
+analysis_failure_test(
+    name = "transitive_target_fails",
+    target_under_test = ":dep_fails",
+    error_message = "This rule fails at analysis phase",
+)
+
+analysis_failure_test(
+    name = "fails_with_wrong_error_message",
+    target_under_test = ":dep_fails",
+    error_message = "This is the wrong error message",
+)
+
+analysis_failure_test(
+    name = "fails_with_target_that_does_not_fail",
+    target_under_test = ":rule_that_does_not_fail",
+    error_message = "This rule fails at analysis phase",
+)
+EOF
+}
+
+
+function test_direct_target_succeeds() {
+  local -r pkg="${FUNCNAME[0]}"
+  create_pkg "$pkg"
+
+  bazel test testdir:direct_target_fails >"$TEST_log" 2>&1 || fail "Expected test to pass"
+
+  expect_log "PASSED"
+}
+
+function test_transitive_target_succeeds() {
+  local -r pkg="${FUNCNAME[0]}"
+  create_pkg "$pkg"
+
+  bazel test testdir:transitive_target_fails  >"$TEST_log" 2>&1 || fail "Expected test to pass"
+
+  expect_log "PASSED"
+}
+
+function test_with_wrong_error_message_fails() {
+  local -r pkg="${FUNCNAME[0]}"
+  create_pkg "$pkg"
+
+  bazel test testdir:fails_with_wrong_error_message --test_output=all --verbose_failures \
+      >"$TEST_log" 2>&1 && fail "Expected test to fail" || true
+
+  expect_log "Expected errors to contain 'This is the wrong error message' but did not. Actual errors:"
+}
+
+function test_with_rule_that_does_not_fail_fails() {
+  local -r pkg="${FUNCNAME[0]}"
+  create_pkg "$pkg"
+
+  bazel test testdir:fails_with_target_that_does_not_fail --test_output=all --verbose_failures \
+      >"$TEST_log" 2>&1 && fail "Expected test to fail" || true
+
+  expect_log "Expected failure of target_under_test, but found success"
+}
+
+
+cd "$TEST_TMPDIR"
+run_suite "analysis_failure_test test suite"
diff --git a/tests/analysis_failure_test_tests.bzl b/tests/analysis_failure_test_tests.bzl
new file mode 100644
index 00000000..75db5fd0
--- /dev/null
+++ b/tests/analysis_failure_test_tests.bzl
@@ -0,0 +1,45 @@
+# Copyright 2021 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for analysis_failure_test.bzl."""
+
+load("//rules:analysis_failure_test.bzl", "analysis_failure_test")
+
+_EXPECTED_ERROR_MESSAGE = "The error message to assert!"
+
+def _dummy_rule_impl(ctx):
+    if ctx.attr.asserted_input != 0:
+        fail(_EXPECTED_ERROR_MESSAGE)
+
+dummy_rule = rule(
+    implementation = _dummy_rule_impl,
+    attrs = {
+        "asserted_input": attr.int(),
+    },
+    doc = "This rule acts as a dummy case to ensure that analysis_failure_test works as expected",
+)
+
+# buildifier: disable=unnamed-macro
+def analysis_failure_test_test_suite():
+    dummy_rule(
+        name = "unit",
+        asserted_input = 42,
+        tags = ["manual"],
+    )
+
+    analysis_failure_test(
+        name = "unit_fails_to_analyse",
+        target_under_test = ":unit",
+        error_message = _EXPECTED_ERROR_MESSAGE,
+    )