conftest verify with 'not' always fails

[midestefanis]

probes.rego

package main

kinds_with_containers := [ "Deployment", "StatefulSet", "DaemonSet", "Job", "CronJob", "Pod" ]

get_container_array_from(object) = array.concat(
    [ x | x := object.spec.jobTemplate.spec.template.spec.initContainers[_] ],
    [ x | x := object.spec.jobTemplate.spec.template.spec.containers[_] ])
{
    object.kind == "CronJob"
} else = array.concat(
    [ x | x := object.spec.initContainers[_] ],
    [ x | x := object.spec.containers[_] ])
{
    object.kind == "Pod"
} else = array.concat(
    [ x | x := object.spec.template.spec.initContainers[_] ],
    [ x | x := object.spec.template.spec.containers[_] ])
{
    object.kind == kinds_with_containers[i]
}

# **************************************************************
# A Deployment must have startup or readiness probes.

deny_deployments_without_probes[msg] {
    input[_].contents.kind == "Deployment"
    deployment := input[_].contents
    deployment_name := deployment.metadata.name
    deployment_namespace := deployment.metadata.namespace

    containers := get_container_array_from(deployment)
    container := containers[_]

    # check for startupProbe or readinessProbe
    not container.startupProbe
    not container.readinessProbe
    
    msg := sprintf("Deployment %s/%s has no readinessProbe or startupProbe for container %s", [deployment_name, deployment_namespace, container.name])
}

probes_test.rego

package main

# Tests for deny_deployments_without_probes rule

# Test that a Deployment with startupProbe and readinessProbe passes validation
test_valid_deployment {
	deployment := {
		"kind": "Deployment",
		"metadata": {"name": "my-deployment", "namespace": "default"},
		"spec": {
			"replicas": 1,
			"selector": {"matchLabels": {"app": "my-app"}},
			"template": {
				"metadata": {"labels": {"app": "my-app"}},
				"spec": {"containers": [{
					"name": "my-container",
					"image": "nginx",
					"startupProbe": {
						"httpGet": {"path": "/", "port": "http"},
						"initialDelaySeconds": 5,
						"periodSeconds": 10,
					},
					"readinessProbe": {
						"httpGet": {"path": "/", "port": "http"},
						"initialDelaySeconds": 5,
						"periodSeconds": 10,
					},
					"livenessProbe": {
						"httpGet": {"path": "/", "port": "http"},
						"initialDelaySeconds": 5,
						"periodSeconds": 10,
					},
				}]},
			},
		},
	}
	not deny_deployments_without_probes with input as deployment
}

Output:

conftest verify --report full
FAILURES
--------------------------------------------------------------------------------
data.main.test_valid_deployment: FAIL (808.584µs)

  query:1                        Enter data.main.test_valid_deployment = _
  query:1                        | Eval data.main.test_valid_deployment = _
  query:1                        | Unify data.main.test_valid_deployment = _
  query:1                        | Index data.main.test_valid_deployment (matched 1 rule, early exit)
  policy/probes_test.rego:6      | Enter data.main.test_valid_deployment
  policy/probes_test.rego:7      | | Eval deployment = {"kind": "Deployment", "metadata": {"name": "my-deployment", "namespace": "default"}, "spec": {"replicas": 1, "selector": {"matchLabels": {"app": "my-app"}}, "template": {"metadata": {"labels": {"app": "my-app"}}, "spec": {"containers": [{"image": "nginx", "livenessProbe": {"httpGet": {"path": "/", "port": "http"}, "initialDelaySeconds": 5, "periodSeconds": 10}, "name": "my-container", "readinessProbe": {"httpGet": {"path": "/", "port": "http"}, "initialDelaySeconds": 5, "periodSeconds": 10}, "startupProbe": {"httpGet": {"path": "/", "port": "http"}, "initialDelaySeconds": 5, "periodSeconds": 10}}]}}}}
  policy/probes_test.rego:7      | | Unify deployment = {"kind": "Deployment", "metadata": {"name": "my-deployment", "namespace": "default"}, "spec": {"replicas": 1, "selector": {"matchLabels": {"app": "my-app"}}, "template": {"metadata": {"labels": {"app": "my-app"}}, "spec": {"containers": [{"image": "nginx", "livenessProbe": {"httpGet": {"path": "/", "port": "http"}, "initialDelaySeconds": 5, "periodSeconds": 10}, "name": "my-container", "readinessProbe": {"httpGet": {"path": "/", "port": "http"}, "initialDelaySeconds": 5, "periodSeconds": 10}, "startupProbe": {"httpGet": {"path": "/", "port": "http"}, "initialDelaySeconds": 5, "periodSeconds": 10}}]}}}}
  policy/probes_test.rego:37     | | Eval not data.main.deny_deployments_without_probes with input as deployment
  policy/probes_test.rego:37     | | Enter data.main.deny_deployments_without_probes
  policy/probes_test.rego:37     | | | Eval data.main.deny_deployments_without_probes
  policy/probes_test.rego:37     | | | Unify data.main.deny_deployments_without_probes = _
  policy/probes_test.rego:37     | | | Index data.main.deny_deployments_without_probes (matched 1 rule)
  policy/probes.rego:25          | | | Enter data.main.deny_deployments_without_probes
  policy/probes.rego:26          | | | | Eval input[_].contents.kind = "Deployment"
  policy/probes.rego:26          | | | | Unify input[_].contents.kind = "Deployment"
  policy/probes.rego:26          | | | | Unify "kind" = _
  policy/probes.rego:26          | | | | Unify "metadata" = _
  policy/probes.rego:26          | | | | Unify "spec" = _
  policy/probes.rego:26          | | | | Fail input[_].contents.kind = "Deployment"
  policy/probes_test.rego:37     | | | Unify set() = _
  policy/probes_test.rego:37     | | | Exit data.main.deny_deployments_without_probes
  policy/probes_test.rego:37     | | Redo data.main.deny_deployments_without_probes
  policy/probes_test.rego:37     | | | Redo data.main.deny_deployments_without_probes
  policy/probes_test.rego:37     | | Fail not data.main.deny_deployments_without_probes with input as deployment
  policy/probes_test.rego:7      | | Redo deployment = {"kind": "Deployment", "metadata": {"name": "my-deployment", "namespace": "default"}, "spec": {"replicas": 1, "selector": {"matchLabels": {"app": "my-app"}}, "template": {"metadata": {"labels": {"app": "my-app"}}, "spec": {"containers": [{"image": "nginx", "livenessProbe": {"httpGet": {"path": "/", "port": "http"}, "initialDelaySeconds": 5, "periodSeconds": 10}, "name": "my-container", "readinessProbe": {"httpGet": {"path": "/", "port": "http"}, "initialDelaySeconds": 5, "periodSeconds": 10}, "startupProbe": {"httpGet": {"path": "/", "port": "http"}, "initialDelaySeconds": 5, "periodSeconds": 10}}]}}}}
  query:1                        | Fail data.main.test_valid_deployment = _

SUMMARY
--------------------------------------------------------------------------------
policy/probes_test.rego:
data.main.test_valid_deployment: FAIL (808.584µs)
--------------------------------------------------------------------------------
FAIL: 1/1

I don't understand what is wrong, any help?

[jpreese]

This looks similar to #781 (comment)

deny_deployments_without_probes will always exist. It will either contain violation messages (non-empty set) or not contain violation messages (empty set).

You need to assert against the size of the set or assert if a specific violation exists in the set.

Alternatively, and what I personally do, you could refactor out the main logic of your policy into another rule and have your deny_... block just call that rule and handle all of the messaging.

For example:

deny[msg] {
  not my_policy
  msg := "my policy violated"
}

my_policy {
  // main policy definition
}

This also enables you to focus your tests purely on the policy and allows you to test for not my_policy

[jpreese]

Resolved in #801