Validate JUnit output against Jenkins JUnit XSD (#134)

* Validate JUnit output against Jenkins JUnix XSD * Add missing Jenkins JUnit XSD * Add time to TestCase for #127
yannh · Oct 16, 2022 · f5338b0 · f5338b0
1 parent f68d6ec
commit f5338b0
Show file tree

Hide file tree

Showing 6 changed files with 142 additions and 17 deletions.
diff --git a/Dockerfile.bats b/Dockerfile.bats
@@ -1,5 +1,5 @@
 FROM bats/bats:v1.2.1
-RUN apk --no-cache add ca-certificates parallel
+RUN apk --no-cache add ca-certificates parallel libxml2-utils
 COPY dist/kubeconform_linux_amd64/kubeconform /code/bin/
 COPY acceptance.bats acceptance-nonetwork.bats /code/
 COPY fixtures /code/fixtures
diff --git a/Makefile b/Makefile
@@ -40,3 +40,6 @@ release:
 update-deps:
 	go get -u ./...
 	go mod tidy
+
+update-junit-xsd:
+	curl https://raw.githubusercontent.com/junit-team/junit5/main/platform-tests/src/test/resources/jenkins-junit.xsd > fixtures/junit.xsd
diff --git a/acceptance.bats b/acceptance.bats
@@ -316,3 +316,10 @@ resetCacheFolder() {
   [ "$status" -eq 0 ]
   [ "$output" = 'Summary: 100000 resources found parsing stdin - Valid: 100000, Invalid: 0, Errors: 0, Skipped: 0' ]
 }
+
+@test "JUnit output can be validated against the Junit schema definition" {
+  run bash -c "bin/kubeconform -output junit -summary fixtures/valid.yaml > output.xml"
+  [ "$status" -eq 0 ]
+  run xmllint --noout --schema fixtures/junit.xsd output.xml
+  [ "$status" -eq 0 ]
+}
diff --git a/fixtures/junit.xsd b/fixtures/junit.xsd
@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Source: https://svn.jenkins-ci.org/trunk/hudson/dtkit/dtkit-format/dtkit-junit-model/src/main/resources/com/thalesgroup/dtkit/junit/model/xsd/junit-4.xsd
+ 
+ This file available under the terms of the MIT License as follows:
+ 
+ *******************************************************************************
+ * Copyright (c) 2010 Thales Corporate Services SAS                             *
+ *                                                                              *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy *
+ * of this software and associated documentation files (the "Software"), to deal*
+ * in the Software without restriction, including without limitation the rights *
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell    *
+ * copies of the Software, and to permit persons to whom the Software is        *
+ * furnished to do so, subject to the following conditions:                     *
+ *                                                                              *
+ * The above copyright notice and this permission notice shall be included in   *
+ * all copies or substantial portions of the Software.                          *
+ *                                                                              *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR   *
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,     *
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE  *
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER       *
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,*
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN    *
+ * THE SOFTWARE.                                                                *
+ ********************************************************************************
+ -->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+
+    <xs:element name="failure">
+        <xs:complexType mixed="true">
+            <xs:attribute name="type" type="xs:string" use="optional"/>
+            <xs:attribute name="message" type="xs:string" use="optional"/>
+        </xs:complexType>
+    </xs:element>
+
+    <xs:element name="error">
+        <xs:complexType mixed="true">
+            <xs:attribute name="type" type="xs:string" use="optional"/>
+            <xs:attribute name="message" type="xs:string" use="optional"/>
+        </xs:complexType>
+    </xs:element>
+
+    <xs:element name="properties">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element ref="property" maxOccurs="unbounded"/>
+            </xs:sequence>
+        </xs:complexType>
+    </xs:element>
+
+    <xs:element name="property">
+        <xs:complexType>
+            <xs:attribute name="name" type="xs:string" use="required"/>
+            <xs:attribute name="value" type="xs:string" use="required"/>
+        </xs:complexType>
+    </xs:element>
+
+    <xs:element name="skipped" type="xs:string"/>
+    <xs:element name="system-err" type="xs:string"/>
+    <xs:element name="system-out" type="xs:string"/>
+
+    <xs:element name="testcase">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element ref="skipped" minOccurs="0" maxOccurs="1"/>
+                <xs:element ref="error" minOccurs="0" maxOccurs="unbounded"/>
+                <xs:element ref="failure" minOccurs="0" maxOccurs="unbounded"/>
+                <xs:element ref="system-out" minOccurs="0" maxOccurs="unbounded"/>
+                <xs:element ref="system-err" minOccurs="0" maxOccurs="unbounded"/>
+            </xs:sequence>
+            <xs:attribute name="name" type="xs:string" use="required"/>
+            <xs:attribute name="assertions" type="xs:string" use="optional"/>
+            <xs:attribute name="time" type="xs:string" use="optional"/>
+            <xs:attribute name="classname" type="xs:string" use="optional"/>
+            <xs:attribute name="status" type="xs:string" use="optional"/>
+        </xs:complexType>
+    </xs:element>
+
+    <xs:element name="testsuite">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element ref="properties" minOccurs="0" maxOccurs="1"/>
+                <xs:element ref="testcase" minOccurs="0" maxOccurs="unbounded"/>
+                <xs:element ref="system-out" minOccurs="0" maxOccurs="1"/>
+                <xs:element ref="system-err" minOccurs="0" maxOccurs="1"/>
+            </xs:sequence>
+            <xs:attribute name="name" type="xs:string" use="required"/>
+            <xs:attribute name="tests" type="xs:string" use="required"/>
+            <xs:attribute name="failures" type="xs:string" use="optional"/>
+            <xs:attribute name="errors" type="xs:string" use="optional"/>
+            <xs:attribute name="time" type="xs:string" use="optional"/>
+            <xs:attribute name="disabled" type="xs:string" use="optional"/>
+            <xs:attribute name="skipped" type="xs:string" use="optional"/>
+            <xs:attribute name="timestamp" type="xs:string" use="optional"/>
+            <xs:attribute name="hostname" type="xs:string" use="optional"/>
+            <xs:attribute name="id" type="xs:string" use="optional"/>
+            <xs:attribute name="package" type="xs:string" use="optional"/>
+        </xs:complexType>
+    </xs:element>
+
+    <xs:element name="testsuites">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element ref="testsuite" minOccurs="0" maxOccurs="unbounded"/>
+            </xs:sequence>
+            <xs:attribute name="name" type="xs:string" use="optional"/>
+            <xs:attribute name="time" type="xs:string" use="optional"/>
+            <xs:attribute name="tests" type="xs:string" use="optional"/>
+            <xs:attribute name="failures" type="xs:string" use="optional"/>
+            <xs:attribute name="disabled" type="xs:string" use="optional"/>
+            <xs:attribute name="errors" type="xs:string" use="optional"/>
+        </xs:complexType>
+    </xs:element>
+
+
+</xs:schema>
diff --git a/pkg/output/junit.go b/pkg/output/junit.go
@@ -33,22 +33,22 @@ type Property struct {
 }
 
 type TestSuite struct {
-	XMLName    xml.Name    `xml:"testsuite"`
-	Properties []*Property `xml:"properties>property,omitempty"`
-	Cases      []TestCase  `xml:"testcase"`
-	Name       string      `xml:"name,attr"`
-	Id         int         `xml:"id,attr"`
-	Tests      int         `xml:"tests,attr"`
-	Failures   int         `xml:"failures,attr"`
-	Errors     int         `xml:"errors,attr"`
-	Disabled   int         `xml:"disabled,attr"`
-	Skipped    int         `xml:"skipped,attr"`
+	XMLName  xml.Name   `xml:"testsuite"`
+	Cases    []TestCase `xml:"testcase"`
+	Name     string     `xml:"name,attr"`
+	Id       int        `xml:"id,attr"`
+	Tests    int        `xml:"tests,attr"`
+	Failures int        `xml:"failures,attr"`
+	Errors   int        `xml:"errors,attr"`
+	Disabled int        `xml:"disabled,attr"`
+	Skipped  int        `xml:"skipped,attr"`
 }
 
 type TestCase struct {
 	XMLName   xml.Name         `xml:"testcase"`
 	Name      string           `xml:"name,attr"`
 	ClassName string           `xml:"classname,attr"`
+	Time      int              `xml:"time,attr"` // Optional, but for Buildkite support  https://github.com/yannh/kubeconform/issues/127
 	Skipped   *TestCaseSkipped `xml:"skipped,omitempty"`
 	Error     *TestCaseError   `xml:"error,omitempty"`
 	Failure   []TestCaseError  `xml:"failure,omitempty"`
@@ -100,8 +100,7 @@ func (o *junito) Write(result validator.Result) error {
 			Name:  result.Resource.Path,
 			Id:    o.id,
 			Tests: 0, Failures: 0, Errors: 0, Disabled: 0, Skipped: 0,
-			Cases:      make([]TestCase, 0),
-			Properties: make([]*Property, 0),
+			Cases: make([]TestCase, 0),
 		}
 		o.suites[result.Resource.Path] = suite
 	}

diff --git a/pkg/output/junit_test.go b/pkg/output/junit_test.go
@@ -48,8 +48,7 @@ metadata:
 			},
 			"<testsuites name=\"kubeconform\" time=\"\" tests=\"1\" failures=\"0\" disabled=\"0\" errors=\"0\">\n" +
 				"  <testsuite name=\"deployment.yml\" id=\"1\" tests=\"1\" failures=\"0\" errors=\"0\" disabled=\"0\" skipped=\"0\">\n" +
-				"    <properties></properties>\n" +
-				"    <testcase name=\"my-app\" classname=\"Deployment@apps/v1\"></testcase>\n" +
+				"    <testcase name=\"my-app\" classname=\"Deployment@apps/v1\" time=\"\"></testcase>\n" +
 				"  </testsuite>\n" +
 				"</testsuites>\n",
 		},
@@ -82,8 +81,7 @@ metadata:
 			},
 			"<testsuites name=\"kubeconform\" time=\"\" tests=\"1\" failures=\"0\" disabled=\"0\" errors=\"0\">\n" +
 				"  <testsuite name=\"deployment.yml\" id=\"1\" tests=\"1\" failures=\"0\" errors=\"0\" disabled=\"0\" skipped=\"0\">\n" +
-				"    <properties></properties>\n" +
-				"    <testcase name=\"my-app\" classname=\"Deployment@apps/v1\"></testcase>\n" +
+				"    <testcase name=\"my-app\" classname=\"Deployment@apps/v1\" time=\"\"></testcase>\n" +
 				"  </testsuite>\n" +
 				"</testsuites>\n",
 		},