Skip to content

Commit

Permalink
Merge pull request #89393 from Jefftree/bdd-linkage
Browse files Browse the repository at this point in the history
Add CLI script for listing untested conformance behaviors
  • Loading branch information
k8s-ci-robot committed Mar 26, 2020
2 parents 8aca6dc + 074a676 commit 0dd3753
Show file tree
Hide file tree
Showing 10 changed files with 222 additions and 61 deletions.
4 changes: 3 additions & 1 deletion test/conformance/BUILD
Expand Up @@ -9,6 +9,7 @@ go_library(
importpath = "k8s.io/kubernetes/test/conformance",
visibility = ["//visibility:private"],
deps = [
"//test/conformance/behaviors:go_default_library",
"//vendor/github.com/onsi/ginkgo/types:go_default_library",
"//vendor/gopkg.in/yaml.v2:go_default_library",
],
Expand All @@ -32,7 +33,7 @@ filegroup(
srcs = [
":package-srcs",
"//test/conformance/behaviors:all-srcs",
"//test/conformance/kubetestgen:all-srcs",
"//test/conformance/kubeconform:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
Expand Down Expand Up @@ -77,6 +78,7 @@ go_test(
srcs = ["walk_test.go"],
data = glob(["testdata/**"]),
embed = [":go_default_library"],
deps = ["//test/conformance/behaviors:go_default_library"],
)

genrule(
Expand Down
18 changes: 18 additions & 0 deletions test/conformance/behaviors/types.go
Expand Up @@ -37,3 +37,21 @@ type Behavior struct {
APIType string `json:"apiType,omitempty"`
Description string `json:"description,omitempty"`
}

// ConformanceData describes the structure of the conformance.yaml file
type ConformanceData struct {
// A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number.
URL string `yaml:"-"`
// Extracted from the "Testname:" comment before the test
TestName string
// CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]`
CodeName string
// Extracted from the "Description:" comment before the test
Description string
// Version when this test is added or modified ex: v1.12, v1.13
Release string
// File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes.
File string
// Behaviors is the list of conformance behaviors tested by a particular e2e test
Behaviors []string `yaml:"behaviors,omitempty"`
}
File renamed without changes.
Expand Up @@ -2,8 +2,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")

go_library(
name = "go_default_library",
srcs = ["kubetestgen.go"],
importpath = "k8s.io/kubernetes/test/conformance/kubetestgen",
srcs = [
"gen.go",
"kubeconform.go",
"link.go",
],
importpath = "k8s.io/kubernetes/test/conformance/kubeconform",
visibility = ["//visibility:private"],
deps = [
"//test/conformance/behaviors:go_default_library",
Expand All @@ -29,7 +33,7 @@ filegroup(
)

go_binary(
name = "kubetestgen",
name = "kubeconform",
data = ["//api/openapi-spec"],
embed = [":go_default_library"],
visibility = ["//visibility:public"],
Expand Down
File renamed without changes.
Expand Up @@ -17,7 +17,6 @@ limitations under the License.
package main

import (
"flag"
"fmt"
"os"
"sort"
Expand All @@ -30,29 +29,10 @@ import (
"k8s.io/kubernetes/test/conformance/behaviors"
)

type options struct {
schemaPath string
resource string
area string
behaviorsDir string
}

func parseFlags() *options {
o := &options{}
flag.StringVar(&o.schemaPath, "schema", "", "Path to the OpenAPI schema")
flag.StringVar(&o.resource, "resource", ".*", "Resource name")
flag.StringVar(&o.area, "area", "default", "Area name to use")
flag.StringVar(&o.behaviorsDir, "dir", "../behaviors/", "Path to the behaviors directory")
flag.Parse()
return o
}

var defMap map[string]analysis.SchemaRef

func main() {
func gen(o *options) {
defMap = make(map[string]analysis.SchemaRef)
o := parseFlags()

d, err := loads.JSONSpec(o.schemaPath)
if err != nil {
fmt.Printf("ERROR: %s\n", err.Error())
Expand Down
64 changes: 64 additions & 0 deletions test/conformance/kubeconform/kubeconform.go
@@ -0,0 +1,64 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"flag"
"fmt"
)

type options struct {
// Flags only used for generating behaviors
schemaPath string
resource string
area string

// Flags only used for linking behaviors
testdata string
listMissing bool

// Flags shared between CLI tools
behaviorsDir string
}

func parseFlags() *options {
o := &options{}

flag.StringVar(&o.schemaPath, "schema", "", "Path to the OpenAPI schema")
flag.StringVar(&o.resource, "resource", ".*", "Resource name")
flag.StringVar(&o.area, "area", "default", "Area name to use")

flag.StringVar(&o.testdata, "testdata", "../testdata/conformance.yaml", "YAML file containing test linkage data")
flag.BoolVar(&o.listMissing, "missing", true, "Only list behaviors missing tests")

flag.StringVar(&o.behaviorsDir, "dir", "../behaviors", "Path to the behaviors directory")

flag.Parse()
return o
}

func main() {
o := parseFlags()
action := flag.Arg(0)
if action == "gen" {
gen(o)
} else if action == "link" {
link(o)
} else {
fmt.Printf("Unknown argument %s\n", action)
}
}
106 changes: 106 additions & 0 deletions test/conformance/kubeconform/link.go
@@ -0,0 +1,106 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"

"gopkg.in/yaml.v2"

"k8s.io/kubernetes/test/conformance/behaviors"
)

func link(o *options) {
var behaviorFiles []string
behaviorsMapping := make(map[string][]string)
var conformanceDataList []behaviors.ConformanceData

err := filepath.Walk(o.behaviorsDir,
func(path string, info os.FileInfo, err error) error {
if err != nil {
fmt.Printf("%v", err)
}
r, _ := regexp.Compile(".+.yaml$")
if r.MatchString(path) {
behaviorFiles = append(behaviorFiles, path)
}
return nil
})
if err != nil {
fmt.Printf("%v", err)
return
}
fmt.Printf("%v", behaviorFiles)

for _, behaviorFile := range behaviorFiles {
var suite behaviors.Suite

yamlFile, err := ioutil.ReadFile(behaviorFile)
if err != nil {
fmt.Printf("%v", err)
return
}
err = yaml.UnmarshalStrict(yamlFile, &suite)
if err != nil {
fmt.Printf("%v", err)
return
}

for _, behavior := range suite.Behaviors {
behaviorsMapping[behavior.ID] = nil
}
}

conformanceYaml, err := ioutil.ReadFile(o.testdata)
if err != nil {
fmt.Printf("%v", err)
return
}

err = yaml.Unmarshal(conformanceYaml, &conformanceDataList)
if err != nil {
fmt.Printf("%v", err)
return
}

for _, data := range conformanceDataList {
for _, behaviorID := range data.Behaviors {
if _, ok := behaviorsMapping[behaviorID]; !ok {
fmt.Printf("Error, cannot find behavior \"%s\"", behaviorID)
return
}
behaviorsMapping[behaviorID] = append(behaviorsMapping[behaviorID], data.CodeName)
}
}
printBehaviorsMapping(behaviorsMapping, o)
}

func printBehaviorsMapping(behaviorsMapping map[string][]string, o *options) {
for behaviorID, tests := range behaviorsMapping {
if o.listMissing {
if tests == nil {
fmt.Println(behaviorID)
} else {
fmt.Println(behaviorID)
}
}
}
}
37 changes: 11 additions & 26 deletions test/conformance/walk.go
Expand Up @@ -34,6 +34,8 @@ import (
"text/template"

"github.com/onsi/ginkgo/types"

"k8s.io/kubernetes/test/conformance/behaviors"
)

var (
Expand Down Expand Up @@ -63,23 +65,6 @@ type frame struct {
Line int
}

type conformanceData struct {
// A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number.
URL string `yaml:"-"`
// Extracted from the "Testname:" comment before the test
TestName string
// CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]`
CodeName string
// Extracted from the "Description:" comment before the test
Description string
// Version when this test is added or modified ex: v1.12, v1.13
Release string
// File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes.
File string
// Behaviors is the list of conformance behaviors tested by a particular e2e test
Behaviors []string `yaml:"behaviors,omitempty"`
}

func main() {
flag.Parse()

Expand All @@ -95,7 +80,7 @@ func main() {

seenLines = map[string]struct{}{}
dec := json.NewDecoder(f)
testInfos := []*conformanceData{}
testInfos := []*behaviors.ConformanceData{}
for {
var spec *types.SpecSummary
if err := dec.Decode(&spec); err == io.EOF {
Expand Down Expand Up @@ -124,8 +109,8 @@ func isConformance(spec *types.SpecSummary) bool {
return strings.Contains(getTestName(spec), "[Conformance]")
}

func getTestInfo(spec *types.SpecSummary) *conformanceData {
var c *conformanceData
func getTestInfo(spec *types.SpecSummary) *behaviors.ConformanceData {
var c *behaviors.ConformanceData
var err error
// The key to this working is that we don't need to parse every file or walk
// every componentCodeLocation. The last componentCodeLocation is going to typically start
Expand Down Expand Up @@ -155,7 +140,7 @@ func getTestName(spec *types.SpecSummary) string {
return strings.Join(spec.ComponentTexts[1:], " ")
}

func saveAllTestInfo(dataSet []*conformanceData) {
func saveAllTestInfo(dataSet []*behaviors.ConformanceData) {
if *confDoc {
// Note: this assumes that you're running from the root of the kube src repo
templ, err := template.ParseFiles("./test/conformance/cf_header.md")
Expand Down Expand Up @@ -186,7 +171,7 @@ func saveAllTestInfo(dataSet []*conformanceData) {
fmt.Println(string(b))
}

func getConformanceDataFromStackTrace(fullstackstrace string) (*conformanceData, error) {
func getConformanceDataFromStackTrace(fullstackstrace string) (*behaviors.ConformanceData, error) {
// The full stacktrace to parse from ginkgo is of the form:
// k8s.io/kubernetes/test/e2e/storage/utils.SIGDescribe(0x51f4c4f, 0xf, 0x53a0dd8, 0xc000ab6e01)\n\ttest/e2e/storage/utils/framework.go:23 +0x75\n ... ...
// So we need to split it into lines, remove whitespace, and then grab the files/lines.
Expand Down Expand Up @@ -240,7 +225,7 @@ func getConformanceDataFromStackTrace(fullstackstrace string) (*conformanceData,

// scanFileForFrame will scan the target and look for a conformance comment attached to the function
// described by the target frame. If the comment can't be found then nil, nil is returned.
func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*conformanceData, error) {
func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*behaviors.ConformanceData, error) {
fset := token.NewFileSet() // positions are relative to fset
f, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
if err != nil {
Expand All @@ -266,7 +251,7 @@ func validateTestName(s string) error {
return nil
}

func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *conformanceData {
func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *behaviors.ConformanceData {
if !shouldProcessCommentGroup(fset, cg, f) {
return nil
}
Expand All @@ -290,10 +275,10 @@ func shouldProcessCommentGroup(fset *token.FileSet, cg *ast.CommentGroup, f fram
return lineDiff > 0 && lineDiff <= conformanceCommentsLineWindow
}

func commentToConformanceData(comment string) *conformanceData {
func commentToConformanceData(comment string) *behaviors.ConformanceData {
lines := strings.Split(comment, "\n")
descLines := []string{}
cd := &conformanceData{}
cd := &behaviors.ConformanceData{}
var curLine string
for _, line := range lines {
line = strings.TrimSpace(line)
Expand Down

0 comments on commit 0dd3753

Please sign in to comment.