Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: node e2e add ginkgo progress reports #116120

Closed
wants to merge 4 commits into from
Closed

WIP: node e2e add ginkgo progress reports #116120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
44cdece
Remove tests argument from GCE image
bobbypage Feb 26, 2023
bcf04c1
test: Emit ginkgo log for each node e2e
bobbypage Feb 25, 2023
653d9bf
test: Add some default flags for node e2e Ginkgo
bobbypage Feb 26, 2023
38100b2
test: Add ginkgo progress reports for node e2e
bobbypage Feb 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 21 additions & 3 deletions test/e2e_node/remote/node_e2e.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,19 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
}
}

// Copy the k8s source. We use this for the gingko --source--root feature which provides progress reports if the test gets stuck.
// To provide line numbers in the progress report it needs to the source. We use git archive to export the k8s srcs into the tar directory, which will be uploaded to the node.
k8sTarDestinationSourcePath := filepath.Join(tardir, "k8s_source_root")
err = os.Mkdir(k8sTarDestinationSourcePath, os.ModePerm)
if err != nil {
return fmt.Errorf("failed to create k8s source root directory %q: %w", tardir, err)
}
cmd := fmt.Sprintf("git -C %s archive --format=tar HEAD | tar -x -C %s", rootDir, k8sTarDestinationSourcePath)
out, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to build git archive tar %v. Output:\n%s", err, out)
}

if systemSpecName != "" {
// Copy system spec file
source := filepath.Join(rootDir, system.SystemSpecPath, systemSpecName+".yaml")
Expand Down Expand Up @@ -196,12 +209,17 @@ func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePr
systemSpecFile = systemSpecName + ".yaml"
}

outputGinkgoFile := filepath.Join(results, fmt.Sprintf("%s-ginkgo.log", host))

ginkgoSourceRootArgument := fmt.Sprintf("--source-root %s", filepath.Join(workspace, "k8s_source_root"))

// Run the tests
klog.V(2).Infof("Starting tests on %q", host)
cmd := getSSHCommand(" && ",
fmt.Sprintf("cd %s", workspace),
fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-name=%s --system-spec-file=%s --extra-envs=%s --runtime-config=%s --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=\"%s\" %s",
timeout.Seconds(), ginkgoArgs, systemSpecName, systemSpecFile, extraEnvs, runtimeConfig, host, results, junitFilePrefix, imageDesc, testArgs),
// Note, we need to have set -o pipefail here to ensure we return the appriorate exit code from ginkgo; not tee
fmt.Sprintf("set -o pipefail; timeout -k 30s %fs ./ginkgo %s %s ./e2e_node.test -- --system-spec-name=%s --system-spec-file=%s --extra-envs=%s --runtime-config=%s --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=\"%s\" %s 2>&1 | tee -i %s",
timeout.Seconds(), ginkgoSourceRootArgument, ginkgoArgs, systemSpecName, systemSpecFile, extraEnvs, runtimeConfig, host, results, junitFilePrefix, imageDesc, testArgs, outputGinkgoFile),
)
return SSH(host, "sh", "-c", cmd)
return SSH(host, "/bin/bash", "-c", cmd)
}
44 changes: 11 additions & 33 deletions test/e2e_node/runner/remote/run_remote.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,6 @@ type GCEImage struct {
Metadata string `json:"metadata"`
Machine string `json:"machine,omitempty"`
Resources Resources `json:"resources,omitempty"`
// This test is for benchmark (no limit verification, more result log, node name has format 'machine-image-uuid') if 'Tests' is non-empty.
Tests []string `json:"tests,omitempty"`
}

type internalImageConfig struct {
Expand All @@ -185,7 +183,6 @@ type internalGCEImage struct {
resources Resources
metadata *compute.Metadata
machine string
tests []string
}

func main() {
Expand Down Expand Up @@ -225,6 +222,11 @@ func main() {
return
}

// Append some default ginkgo flags. We use similar defaults here as hack/ginkgo-e2e.sh
// Note, to provide line numbers in progress reports with --poll-progress.*, --source--root pointing to k8s source is also required and added later.
allGinkgoFlags := fmt.Sprintf("%s --no-color -v --poll-progress-after=300s --poll-progress-interval=60s", *ginkgoFlags)
fmt.Printf("Will use ginkgo flags as: %s", allGinkgoFlags)

var gceImages *internalImageConfig
if *mode == "gce" {
if *hosts == "" && *imageConfigFile == "" && *images == "" {
Expand Down Expand Up @@ -263,8 +265,9 @@ func main() {
imageConfig := gceImages.images[shortName]
fmt.Printf("Initializing e2e tests using image %s/%s/%s.\n", shortName, imageConfig.project, imageConfig.image)
running++

go func(image *internalGCEImage, junitFileName string) {
results <- testImage(image, junitFileName)
results <- testImage(image, junitFileName, allGinkgoFlags)
}(&imageConfig, shortName)
}
}
Expand All @@ -273,7 +276,7 @@ func main() {
fmt.Printf("Initializing e2e tests using host %s.\n", host)
running++
go func(host string, junitFileName string) {
results <- testHost(host, *cleanup, "", junitFileName, *ginkgoFlags)
results <- testHost(host, *cleanup, "", junitFileName, allGinkgoFlags)
}(host, host)
}
}
Expand All @@ -292,9 +295,9 @@ func main() {
fmt.Printf("%s\n", tr.output)
if tr.err != nil {
errCount++
fmt.Printf("Failure Finished Test Suite on Host %s\n%v\n", host, tr.err)
fmt.Printf("Failure Finished Test Suite on Host %s. Refer to artifacts directory for ginkgo log for this host.\n%v\n", host, tr.err)
} else {
fmt.Printf("Success Finished Test Suite on Host %s\n", host)
fmt.Printf("Success Finished Test Suite on Host %s. Refer to artifacts directory for ginkgo log for this host.\n", host)
}
exitOk = exitOk && tr.exitOk
fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour)
Expand Down Expand Up @@ -359,7 +362,6 @@ func prepareGceImages() (*internalImageConfig, error) {
metadata: getImageMetadata(metadata),
kernelArguments: imageConfig.KernelArguments,
machine: imageConfig.Machine,
tests: imageConfig.Tests,
resources: imageConfig.Resources,
}
if gceImage.imageDesc == "" {
Expand Down Expand Up @@ -536,18 +538,7 @@ func getGCEImage(imageRegex, imageFamily string, project string) (string, error)

// Provision a gce instance using image and run the tests in archive against the instance.
// Delete the instance afterward.
func testImage(imageConfig *internalGCEImage, junitFileName string) *TestResult {
ginkgoFlagsStr := *ginkgoFlags
// Check whether the test is for benchmark.
if len(imageConfig.tests) > 0 {
// Benchmark needs machine type non-empty.
if imageConfig.machine == "" {
imageConfig.machine = defaultMachine
}
// Use the Ginkgo focus in benchmark config.
ginkgoFlagsStr += (" " + testsToGinkgoFocus(imageConfig.tests))
}

func testImage(imageConfig *internalGCEImage, junitFileName string, ginkgoFlagsStr string) *TestResult {
host, err := createInstance(imageConfig)
if *deleteInstances {
defer deleteInstance(host)
Expand Down Expand Up @@ -963,16 +954,3 @@ func machineType(machine string) string {
}
return fmt.Sprintf("zones/%s/machineTypes/%s", *zone, machine)
}

// testsToGinkgoFocus converts the test string list to Ginkgo focus
func testsToGinkgoFocus(tests []string) string {
focus := "--focus=\""
for i, test := range tests {
if i == 0 {
focus += test
} else {
focus += ("|" + test)
}
}
return focus + "\""
}