From e320523e142f08e105a14ace1a6cb183119b0615 Mon Sep 17 00:00:00 2001 From: Ilya Date: Tue, 9 Aug 2022 15:44:06 +0100 Subject: [PATCH] Add variables tests step Add e2e TestVariables that checks step variables Signed-off-by: Ilya --- pkg/runner/test_steps_variables.go | 21 ++- plugins/teststeps/variables/readme.md | 33 ++++ plugins/teststeps/variables/variables.go | 78 ++++++++ plugins/teststeps/variables/variables_test.go | 175 ++++++++++++++++++ tests/e2e/e2e_test.go | 92 +++++---- tests/e2e/test-variables.yaml | 34 ++++ 6 files changed, 392 insertions(+), 41 deletions(-) create mode 100644 plugins/teststeps/variables/readme.md create mode 100644 plugins/teststeps/variables/variables.go create mode 100644 plugins/teststeps/variables/variables_test.go create mode 100644 tests/e2e/test-variables.yaml diff --git a/pkg/runner/test_steps_variables.go b/pkg/runner/test_steps_variables.go index 7836c6e6..5263c540 100644 --- a/pkg/runner/test_steps_variables.go +++ b/pkg/runner/test_steps_variables.go @@ -126,14 +126,21 @@ func newStepVariablesAccessor(stepLabel string, tsv *testStepsVariables) *stepVa } func (sva *stepVariablesAccessor) Add(tgtID string, name string, in interface{}) error { - if len(sva.stepLabel) == 0 { - return nil - } - b, err := json.Marshal(in) - if err != nil { - return fmt.Errorf("failed to serialize variable: %v", in) + var marshalled []byte + if raw, ok := in.(json.RawMessage); ok { + var v interface{} + if err := json.Unmarshal(raw, &v); err != nil { + return fmt.Errorf("invalid input, failed to unmarshal: %v", err) + } + marshalled = raw + } else { + var err error + marshalled, err = json.Marshal(in) + if err != nil { + return fmt.Errorf("failed to serialize variable: %v", in) + } } - return sva.tsv.Add(tgtID, sva.stepLabel, name, b) + return sva.tsv.Add(tgtID, sva.stepLabel, name, marshalled) } func (sva *stepVariablesAccessor) Get(tgtID string, stepLabel, name string, out interface{}) error { diff --git a/plugins/teststeps/variables/readme.md b/plugins/teststeps/variables/readme.md new file mode 100644 index 00000000..5aa253a1 --- /dev/null +++ b/plugins/teststeps/variables/readme.md @@ -0,0 +1,33 @@ +# Variables plugin + +The *variables* plugin adds its input parameters as step plugin variables that could be later referred to by other plugins. + +## Parameters + +Any parameter should be a single-value parameter that will be added as a test-variable. +For example: + +{ + "name": "variables", + "label": "variablesstep" + "parameters": { + "string_variable": ["Hello"], + "int_variable": [123], + "complex_variable": [{"hello": "world"}] + } +} + +Will generate a string, int and a json-object variables respectively. + +These parameters could be later accessed in the following manner in accordance with step variables guidance: + +{ + "name": "cmd", + "label": "cmdstep", + "parameters": { + "executable": [echo], + "args": ["{{ StringVar \"variablesstep.string_variable\" }} world number {{ IntVar \"variablesstep.int_variable\" }}"], + "emit_stdout": [true], + "emit_stderr": [true] + } +} diff --git a/plugins/teststeps/variables/variables.go b/plugins/teststeps/variables/variables.go new file mode 100644 index 00000000..f0c00d05 --- /dev/null +++ b/plugins/teststeps/variables/variables.go @@ -0,0 +1,78 @@ +package variables + +import ( + "encoding/json" + "fmt" + "github.com/linuxboot/contest/pkg/event" + "github.com/linuxboot/contest/pkg/event/testevent" + "github.com/linuxboot/contest/pkg/target" + "github.com/linuxboot/contest/pkg/test" + "github.com/linuxboot/contest/pkg/xcontext" + "github.com/linuxboot/contest/plugins/teststeps" +) + +// Name is the name used to look this plugin up. +const Name = "variables" + +// Events defines the events that a TestStep is allowed to emit +var Events []event.Name + +// Variables creates variables that can be used by other test steps +type Variables struct { +} + +// Name returns the plugin name. +func (ts *Variables) Name() string { + return Name +} + +// Run executes the cmd step. +func (ts *Variables) Run( + ctx xcontext.Context, + ch test.TestStepChannels, + ev testevent.Emitter, + stepsVars test.StepsVariables, + inputParams test.TestStepParameters, + resumeState json.RawMessage, +) (json.RawMessage, error) { + if err := ts.ValidateParameters(ctx, inputParams); err != nil { + return nil, err + } + return teststeps.ForEachTarget(Name, ctx, ch, func(ctx xcontext.Context, target *target.Target) error { + for name, ps := range inputParams { + ctx.Debugf("add variable %s, value: %s", name, ps[0]) + if err := stepsVars.Add(target.ID, name, ps[0].RawMessage); err != nil { + return err + } + } + return nil + }) +} + +// ValidateParameters validates the parameters associated to the TestStep +func (ts *Variables) ValidateParameters(ctx xcontext.Context, params test.TestStepParameters) error { + for name, ps := range params { + if err := test.CheckIdentifier(name); err != nil { + return fmt.Errorf("invalid variable name: '%s': %w", name, err) + } + if len(ps) != 1 { + return fmt.Errorf("invalid number of parameter '%s' values: %d (expected 1)", name, len(ps)) + } + + var res interface{} + if err := json.Unmarshal(ps[0].RawMessage, &res); err != nil { + return fmt.Errorf("invalid json '%s': %w", ps[0].RawMessage, err) + } + } + return nil +} + +// New initializes and returns a new Variables test step. +func New() test.TestStep { + return &Variables{} +} + +// Load returns the name, factory and events which are needed to register the step. +func Load() (string, test.TestStepFactory, []event.Name) { + return Name, New, Events +} diff --git a/plugins/teststeps/variables/variables_test.go b/plugins/teststeps/variables/variables_test.go new file mode 100644 index 00000000..ce7a7698 --- /dev/null +++ b/plugins/teststeps/variables/variables_test.go @@ -0,0 +1,175 @@ +package variables + +import ( + "encoding/json" + "fmt" + "sync" + "testing" + + "github.com/linuxboot/contest/pkg/event/testevent" + "github.com/linuxboot/contest/pkg/storage" + "github.com/linuxboot/contest/pkg/target" + "github.com/linuxboot/contest/pkg/test" + "github.com/linuxboot/contest/pkg/xcontext" + "github.com/linuxboot/contest/plugins/storage/memory" + "github.com/stretchr/testify/require" +) + +func TestCreation(t *testing.T) { + obj := New() + require.NotNil(t, obj) + require.Equal(t, Name, obj.Name()) +} + +func TestValidateParameters(t *testing.T) { + obj := New() + require.NotNil(t, obj) + + require.NoError(t, obj.ValidateParameters(xcontext.Background(), nil)) + require.NoError(t, obj.ValidateParameters(xcontext.Background(), test.TestStepParameters{ + "var1": []test.Param{ + { + RawMessage: json.RawMessage("123"), + }, + }, + })) + // invalid variable name + require.Error(t, obj.ValidateParameters(xcontext.Background(), test.TestStepParameters{ + "var var": []test.Param{ + { + RawMessage: json.RawMessage("123"), + }, + }, + })) + // invalid value + require.Error(t, obj.ValidateParameters(xcontext.Background(), test.TestStepParameters{ + "var1": []test.Param{ + { + RawMessage: json.RawMessage("ALALALALA[}"), + }, + }, + })) +} + +func TestVariablesEmission(t *testing.T) { + ctx, cancel := xcontext.WithCancel(xcontext.Background()) + defer cancel() + + obj := New() + require.NotNil(t, obj) + + in := make(chan *target.Target, 1) + out := make(chan test.TestStepResult, 1) + + m, err := memory.New() + if err != nil { + t.Fatalf("could not initialize memory storage: '%v'", err) + } + storageEngineVault := storage.NewSimpleEngineVault() + if err := storageEngineVault.StoreEngine(m, storage.SyncEngine); err != nil { + t.Fatalf("Failed to set memory storage: '%v'", err) + } + ev := storage.NewTestEventEmitterFetcher(storageEngineVault, testevent.Header{ + JobID: 12345, + TestName: "variables_tests", + TestStepLabel: "variables", + }) + + svm := newStepsVariablesMock() + + tgt := target.Target{ID: "id1"} + in <- &tgt + close(in) + + state, err := obj.Run(ctx, test.TestStepChannels{In: in, Out: out}, ev, svm, test.TestStepParameters{ + "str_variable": []test.Param{ + { + RawMessage: json.RawMessage("\"dummy\""), + }, + }, + "int_variable": []test.Param{ + { + RawMessage: json.RawMessage("123"), + }, + }, + "complex_variable": []test.Param{ + { + RawMessage: json.RawMessage("{\"name\":\"value\"}"), + }, + }, + }, nil) + require.NoError(t, err) + require.Empty(t, state) + + stepResult := <-out + require.Equal(t, tgt, *stepResult.Target) + require.NoError(t, stepResult.Err) + + var strVar string + require.NoError(t, svm.get(tgt.ID, "str_variable", &strVar)) + require.Equal(t, "dummy", strVar) + + var intVar int + require.NoError(t, svm.get(tgt.ID, "int_variable", &intVar)) + require.Equal(t, 123, intVar) + + var complexVar dummyStruct + require.NoError(t, svm.get(tgt.ID, "complex_variable", &complexVar)) + require.Equal(t, dummyStruct{Name: "value"}, complexVar) +} + +type dummyStruct struct { + Name string `json:"name"` +} + +type stepsVariablesMock struct { + mu sync.Mutex + variables map[string]map[string]json.RawMessage +} + +func newStepsVariablesMock() *stepsVariablesMock { + return &stepsVariablesMock{ + variables: make(map[string]map[string]json.RawMessage), + } +} + +func (svm *stepsVariablesMock) AddRaw(tgtID string, name string, b json.RawMessage) error { + svm.mu.Lock() + defer svm.mu.Unlock() + + targetVars := svm.variables[tgtID] + if targetVars == nil { + targetVars = make(map[string]json.RawMessage) + svm.variables[tgtID] = targetVars + } + targetVars[name] = b + return nil +} + +func (svm *stepsVariablesMock) Add(tgtID string, name string, v interface{}) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + + return svm.AddRaw(tgtID, name, b) +} + +func (svm *stepsVariablesMock) Get(tgtID string, stepLabel, name string, out interface{}) error { + panic("not implemented") +} + +func (svm *stepsVariablesMock) get(tgtID string, name string, out interface{}) error { + svm.mu.Lock() + defer svm.mu.Unlock() + + targetVars := svm.variables[tgtID] + if targetVars == nil { + return fmt.Errorf("no target: %s", tgtID) + } + b, found := targetVars[name] + if !found { + return fmt.Errorf("no variable: %s", name) + } + return json.Unmarshal(b, out) +} diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index ae6b2a53..a7de45e1 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -45,6 +45,7 @@ import ( "github.com/linuxboot/contest/plugins/testfetchers/literal" "github.com/linuxboot/contest/plugins/teststeps/cmd" "github.com/linuxboot/contest/plugins/teststeps/sleep" + "github.com/linuxboot/contest/plugins/teststeps/variables" "github.com/linuxboot/contest/plugins/teststeps/waitport" testsCommon "github.com/linuxboot/contest/tests/common" "github.com/linuxboot/contest/tests/common/goroutine_leak_check" @@ -122,7 +123,7 @@ func (ts *E2ETestSuite) startServer(extraArgs ...string) { targetlist_with_state.Load, }, TestFetcherLoaders: []test.TestFetcherLoader{literal.Load}, - TestStepLoaders: []test.TestStepLoader{cmd.Load, sleep.Load, waitport.Load}, + TestStepLoaders: []test.TestStepLoader{cmd.Load, sleep.Load, waitport.Load, variables.Load}, ReporterLoaders: []job.ReporterLoader{targetsuccess.Load, noop.Load}, } err := server.Main(&pc, "contest", args, serverSigs) @@ -149,6 +150,22 @@ func (ts *E2ETestSuite) startServer(extraArgs ...string) { } } +func (ts *E2ETestSuite) startJob(descriptorFile string) types.JobID { + // No jobs to begin with. + var listResp api.ListResponse + _, err := ts.runClient(&listResp, "list") + require.NoError(ts.T(), err) + require.Empty(ts.T(), listResp.Data.JobIDs) + + // Start a job. + var resp api.StartResponse + _, err = ts.runClient(&resp, "start", "-Y", descriptorFile) + require.NoError(ts.T(), err) + ctx.Infof("%+v", resp) + require.NotEqual(ts.T(), 0, resp.Data.JobID) + return resp.Data.JobID +} + func (ts *E2ETestSuite) stopServer(timeout time.Duration) error { if ts.serverSigs == nil { return nil @@ -217,22 +234,8 @@ func (ts *E2ETestSuite) TestCLIErrors() { } func (ts *E2ETestSuite) TestSimple() { - var jobID types.JobID ts.startServer() - { // No jobs to begin with. - var resp api.ListResponse - _, err := ts.runClient(&resp, "list") - require.NoError(ts.T(), err) - require.Empty(ts.T(), resp.Data.JobIDs) - } - { // Start a job. - var resp api.StartResponse - _, err := ts.runClient(&resp, "start", "-Y", "test-simple.yaml") - require.NoError(ts.T(), err) - ctx.Infof("%+v", resp) - require.NotEqual(ts.T(), 0, resp.Data.JobID) - jobID = resp.Data.JobID - } + jobID := ts.startJob("test-simple.yaml") { // Wait for the job to finish var resp api.StatusResponse for i := 1; i < 5; i++ { @@ -278,17 +281,46 @@ func (ts *E2ETestSuite) TestSimple() { require.NoError(ts.T(), ts.stopServer(5*time.Second)) } +func (ts *E2ETestSuite) TestVariables() { + ts.startServer() + jobID := ts.startJob("test-variables.yaml") + + { // Wait for the job to finish + var resp api.StatusResponse + for i := 1; i < 5; i++ { + time.Sleep(1 * time.Second) + stdout, err := ts.runClient(&resp, "status", fmt.Sprintf("%d", jobID)) + require.NoError(ts.T(), err) + require.Nil(ts.T(), resp.Err, "error: %s", resp.Err) + ctx.Infof("Job %d state %s", jobID, resp.Data.Status.State) + if resp.Data.Status.State == string(job.EventJobCompleted) { + ctx.Debugf("Job %d status: %s", jobID, stdout) + break + } + } + require.Equal(ts.T(), string(job.EventJobCompleted), resp.Data.Status.State) + } + { // Verify step output. + es := testsCommon.GetJobEventsAsString(ctx, ts.st, jobID, []event.Name{ + cmd.EventCmdStdout, target.EventTargetAcquired, target.EventTargetReleased, + }) + ctx.Debugf("%s", es) + require.Equal(ts.T(), + fmt.Sprintf(` +{[%d 1 Test 1 0 ][Target{ID: "T1"} TargetAcquired]} +{[%d 1 Test 1 0 cmdstep][Target{ID: "T1"} CmdStdout &"{\"Msg\":\"Hello\\n\"}"]} +{[%d 1 Test 1 0 ][Target{ID: "T1"} TargetReleased]} +`, jobID, jobID, jobID), + es, + ) + } + require.NoError(ts.T(), ts.stopServer(5*time.Second)) +} + func (ts *E2ETestSuite) TestPauseResume() { - var jobID types.JobID ts.startServer("--pauseTimeout=60s", "--resumeJobs") - { // Start a job. - var resp api.StartResponse - _, err := ts.runClient(&resp, "start", "-Y", "test-resume.yaml") - require.NoError(ts.T(), err) - ctx.Infof("%+v", resp) - require.NotEqual(ts.T(), 0, resp.Data.JobID) - jobID = resp.Data.JobID - } + jobID := ts.startJob("test-resume.yaml") + start := time.Now() { // Stop/start the server up to 20 times or until the job completes. var resp api.StatusResponse @@ -371,15 +403,7 @@ func (ts *E2ETestSuite) TestRetries() { }() require.NoError(ts.T(), templ.Execute(tmpFile, testDescriptorCustomisation{WaitPort: waitPort})) - var jobID types.JobID - { // Start a job. - var resp api.StartResponse - _, err := ts.runClient(&resp, "start", "-Y", tmpFile.Name()) - require.NoError(ts.T(), err) - ctx.Infof("%+v", resp) - require.NotEqual(ts.T(), 0, resp.Data.JobID) - jobID = resp.Data.JobID - } + jobID := ts.startJob(tmpFile.Name()) <-time.After(5 * time.Second) listener, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", waitPort)) diff --git a/tests/e2e/test-variables.yaml b/tests/e2e/test-variables.yaml new file mode 100644 index 00000000..f21da889 --- /dev/null +++ b/tests/e2e/test-variables.yaml @@ -0,0 +1,34 @@ +JobName: A variables test job +Runs: 1 +RunInterval: 1s +Tags: + - test + - variables +TestDescriptors: + - TargetManagerName: TargetList + TargetManagerAcquireParameters: + Targets: + - ID: T1 + TargetManagerReleaseParameters: + TestFetcherName: literal + TestFetcherFetchParameters: + TestName: Test 1 + Steps: + - name: variables + label: variablesstep + parameters: + message: ["Hello"] + - name: cmd + label: cmdstep + parameters: + executable: [echo] + args: ["{{ StringVar \"variablesstep.message\" }}"] + emit_stdout: [true] + emit_stderr: [true] +Reporting: + RunReporters: + - name: TargetSuccess + parameters: + SuccessExpression: "=100%" + FinalReporters: + - name: noop \ No newline at end of file