From 26d8f7e5db7f5e901a1e4f7153995c97af194415 Mon Sep 17 00:00:00 2001 From: Dave Shanley Date: Thu, 14 Jul 2022 11:53:27 -0400 Subject: [PATCH] Created new custom functions code. Integrated into linting command, need to do the same for the other commands. Now anyone can build and load custom functions without needing to modify vacuum source code. --- cmd/lint.go | 10 +++- cmd/root.go | 1 + cmd/shared_functions.go | 18 +++++++ functions/functions.go | 19 +++++++- motor/rule_applicator.go | 16 ++++-- plugin/plugin_loader.go | 51 ++++++++++++++++++++ plugin/plugin_loader_test.go | 29 +++++++++++ plugin/plugin_manager.go | 33 +++++++++++++ plugin/plugin_manager_test.go | 16 ++++++ plugin/sample/boot.go | 15 ++++++ plugin/sample/sample_a.go | 32 ++++++++++++ plugin/sample/sample_b.go | 42 ++++++++++++++++ rulesets/examples/sample-plugin-ruleset.yaml | 21 ++++++++ 13 files changed, 296 insertions(+), 7 deletions(-) create mode 100644 plugin/plugin_loader.go create mode 100644 plugin/plugin_loader_test.go create mode 100644 plugin/plugin_manager.go create mode 100644 plugin/plugin_manager_test.go create mode 100644 plugin/sample/boot.go create mode 100644 plugin/sample/sample_a.go create mode 100644 plugin/sample/sample_b.go create mode 100644 rulesets/examples/sample-plugin-ruleset.yaml diff --git a/cmd/lint.go b/cmd/lint.go index 0a2c9a6a..66d75fb3 100644 --- a/cmd/lint.go +++ b/cmd/lint.go @@ -35,6 +35,8 @@ func GetLintCommand() *cobra.Command { rulesetFlag, _ := cmd.Flags().GetString("ruleset") silent, _ := cmd.Flags().GetBool("silent") + functionsFlag, _ := cmd.Flags().GetString("functions") + if !silent { PrintBanner() } @@ -65,6 +67,8 @@ func GetLintCommand() *cobra.Command { // default is recommended rules, based on spectral (for now anyway) selectedRS := defaultRuleSets.GenerateOpenAPIRecommendedRuleSet() + customFunctions, _ := LoadCustomFunctions(functionsFlag) + // if ruleset has been supplied, lets make sure it exists, then load it in // and see if it's valid. If so - let's go! if rulesetFlag != "" { @@ -75,6 +79,7 @@ func GetLintCommand() *cobra.Command { pterm.Println() return rsErr } + selectedRS, rsErr = BuildRuleSetFromUserSuppliedSet(rsBytes, defaultRuleSets) if rsErr != nil { return rsErr @@ -84,8 +89,9 @@ func GetLintCommand() *cobra.Command { pterm.Info.Printf("Linting against %d rules: %s\n", len(selectedRS.Rules), selectedRS.DocumentationURI) start := time.Now() result := motor.ApplyRulesToRuleSet(&motor.RuleSetExecution{ - RuleSet: selectedRS, - Spec: specBytes, + RuleSet: selectedRS, + Spec: specBytes, + CustomFunctions: customFunctions, }) results := result.Results diff --git a/cmd/root.go b/cmd/root.go index 3c7ff2c1..74e3f921 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -50,6 +50,7 @@ func init() { cobra.OnInitialize(initConfig) rootCmd.PersistentFlags().BoolP("time", "t", false, "Show how long vacuum took to run") rootCmd.PersistentFlags().StringP("ruleset", "r", "", "Path to a spectral ruleset configuration") + rootCmd.PersistentFlags().StringP("functions", "f", "", "Path to custom functions") rootCmd.AddCommand(GetLintCommand()) rootCmd.AddCommand(GetVacuumReportCommand()) diff --git a/cmd/shared_functions.go b/cmd/shared_functions.go index 2e85dee7..050171c2 100644 --- a/cmd/shared_functions.go +++ b/cmd/shared_functions.go @@ -5,6 +5,8 @@ package cmd import ( "fmt" + "github.com/daveshanley/vacuum/model" + "github.com/daveshanley/vacuum/plugin" "github.com/daveshanley/vacuum/rulesets" "github.com/pterm/pterm" "github.com/pterm/pterm/putils" @@ -43,3 +45,19 @@ func PrintBanner() { pterm.Printf("version: %s | compiled: %s\n\n", Version, Date) pterm.Println() } + +// LoadCustomFunctions will scan for (and load) custom functions defined as vacuum plugins. +func LoadCustomFunctions(functionsFlag string) (map[string]model.RuleFunction, error) { + // check custom functions + if functionsFlag != "" { + pm, err := plugin.LoadFunctions(functionsFlag) + if err != nil { + pterm.Error.Printf("Unable to open custom functions: %v\n", err) + pterm.Println() + return nil, err + } + pterm.Info.Printf("Loaded %d custom function(s) successfully.\n", pm.LoadedFunctionCount()) + return pm.GetCustomFunctions(), nil + } + return nil, nil +} diff --git a/functions/functions.go b/functions/functions.go index 9d169898..8085b039 100644 --- a/functions/functions.go +++ b/functions/functions.go @@ -7,11 +7,18 @@ import ( "github.com/daveshanley/vacuum/functions/core" openapi_functions "github.com/daveshanley/vacuum/functions/openapi" "github.com/daveshanley/vacuum/model" + "github.com/daveshanley/vacuum/plugin" "sync" ) +type customFunction struct { + functionHook plugin.FunctionHook + schemaHook plugin.FunctionSchema +} + type functionsModel struct { - functions map[string]model.RuleFunction + functions map[string]model.RuleFunction + customFunctions map[string]customFunction } // Functions is used to Query available functions loaded into vacuum @@ -32,13 +39,17 @@ func MapBuiltinFunctions() Functions { coreFunctionGrab.Do(func() { var funcs map[string]model.RuleFunction + var customFuncs map[string]customFunction if functionsSingleton != nil { funcs = functionsSingleton.functions + customFuncs = functionsSingleton.customFunctions } else { funcs = make(map[string]model.RuleFunction) + customFuncs = make(map[string]customFunction) functionsSingleton = &functionsModel{ - functions: funcs, + functions: funcs, + customFunctions: customFuncs, } } @@ -90,6 +101,10 @@ func MapBuiltinFunctions() Functions { return functionsSingleton } +func (fm functionsModel) RegisterCustomFunction(name string, function plugin.FunctionHook, schema plugin.FunctionSchema) { + fm.customFunctions[name] = customFunction{functionHook: function, schemaHook: schema} +} + func (fm functionsModel) GetAllFunctions() map[string]model.RuleFunction { return fm.functions } diff --git a/motor/rule_applicator.go b/motor/rule_applicator.go index 371f7262..8919606f 100644 --- a/motor/rule_applicator.go +++ b/motor/rule_applicator.go @@ -23,14 +23,15 @@ type ruleContext struct { errors *[]error index *model.SpecIndex specInfo *model.SpecInfo + customFunctions map[string]model.RuleFunction } // RuleSetExecution is an instruction set for executing a ruleset. It's a convenience structure to allow the signature // of ApplyRules to change, without a huge refactor. The ApplyRules function only returns a single error also. type RuleSetExecution struct { - RuleSet *rulesets.RuleSet // The RuleSet in which to apply - Spec []byte // The raw bytes of the OpenAPI specification. - + RuleSet *rulesets.RuleSet // The RuleSet in which to apply + Spec []byte // The raw bytes of the OpenAPI specification. + CustomFunctions map[string]model.RuleFunction // custom functions loaded from plugin. } // RuleSetExecutionResult returns the results of running the ruleset against the supplied spec. @@ -136,6 +137,7 @@ func ApplyRulesToRuleSet(execution *RuleSetExecution) *RuleSetExecutionResult { errors: &errors, index: index, specInfo: specInfo, + customFunctions: execution.CustomFunctions, } go runRule(ctx) } @@ -316,6 +318,14 @@ var lock sync.Mutex func buildResults(ctx ruleContext, ruleAction model.RuleAction, nodes []*yaml.Node) *[]model.RuleFunctionResult { ruleFunction := ctx.builtinFunctions.FindFunction(ruleAction.Function) + // not found, check if it's been registered as a custom function + if ruleFunction == nil { + if ctx.customFunctions != nil { + if ctx.customFunctions[ruleAction.Function] != nil { + ruleFunction = ctx.customFunctions[ruleAction.Function] + } + } + } if ruleFunction != nil { diff --git a/plugin/plugin_loader.go b/plugin/plugin_loader.go new file mode 100644 index 00000000..87c1e8f5 --- /dev/null +++ b/plugin/plugin_loader.go @@ -0,0 +1,51 @@ +package plugin + +import ( + "github.com/pterm/pterm" + "io/ioutil" + "path/filepath" + "plugin" + "strings" +) + +// LoadFunctions will load custom functions found in the supplied path +func LoadFunctions(path string) (*Manager, error) { + + dirEntries, err := ioutil.ReadDir(path) + if err != nil { + return nil, err + } + + pm := createPluginManager() + + for _, entry := range dirEntries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".so") { + fPath := filepath.Join(path, entry.Name()) + + // found something + pterm.Info.Printf("Located custom function plugin: %s\n", fPath) + + // let's try and open it. + p, e := plugin.Open(fPath) + if e != nil { + return nil, e + } + + // look up the Boot function and store as a Symbol + var bootFunc plugin.Symbol + bootFunc, e = p.Lookup("Boot") + if err != nil { + return nil, err + } + + // lets go pedro! + if bootFunc != nil { + bootFunc.(func(*Manager))(pm) + } else { + pterm.Error.Printf("Unable to boot plugin") + } + } + } + + return pm, nil +} diff --git a/plugin/plugin_loader_test.go b/plugin/plugin_loader_test.go new file mode 100644 index 00000000..a072bf78 --- /dev/null +++ b/plugin/plugin_loader_test.go @@ -0,0 +1,29 @@ +package plugin + +import ( + "github.com/stretchr/testify/assert" + "runtime" + "testing" +) + +func TestLoadFunctions_Nowhere(t *testing.T) { + pm, err := LoadFunctions("nowhere") + assert.Nil(t, pm) + assert.Error(t, err) +} + +func TestLoadFunctions(t *testing.T) { + pm, err := LoadFunctions("../model/test_files") + assert.NotNil(t, pm) + assert.NoError(t, err) + assert.Equal(t, 0, pm.LoadedFunctionCount()) +} + +func TestLoadFunctions_Sample(t *testing.T) { + pm, err := LoadFunctions("sample") + if runtime.GOOS != "windows" { // windows does not support this feature, at all. + assert.NotNil(t, pm) + assert.NoError(t, err) + assert.Equal(t, 0, pm.LoadedFunctionCount()) + } +} diff --git a/plugin/plugin_manager.go b/plugin/plugin_manager.go new file mode 100644 index 00000000..e66ca426 --- /dev/null +++ b/plugin/plugin_manager.go @@ -0,0 +1,33 @@ +package plugin + +import ( + "github.com/daveshanley/vacuum/model" + "gopkg.in/yaml.v3" +) + +type FunctionSchema func() model.RuleFunctionSchema +type FunctionHook func(nodes []*yaml.Node, context model.RuleFunctionContext) []model.RuleFunctionResult + +type Manager struct { + customFunctions map[string]model.RuleFunction +} + +func createPluginManager() *Manager { + return &Manager{ + customFunctions: make(map[string]model.RuleFunction), + } +} + +// RegisterFunction allows a custom function to be hooked in +func (pm *Manager) RegisterFunction(name string, ruleFunction model.RuleFunction) { + pm.customFunctions[name] = ruleFunction +} + +// LoadedFunctionCount returns the number of available and ready to use functions. +func (pm *Manager) LoadedFunctionCount() int { + return len(pm.customFunctions) +} + +func (pm *Manager) GetCustomFunctions() map[string]model.RuleFunction { + return pm.customFunctions +} diff --git a/plugin/plugin_manager_test.go b/plugin/plugin_manager_test.go new file mode 100644 index 00000000..4b3e6d0c --- /dev/null +++ b/plugin/plugin_manager_test.go @@ -0,0 +1,16 @@ +package plugin + +import ( + "github.com/daveshanley/vacuum/functions/core" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestPluginManager_RegisterFunction(t *testing.T) { + + pm := createPluginManager() + + pm.RegisterFunction("defined", core.Defined{}) + assert.Len(t, pm.GetCustomFunctions(), 1) + +} diff --git a/plugin/sample/boot.go b/plugin/sample/boot.go new file mode 100644 index 00000000..981da72e --- /dev/null +++ b/plugin/sample/boot.go @@ -0,0 +1,15 @@ +package main + +import "github.com/daveshanley/vacuum/plugin" + +// Boot is called by the Manager when the module is located. +// all custom functions should be registered here. +func Boot(pm *plugin.Manager) { + + sampleA := SampleRuleFunction_A{} + sampleB := SampleRuleFunction_B{} + + // register custom functions with vacuum plugin manager. + pm.RegisterFunction(sampleA.GetSchema().Name, sampleA) + pm.RegisterFunction(sampleB.GetSchema().Name, sampleB) +} diff --git a/plugin/sample/sample_a.go b/plugin/sample/sample_a.go new file mode 100644 index 00000000..f2224230 --- /dev/null +++ b/plugin/sample/sample_a.go @@ -0,0 +1,32 @@ +package main + +import ( + "github.com/daveshanley/vacuum/model" + "gopkg.in/yaml.v3" +) + +// SampleRuleFunction_A is an example custom rule that does nothing. +type SampleRuleFunction_A struct { +} + +// GetSchema returns a model.RuleFunctionSchema defining the schema of the Defined rule. +func (s SampleRuleFunction_A) GetSchema() model.RuleFunctionSchema { + return model.RuleFunctionSchema{ + Name: "uselessFunc", + } +} + +// RunRule will execute the Sample rule, based on supplied context and a supplied []*yaml.Node slice. +func (s SampleRuleFunction_A) RunRule(nodes []*yaml.Node, context model.RuleFunctionContext) []model.RuleFunctionResult { + + // return a single result, for a made up linting failure. + return []model.RuleFunctionResult{ + { + Message: "this is a useless function that will always error out.", + StartNode: &yaml.Node{Line: 1, Column: 0}, + EndNode: &yaml.Node{Line: 2, Column: 0}, + Path: "$.i.do.not.exist", + Rule: context.Rule, + }, + } +} diff --git a/plugin/sample/sample_b.go b/plugin/sample/sample_b.go new file mode 100644 index 00000000..62c9505f --- /dev/null +++ b/plugin/sample/sample_b.go @@ -0,0 +1,42 @@ +package main + +import ( + "fmt" + "github.com/daveshanley/vacuum/model" + "gopkg.in/yaml.v3" +) + +// SampleRuleFunction_B is an example custom rule that checks only a single path exists. +type SampleRuleFunction_B struct { +} + +// GetSchema returns a model.RuleFunctionSchema defining the schema of the Defined rule. +func (s SampleRuleFunction_B) GetSchema() model.RuleFunctionSchema { + return model.RuleFunctionSchema{ + Name: "checkSinglePathExists", + } +} + +// RunRule will execute the Sample rule, based on supplied context and a supplied []*yaml.Node slice. +func (s SampleRuleFunction_B) RunRule(nodes []*yaml.Node, context model.RuleFunctionContext) []model.RuleFunctionResult { + + // get the index https://quobix.com/vacuum/api/spec-index/ + index := context.Index + + // get the paths node from the index. + paths := index.GetPathsNode() + + // checks if there are more than two nodes present in the paths node, if so, more than one path is present. + if len(paths.Content) > 2 { + return []model.RuleFunctionResult{ + { + Message: fmt.Sprintf("more than a single path exists, there are %v", len(paths.Content)/2), + StartNode: paths, + EndNode: paths, + Path: "$.paths", + Rule: context.Rule, + }, + } + } + return nil +} diff --git a/rulesets/examples/sample-plugin-ruleset.yaml b/rulesets/examples/sample-plugin-ruleset.yaml new file mode 100644 index 00000000..43aed6fe --- /dev/null +++ b/rulesets/examples/sample-plugin-ruleset.yaml @@ -0,0 +1,21 @@ +extends: [[spectral:oas, off]] +documentationUrl: https://quobix.com/vacuum/rulesets/custom-rulesets +rules: + sample-plugin-rule: + description: Load a custom function that does nothing useful + severity: error + recommended: true + formats: [oas2, oas3] + given: $ + then: + function: uselessFunc + howToFix: You can't, it's just an example. + sample-paths-rule: + description: Load a custom function that checks for a single path + severity: error + recommended: true + formats: [ oas2, oas3 ] + given: $ + then: + function: checkSinglePathExists + howToFix: use a spec with only a single path defined. \ No newline at end of file