adding global cache clear + shortening var names

v2/cmd/integration-test/code.go

@@ -29,6 +29,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	templatesCache "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/ratelimit"
@@ -95,22 +96,24 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
 	catalog := disk.NewCatalog(path.Join(home, "nuclei-templates"))
 	ratelimiter := ratelimit.New(context.Background(), 150, time.Second)
 	defer ratelimiter.Stop()
-	executerOpts := protocols.ExecuterOptions{
-		Output:          outputWriter,
-		Options:         defaultOpts,
-		Progress:        mockProgress,
-		Catalog:         catalog,
-		IssuesClient:    reportingClient,
-		RateLimiter:     ratelimiter,
-		Interactsh:      interactClient,
-		HostErrorsCache: cache,
-		Colorizer:       aurora.NewAurora(true),
-		ResumeCfg:       types.NewResumeCfg(),
+	executerOpts := &protocols.ExecuterOptions{
+		Output:                    outputWriter,
+		Options:                   defaultOpts,
+		Progress:                  mockProgress,
+		Catalog:                   catalog,
+		IssuesClient:              reportingClient,
+		RateLimiter:               ratelimiter,
+		Interactsh:                interactClient,
+		HostErrorsCache:           cache,
+		Colorizer:                 aurora.NewAurora(true),
+		ResumeCfg:                 types.NewResumeCfg(),
+		CompiledTemplatesCache:    templatesCache.New(),
+		UnmarshaledTemplatesCache: templatesCache.New(),
 	}
 	engine := core.New(defaultOpts)
 	engine.SetExecuterOptions(executerOpts)
 
-	workflowLoader, err := parsers.NewLoader(&executerOpts)
+	workflowLoader, err := parsers.NewLoader(executerOpts)
 	if err != nil {
 		log.Fatalf("Could not create workflow loader: %s\n", err)
 	}

v2/examples/simple.go

@@ -25,6 +25,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	templatesCache "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/ratelimit"
@@ -59,22 +60,24 @@ func main() {
 
 	home, _ := os.UserHomeDir()
 	catalog := disk.NewCatalog(path.Join(home, "nuclei-templates"))
-	executerOpts := protocols.ExecuterOptions{
-		Output:          outputWriter,
-		Options:         defaultOpts,
-		Progress:        mockProgress,
-		Catalog:         catalog,
-		IssuesClient:    reportingClient,
-		RateLimiter:     ratelimit.New(context.Background(), 150, time.Second),
-		Interactsh:      interactClient,
-		HostErrorsCache: cache,
-		Colorizer:       aurora.NewAurora(true),
-		ResumeCfg:       types.NewResumeCfg(),
+	executerOpts := &protocols.ExecuterOptions{
+		Output:                    outputWriter,
+		Options:                   defaultOpts,
+		Progress:                  mockProgress,
+		Catalog:                   catalog,
+		IssuesClient:              reportingClient,
+		RateLimiter:               ratelimit.New(context.Background(), 150, time.Second),
+		Interactsh:                interactClient,
+		HostErrorsCache:           cache,
+		Colorizer:                 aurora.NewAurora(true),
+		ResumeCfg:                 types.NewResumeCfg(),
+		CompiledTemplatesCache:    templatesCache.New(),
+		UnmarshaledTemplatesCache: templatesCache.New(),
 	}
 	engine := core.New(defaultOpts)
 	engine.SetExecuterOptions(executerOpts)
 
-	workflowLoader, err := parsers.NewLoader(&executerOpts)
+	workflowLoader, err := parsers.NewLoader(executerOpts)
 	if err != nil {
 		log.Fatalf("Could not create workflow loader: %s\n", err)
 	}

v2/internal/runner/enumerate.go

@@ -27,7 +27,7 @@ import (
 )
 
 // runStandardEnumeration runs standard enumeration
-func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
+func (r *Runner) runStandardEnumeration(executerOpts *protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
 	if r.options.AutomaticScan {
 		return r.executeSmartWorkflowInput(executerOpts, store, engine)
 	}

v2/internal/runner/runner.go

@@ -51,6 +51,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
@@ -80,6 +81,7 @@ type Runner struct {
 	customTemplates   []customtemplates.Provider
 	cloudClient       *nucleicloud.Client
 	cloudTargets      []string
+	executerOptions   *protocols.ExecuterOptions
 }
 
 const pprofServerAddress = "127.0.0.1:8086"
@@ -347,6 +349,9 @@ func (r *Runner) Close() {
 	if r.ratelimiter != nil {
 		r.ratelimiter.Stop()
 	}
+	if r.executerOptions != nil {
+		r.executerOptions = nil
+	}
 }
 
 // RunEnumeration sets up the input layer for giving input nuclei.
@@ -395,21 +400,25 @@ func (r *Runner) RunEnumeration() error {
 
 	// Create the executer options which will be used throughout the execution
 	// stage by the nuclei engine modules.
-	executerOpts := protocols.ExecuterOptions{
-		Output:          r.output,
-		Options:         r.options,
-		Progress:        r.progress,
-		Catalog:         r.catalog,
-		IssuesClient:    r.issuesClient,
-		RateLimiter:     r.ratelimiter,
-		Interactsh:      r.interactsh,
-		ProjectFile:     r.projectFile,
-		Browser:         r.browser,
-		Colorizer:       r.colorizer,
-		ResumeCfg:       r.resumeCfg,
-		ExcludeMatchers: excludematchers.New(r.options.ExcludeMatchers),
-		InputHelper:     input.NewHelper(),
-	}
+	executerOpts := &protocols.ExecuterOptions{
+		Output:                    r.output,
+		Options:                   r.options,
+		Progress:                  r.progress,
+		Catalog:                   r.catalog,
+		IssuesClient:              r.issuesClient,
+		RateLimiter:               r.ratelimiter,
+		Interactsh:                r.interactsh,
+		ProjectFile:               r.projectFile,
+		Browser:                   r.browser,
+		Colorizer:                 r.colorizer,
+		ResumeCfg:                 r.resumeCfg,
+		ExcludeMatchers:           excludematchers.New(r.options.ExcludeMatchers),
+		InputHelper:               input.NewHelper(),
+		CompiledTemplatesCache:    cache.New(),
+		UnmarshaledTemplatesCache: cache.New(),
+	}
+
+	r.executerOptions = executerOpts
 
 	if r.options.ShouldUseHostError() {
 		cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount)
@@ -421,7 +430,7 @@ func (r *Runner) RunEnumeration() error {
 	engine := core.New(r.options)
 	engine.SetExecuterOptions(executerOpts)
 
-	workflowLoader, err := parsers.NewLoader(&executerOpts)
+	workflowLoader, err := parsers.NewLoader(executerOpts)
 	if err != nil {
 		return errors.Wrap(err, "Could not create loader.")
 	}
@@ -586,7 +595,7 @@ func (r *Runner) isInputNonHTTP() bool {
 	return nonURLInput
 }
 
-func (r *Runner) executeSmartWorkflowInput(executerOpts protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
+func (r *Runner) executeSmartWorkflowInput(executerOpts *protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
 	r.progress.Init(r.hmapInputProvider.Count(), 0, 0)
 
 	service, err := automaticscan.New(automaticscan.Options{

v2/internal/runner/templates.go

@@ -18,7 +18,7 @@ import (
 
 // log available templates for verbose (-vv)
 func (r *Runner) logAvailableTemplate(tplPath string) {
-	t, err := parsers.ParseTemplate(tplPath, r.catalog)
+	t, err := parsers.ParseTemplate(tplPath, r.executerOptions)
 	if err != nil {
 		gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
 	} else {

v2/pkg/catalog/loader/loader.go

@@ -42,7 +42,7 @@ type Config struct {
 	IncludeConditions []string
 
 	Catalog            catalog.Catalog
-	ExecutorOptions    protocols.ExecuterOptions
+	ExecutorOptions    *protocols.ExecuterOptions
 	TemplatesDirectory string
 }
 
@@ -65,7 +65,7 @@ type Store struct {
 }
 
 // NewConfig returns a new loader config
-func NewConfig(options *types.Options, templateConfig *config.Config, catalog catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config {
+func NewConfig(options *types.Options, templateConfig *config.Config, catalog catalog.Catalog, executerOpts *protocols.ExecuterOptions) *Config {
 	loaderConfig := Config{
 		Templates:                options.Templates,
 		Workflows:                options.Workflows,
@@ -194,13 +194,13 @@ func (store *Store) ValidateTemplates() error {
 
 func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool {
 	return areWorkflowOrTemplatesValid(store, filteredWorkflowPaths, true, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
-		return parsers.LoadWorkflow(templatePath, store.config.Catalog)
+		return parsers.LoadWorkflow(templatePath, store.config.ExecutorOptions)
 	})
 }
 
 func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool {
 	return areWorkflowOrTemplatesValid(store, filteredTemplatePaths, false, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
-		return parsers.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
+		return parsers.LoadTemplate(templatePath, store.tagFilter, nil, store.config.ExecutorOptions)
 	})
 }
 
@@ -275,7 +275,7 @@ func (store *Store) LoadTemplates(templatesList []string) []*templates.Template
 
 	loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
 	for templatePath := range templatePathMap {
-		loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
+		loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil, store.config.ExecutorOptions)
 		if loaded || store.pathFilter.MatchIncluded(templatePath) {
 			parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
 			if err != nil {
@@ -308,7 +308,7 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template
 
 	loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
 	for workflowPath := range workflowPathMap {
-		loaded, err := parsers.LoadWorkflow(workflowPath, store.config.Catalog)
+		loaded, err := parsers.LoadWorkflow(workflowPath, store.config.ExecutorOptions)
 		if err != nil {
 			gologger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
 		}
@@ -333,7 +333,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
 
 	loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
 	for templatePath := range templatePathMap {
-		loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog)
+		loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, tags, store.config.ExecutorOptions)
 		if loaded || store.pathFilter.MatchIncluded(templatePath) {
 			parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
 			if err != nil {

v2/pkg/core/engine.go

@@ -18,7 +18,7 @@ import (
 type Engine struct {
 	workPool     *WorkPool
 	options      *types.Options
-	executerOpts protocols.ExecuterOptions
+	executerOpts *protocols.ExecuterOptions
 	Callback     func(*output.ResultEvent) // Executed on results
 }
 
@@ -58,12 +58,12 @@ func (e *Engine) GetWorkPool() *WorkPool {
 
 // SetExecuterOptions sets the executer options for the engine. This is required
 // before using the engine to perform any execution.
-func (e *Engine) SetExecuterOptions(options protocols.ExecuterOptions) {
+func (e *Engine) SetExecuterOptions(options *protocols.ExecuterOptions) {
 	e.executerOpts = options
 }
 
 // ExecuterOptions returns protocols.ExecuterOptions for nuclei engine.
-func (e *Engine) ExecuterOptions() protocols.ExecuterOptions {
+func (e *Engine) ExecuterOptions() *protocols.ExecuterOptions {
 	return e.executerOpts
 }
 

v2/pkg/operators/common/dsl/dsl.go

@@ -49,7 +49,7 @@ const (
 )
 
 var invalidDslFunctionError = errors.New("invalid DSL function signature")
-var invalidDslFunctionMessageTemplate = "%w. correct method signature %q"
+var invalidDslFunctionMessage = "%w. correct method signature %q"
 
 var dslFunctions map[string]dslFunction
 
@@ -961,7 +961,7 @@ func makeDslFunction(numberOfParameters int, dslFunctionLogic govaluate.Expressi
 			[]string{signature},
 			func(args ...interface{}) (interface{}, error) {
 				if len(args) != numberOfParameters {
-					return nil, fmt.Errorf(invalidDslFunctionMessageTemplate, invalidDslFunctionError, signature)
+					return nil, fmt.Errorf(invalidDslFunctionMessage, invalidDslFunctionError, signature)
 				}
 				return dslFunctionLogic(args...)
 			},

v2/pkg/operators/common/dsl/dsl_test.go

@@ -53,7 +53,7 @@ func TestDSLGzipSerialize(t *testing.T) {
 
 func TestDslFunctionSignatures(t *testing.T) {
 	createSignatureError := func(signature string) string {
-		return fmt.Errorf(invalidDslFunctionMessageTemplate, invalidDslFunctionError, signature).Error()
+		return fmt.Errorf(invalidDslFunctionMessage, invalidDslFunctionError, signature).Error()
 	}
 
 	toUpperSignatureError := createSignatureError("to_upper(arg1 interface{}) interface{}")