Disable Template Locations (#3705) (#3926)

* Download override option definition

* Update the variable names for consistency

* Add checks for custom template disable flags

* Environment variable controlled template downloads

* Switch env naming per feedback from @ehsandeep

* minor changes

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>

v2/internal/installer/template.go

@@ -57,7 +57,9 @@ func (t *templateUpdateResults) String() string {
 // TemplateManager is a manager for templates.
 // It downloads / updates / installs templates.
 type TemplateManager struct {
-	CustomTemplates *customtemplates.CustomTemplatesManager // optional if given tries to download custom templates
+	CustomTemplates        *customtemplates.CustomTemplatesManager // optional if given tries to download custom templates
+	DisablePublicTemplates bool                                    // if true,
+	// public templates are not downloaded from the GitHub nuclei-templates repository
 }
 
 // FreshInstallIfNotExists installs templates if they are not already installed
@@ -78,7 +80,7 @@ func (t *TemplateManager) FreshInstallIfNotExists() error {
 
 // UpdateIfOutdated updates templates if they are outdated
 func (t *TemplateManager) UpdateIfOutdated() error {
-	// if folder does not exist, its a fresh install and not update
+	// if the templates folder does not exist, it's a fresh installation and do not update
 	if !fileutil.FolderExists(config.DefaultConfig.TemplatesDirectory) {
 		return t.FreshInstallIfNotExists()
 	}
@@ -95,12 +97,16 @@ func (t *TemplateManager) installTemplatesAt(dir string) error {
 			return errorutil.NewWithErr(err).Msgf("failed to create directory at %s", dir)
 		}
 	}
+	if t.DisablePublicTemplates {
+		gologger.Info().Msgf("Skipping installation of public nuclei-templates")
+		return nil
+	}
 	ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName)
 	if err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir)
 	}
 	// write templates to disk
-	if err := t.writeTemplatestoDisk(ghrd, dir); err != nil {
+	if err := t.writeTemplatesToDisk(ghrd, dir); err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to write templates to disk at %s", dir)
 	}
 	gologger.Info().Msgf("Successfully installed nuclei-templates at %s", dir)
@@ -109,10 +115,14 @@ func (t *TemplateManager) installTemplatesAt(dir string) error {
 
 // updateTemplatesAt updates templates at given directory
 func (t *TemplateManager) updateTemplatesAt(dir string) error {
-	// firstly read checksums from .checksum file these are used to generate stats
+	if t.DisablePublicTemplates {
+		gologger.Info().Msgf("Skipping update of public nuclei-templates")
+		return nil
+	}
+	// firstly, read checksums from .checksum file these are used to generate stats
 	oldchecksums, err := t.getChecksumFromDir(dir)
 	if err != nil {
-		// if something went wrong overwrite all files
+		// if something went wrong, overwrite all files
 		oldchecksums = make(map[string]string)
 	}
 
@@ -124,7 +134,7 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error {
 	gologger.Info().Msgf("Your current nuclei-templates %s are outdated. Latest is %s\n", config.DefaultConfig.TemplateVersion, ghrd.Latest.GetTagName())
 
 	// write templates to disk
-	if err := t.writeTemplatestoDisk(ghrd, dir); err != nil {
+	if err := t.writeTemplatesToDisk(ghrd, dir); err != nil {
 		return err
 	}
 
@@ -173,9 +183,9 @@ func (t *TemplateManager) summarizeChanges(old, new map[string]string) *template
 	return results
 }
 
-// getAbsoluteFilePath returns absolute path where a file should be written based on given uri(i.e files in zip)
-// if returned path is empty, it means that file should not be written and skipped
-func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.FileInfo) string {
+// getAbsoluteFilePath returns an absolute path where a file should be written based on given uri(i.e., files in zip)
+// if a returned path is empty, it means that file should not be written and skipped
+func (t *TemplateManager) getAbsoluteFilePath(templateDir, uri string, f fs.FileInfo) string {
 	// overwrite .nuclei-ignore everytime nuclei-templates are downloaded
 	if f.Name() == config.NucleiIgnoreFileName {
 		return config.DefaultConfig.GetIgnoreFilePath()
@@ -194,7 +204,7 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File
 	if index == -1 {
 		// zip files does not have directory at all , in this case log error but continue
 		gologger.Warning().Msgf("failed to get directory name from uri: %s", uri)
-		return filepath.Join(templatedir, uri)
+		return filepath.Join(templateDir, uri)
 	}
 	// seperator is also included in rootDir
 	rootDirectory := uri[:index+1]
@@ -205,14 +215,14 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File
 		return ""
 	}
 
-	newPath := filepath.Clean(filepath.Join(templatedir, relPath))
+	newPath := filepath.Clean(filepath.Join(templateDir, relPath))
 
-	if !strings.HasPrefix(newPath, templatedir) {
+	if !strings.HasPrefix(newPath, templateDir) {
 		// we don't allow LFI
 		return ""
 	}
 
-	if newPath == templatedir || newPath == templatedir+string(os.PathSeparator) {
+	if newPath == templateDir || newPath == templateDir+string(os.PathSeparator) {
 		// skip writing the folder itself since it already exists
 		return ""
 	}
@@ -228,12 +238,12 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File
 }
 
 // writeChecksumFileInDir is actual method responsible for writing all templates to directory
-func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error {
-	LocaltemplatesIndex, err := config.GetNucleiTemplatesIndex()
+func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error {
+	localTemplatesIndex, err := config.GetNucleiTemplatesIndex()
 	if err != nil {
 		gologger.Warning().Msgf("failed to get local nuclei-templates index: %s", err)
-		if LocaltemplatesIndex == nil {
-			LocaltemplatesIndex = map[string]string{} // no-op
+		if localTemplatesIndex == nil {
+			localTemplatesIndex = map[string]string{} // no-op
 		}
 	}
 
@@ -253,10 +263,10 @@ func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownlo
 		// instead of creating it from scratch
 		id, _ := config.GetTemplateIDFromReader(bytes.NewReader(bin), uri)
 		if id != "" {
-			// based on template id, check if we are updating path of official nuclei template
-			if oldPath, ok := LocaltemplatesIndex[id]; ok {
+			// based on template id, check if we are updating a path of official nuclei template
+			if oldPath, ok := localTemplatesIndex[id]; ok {
 				if oldPath != writePath {
-					// write new template at new path and delete old template
+					// write new template at a new path and delete old template
 					if err := os.WriteFile(writePath, bin, f.Mode()); err != nil {
 						return errorutil.NewWithErr(err).Msgf("failed to write file %s", uri)
 					}
@@ -303,12 +313,12 @@ func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownlo
 		return errorutil.NewWithErr(err).Msgf("failed to write nuclei templates index")
 	}
 
-	// after installation create and write checksums to .checksum file
+	// after installation, create and write checksums to .checksum file
 	return t.writeChecksumFileInDir(dir)
 }
 
 // getChecksumFromDir returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension)
-// if .checksum file does not exist checksums are calculated and returned
+// if .checksum file does not exist, checksums are calculated and returned
 func (t *TemplateManager) getChecksumFromDir(dir string) (map[string]string, error) {
 	checksumFilePath := config.DefaultConfig.GetChecksumFilePath()
 	if fileutil.FileExists(checksumFilePath) {

v2/internal/runner/options.go

@@ -29,7 +29,7 @@ import (
 
 func ConfigureOptions() error {
 	// with FileStringSliceOptions, FileNormalizedStringSliceOptions, FileCommaSeparatedStringSliceOptions
-	// if file has extension `.yaml,.json` we consider those as strings and not files to be read
+	// if file has the extension `.yaml` or `.json` we consider those as strings and not files to be read
 	isFromFileFunc := func(s string) bool {
 		return !config.IsTemplate(s)
 	}
@@ -136,23 +136,23 @@ func validateOptions(options *types.Options) error {
 		validateCertificatePaths(options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile)
 	}
 	// Verify AWS secrets are passed if a S3 template bucket is passed
-	if options.AwsBucketName != "" && options.UpdateTemplates {
+	if options.AwsBucketName != "" && options.UpdateTemplates && !options.AwsTemplateDisableDownload {
 		missing := validateMissingS3Options(options)
 		if missing != nil {
 			return fmt.Errorf("aws s3 bucket details are missing. Please provide %s", strings.Join(missing, ","))
 		}
 	}
 
 	// Verify Azure connection configuration is passed if the Azure template bucket is passed
-	if options.AzureContainerName != "" && options.UpdateTemplates {
+	if options.AzureContainerName != "" && options.UpdateTemplates && !options.AzureTemplateDisableDownload {
 		missing := validateMissingAzureOptions(options)
 		if missing != nil {
 			return fmt.Errorf("azure connection details are missing. Please provide %s", strings.Join(missing, ","))
 		}
 	}
 
 	// Verify that all GitLab options are provided if the GitLab server or token is provided
-	if len(options.GitLabTemplateRepositoryIDs) != 0 && options.UpdateTemplates {
+	if len(options.GitLabTemplateRepositoryIDs) != 0 && options.UpdateTemplates && !options.GitLabTemplateDisableDownload {
 		missing := validateMissingGitLabOptions(options)
 		if missing != nil {
 			return fmt.Errorf("gitlab server details are missing. Please provide %s", strings.Join(missing, ","))
@@ -292,7 +292,7 @@ func configureOutput(options *types.Options) {
 	logutil.DisableDefaultLogger()
 }
 
-// loadResolvers loads resolvers from both user provided flag and file
+// loadResolvers loads resolvers from both user-provided flags and file
 func loadResolvers(options *types.Options) {
 	if options.ResolversFile == "" {
 		return
@@ -396,4 +396,20 @@ func readEnvInputVars(options *types.Options) {
 	options.AzureClientID = os.Getenv("AZURE_CLIENT_ID")
 	options.AzureClientSecret = os.Getenv("AZURE_CLIENT_SECRET")
 	options.AzureServiceURL = os.Getenv("AZURE_SERVICE_URL")
+
+	// General options to disable the template download locations from being used.
+	// This will override the default behavior of downloading templates from the default locations as well as the
+	// custom locations.
+	// The primary use-case is when the user wants to use custom templates only and does not want to download any
+	// templates from the default locations or is unable to connect to the public internet.
+	options.PublicTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_PUBLIC_DOWNLOAD")
+	options.GitHubTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_GITHUB_DOWNLOAD")
+	options.GitLabTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_GITLAB_DOWNLOAD")
+	options.AwsTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_AWS_DOWNLOAD")
+	options.AzureTemplateDisableDownload = getBoolEnvValue("DISABLE_NUCLEI_TEMPLATES_AZURE_DOWNLOAD")
+}
+
+func getBoolEnvValue(key string) bool {
+	value := os.Getenv(key)
+	return strings.EqualFold(value, "true")
 }

v2/internal/runner/runner.go

@@ -112,7 +112,10 @@ func New(options *types.Options) (*Runner, error) {
 
 		// Check for template updates and update if available.
 		// If the custom templates manager is not nil, we will install custom templates if there is a fresh installation
-		tm := &installer.TemplateManager{CustomTemplates: ctm}
+		tm := &installer.TemplateManager{
+			CustomTemplates:        ctm,
+			DisablePublicTemplates: options.PublicTemplateDisableDownload,
+		}
 		if err := tm.FreshInstallIfNotExists(); err != nil {
 			gologger.Warning().Msgf("failed to install nuclei templates: %s\n", err)
 		}

v2/pkg/catalog/config/constants.go

@@ -14,7 +14,7 @@ const (
 	NucleiTemplatesIndexFileName    = ".templates-index" // contains index of official nuclei templates
 	NucleiTemplatesCheckSumFileName = ".checksum"
 	NewTemplateAdditionsFileName    = ".new-additions"
-	CLIConifgFileName               = "config.yaml"
+	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
 	Version = `v2.9.8`
@@ -25,11 +25,12 @@ const (
 	CustomGitLabTemplatesDirName = "gitlab"
 )
 
-// IsOutdatedVersion compares two versions and returns true if the current version is outdated
+// IsOutdatedVersion compares two versions and returns true
+// if the current version is outdated
 func IsOutdatedVersion(current, latest string) bool {
 	if latest == "" {
-		// if pdtm api call failed it's assumed that current version is outdated
-		// and it will be confirmed while updating from github
+		// if pdtm api call failed it's assumed that the current version is outdated
+		// and it will be confirmed while updating from GitHub
 		// this fixes `version string empty` errors
 		return true
 	}

v2/pkg/catalog/config/nucleiconfig.go

@@ -135,7 +135,7 @@ func (c *Config) GetChecksumFilePath() string {
 
 // GetCLIOptsConfigFilePath returns the nuclei cli config file path
 func (c *Config) GetFlagsConfigFilePath() string {
-	return filepath.Join(c.configDir, CLIConifgFileName)
+	return filepath.Join(c.configDir, CLIConfigFileName)
 }
 
 // GetNewAdditions returns new template additions in current template release

v2/pkg/external/customtemplates/azure_blob.go

@@ -25,7 +25,7 @@ type customTemplateAzureBlob struct {
 // NewAzureProviders creates a new Azure Blob Storage provider for downloading custom templates
 func NewAzureProviders(options *types.Options) ([]*customTemplateAzureBlob, error) {
 	providers := []*customTemplateAzureBlob{}
-	if options.AzureContainerName != "" {
+	if options.AzureContainerName != "" && !options.AzureTemplateDisableDownload {
 		// Establish a connection to Azure and build a client object with which to download templates from Azure Blob Storage
 		azClient, err := getAzureBlobClient(options.AzureTenantID, options.AzureClientID, options.AzureClientSecret, options.AzureServiceURL)
 		if err != nil {

v2/pkg/external/customtemplates/github.go

@@ -58,11 +58,15 @@ func (customTemplate *customTemplateGithubRepo) Update(ctx context.Context) {
 	}
 }
 
-// NewGithubProviders returns new instance of github providers for downloading custom templates
+// NewGithubProviders returns new instance of GitHub providers for downloading custom templates
 func NewGithubProviders(options *types.Options) ([]*customTemplateGithubRepo, error) {
 	providers := []*customTemplateGithubRepo{}
 	gitHubClient := getGHClientIncognito()
 
+	if options.GitHubTemplateDisableDownload {
+		return providers, nil
+	}
+
 	for _, repoName := range options.GithubTemplateRepo {
 		owner, repo, err := getOwnerAndRepo(repoName)
 		if err != nil {
@@ -86,8 +90,8 @@ func NewGithubProviders(options *types.Options) ([]*customTemplateGithubRepo, er
 }
 
 // getOwnerAndRepo returns the owner, repo, err from the given string
-// eg. it takes input projectdiscovery/nuclei-templates and
-// returns owner=> projectdiscovery , repo => nuclei-templates
+// e.g., it takes input projectdiscovery/nuclei-templates and
+// returns owner => projectdiscovery, repo => nuclei-templates
 func getOwnerAndRepo(reponame string) (owner string, repo string, err error) {
 	s := strings.Split(reponame, "/")
 	if len(s) != 2 {
@@ -118,7 +122,7 @@ getRepo:
 	return repo, nil
 }
 
-// download the git repo to given path
+// download the git repo to a given path
 func (ctr *customTemplateGithubRepo) cloneRepo(clonePath, githubToken string) error {
 	r, err := git.PlainClone(clonePath, false, &git.CloneOptions{
 		URL:  ctr.gitCloneURL,
@@ -127,7 +131,7 @@ func (ctr *customTemplateGithubRepo) cloneRepo(clonePath, githubToken string) er
 	if err != nil {
 		return errors.Errorf("%s/%s: %s", ctr.owner, ctr.reponame, err.Error())
 	}
-	// Add the user as well in the config. By default user is not set
+	// Add the user as well in the config. By default, user is not set
 	config, _ := r.Storer.Config()
 	config.User.Name = ctr.owner
 	return r.SetConfig(config)

v2/pkg/external/customtemplates/gitlab.go

@@ -24,7 +24,7 @@ type customTemplateGitLabRepo struct {
 // NewGitlabProviders returns a new list of GitLab providers for downloading custom templates
 func NewGitlabProviders(options *types.Options) ([]*customTemplateGitLabRepo, error) {
 	providers := []*customTemplateGitLabRepo{}
-	if options.GitLabToken != "" {
+	if options.GitLabToken != "" && !options.GitLabTemplateDisableDownload {
 		// Establish a connection to GitLab and build a client object with which to download templates from GitLab
 		gitLabClient, err := getGitLabClient(options.GitLabServerURL, options.GitLabToken)
 		if err != nil {

v2/pkg/external/customtemplates/s3.go

@@ -61,7 +61,7 @@ func (bk *customTemplateS3Bucket) Update(ctx context.Context) {
 // NewS3Providers returns a new instances of a s3 providers for downloading custom templates
 func NewS3Providers(options *types.Options) ([]*customTemplateS3Bucket, error) {
 	providers := []*customTemplateS3Bucket{}
-	if options.AwsBucketName != "" {
+	if options.AwsBucketName != "" && !options.AwsTemplateDisableDownload {
 		s3c, err := getS3Client(context.TODO(), options.AwsAccessKey, options.AwsSecretKey, options.AwsRegion)
 		if err != nil {
 			return nil, errorutil.NewWithErr(err).Msgf("error downloading s3 bucket %s", options.AwsBucketName)

v2/pkg/types/types.go

@@ -337,16 +337,22 @@ type Options struct {
 	ScanAllIPs bool
 	// IPVersion to scan (4,6)
 	IPVersion goflags.StringSlice
+	// PublicTemplateDisableDownload disables downloading templates from the nuclei-templates public repository
+	PublicTemplateDisableDownload bool
 	// GitHub token used to clone/pull from private repos for custom templates
 	GithubToken string
 	// GithubTemplateRepo is the list of custom public/private templates GitHub repos
 	GithubTemplateRepo []string
+	// GitHubTemplateDisableDownload disables downloading templates from custom GitHub repositories
+	GitHubTemplateDisableDownload bool
 	// GitLabServerURL is the gitlab server to use for custom templates
 	GitLabServerURL string
 	// GitLabToken used to clone/pull from private repos for custom templates
 	GitLabToken string
 	// GitLabTemplateRepositoryIDs is the comma-separated list of custom gitlab repositories IDs
 	GitLabTemplateRepositoryIDs []int
+	// GitLabTemplateDisableDownload disables downloading templates from custom GitLab repositories
+	GitLabTemplateDisableDownload bool
 	// AWS access key for downloading templates from S3 bucket
 	AwsAccessKey string
 	// AWS secret key for downloading templates from S3 bucket
@@ -355,6 +361,8 @@ type Options struct {
 	AwsBucketName string
 	// AWS Region name where AWS S3 bucket is located
 	AwsRegion string
+	// AwsTemplateDisableDownload disables downloading templates from AWS S3 buckets
+	AwsTemplateDisableDownload bool
 	// AzureContainerName for downloading templates from Azure Blob Storage. Example: templates
 	AzureContainerName string
 	// AzureTenantID for downloading templates from Azure Blob Storage. Example: 00000000-0000-0000-0000-000000000000
@@ -365,6 +373,8 @@ type Options struct {
 	AzureClientSecret string
 	// AzureServiceURL for downloading templates from Azure Blob Storage. Example: https://XXXXXXXXXX.blob.core.windows.net/
 	AzureServiceURL string
+	// AzureTemplateDisableDownload disables downloading templates from Azure Blob Storage
+	AzureTemplateDisableDownload bool
 	// Scan Strategy (auto,hosts-spray,templates-spray)
 	ScanStrategy string
 	// Fuzzing Type overrides template level fuzzing-type configuration