Merge pull request #2006 from projectdiscovery/dev

v2.7.1 Release

.github/workflows/dockerhub-push.yml

@@ -20,19 +20,19 @@ jobs:
           echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/nuclei/releases/latest" | jq -r .tag_name)"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v2 
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           platforms: linux/amd64,linux/arm64

.github/workflows/lint-test.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.1.0
+        uses: golangci/golangci-lint-action@v3.2.0
         with:
           version: latest
           args: --timeout 5m

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18.1-alpine as build-env
+FROM golang:1.18.2-alpine as build-env
 RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
 
 FROM alpine:3.15.4

README.md

@@ -84,7 +84,7 @@ nuclei -h
 This will display help for the tool. Here are all the switches it supports.
 
 
-```yaml
+```console
 Nuclei is a fast, template based vulnerability scanner focusing
 on extensive configurability, massive extensibility and ease of use.
 
@@ -123,8 +123,8 @@ FILTERING:
 
 OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
-   -silent                       display findings only
-   -nc, -no-color                disable output content coloring (ANSI escape codes)
+   -sresp, -store-resp           store all request/response passed through nuclei to output directory
+   -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
    -json                         write output in JSONL(ines) format
    -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
    -nm, -no-meta                 disable printing result metadata in cli output
@@ -133,6 +133,8 @@ OUTPUT:
    -ms, -matcher-status          display match failure status
    -me, -markdown-export string  directory to export results in markdown format
    -se, -sarif-export string     file to export results in SARIF format
+   -silent                       display findings only
+   -nc, -no-color                disable output content coloring (ANSI escape codes)
 
 CONFIGURATIONS:
    -config string              path to the nuclei configuration file
@@ -149,7 +151,8 @@ CONFIGURATIONS:
    -cc, -client-cert string    client certificate file (PEM-encoded) used for authenticating against scanned hosts
    -ck, -client-key string     client key file (PEM-encoded) used for authenticating against scanned hosts
    -ca, -client-ca string      client certificate authority file (PEM-encoded) used for authenticating against scanned hosts
-   -ztls                       Use ztls library with autofallback to standard one for tls13
+   -ztls                       use ztls library with autofallback to standard one for tls13
+   -sni string                 tls sni hostname to use (default: input domain name)
 
 INTERACTSH:
    -iserver, -interactsh-server string  interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me)
@@ -188,8 +191,6 @@ DEBUG:
    -debug                        display all requests and responses
    -dreq, -debug-req             display all sent requests
    -dresp, -debug-resp           display all received responses
-   -sresp, -store-resp           store all request/response passed through nuclei to output directory
-   -srd, -store-resp-dir string  store all request/response passed through nuclei to custom directory (default "output")
    -p, -proxy string[]           list of http/socks5 proxy to use (comma separated or file input)
    -pi, -proxy-internal          proxy all internal requests
    -tlog, -trace-log string      file to write sent requests trace log

integration_tests/http/get-override-sni.yaml

@@ -0,0 +1,18 @@
+id: basic-raw-http-example
+
+info:
+  name: Test RAW GET Template
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        @tls-sni:request.host
+        GET / HTTP/1.1
+        Host: test
+
+    matchers:
+      - type: word
+        words:
+          - "test-ok"

integration_tests/http/get-sni.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request with CLI SNI
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "test-ok"

v2/cmd/cve-annotate/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -10,10 +11,17 @@ import (
 	"strings"
 
 	"github.com/projectdiscovery/nvd"
+	"github.com/projectdiscovery/sliceutil"
+	"github.com/projectdiscovery/stringsutil"
+	"gopkg.in/yaml.v3"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 )
 
+const (
+	yamlIndentSpaces = 2
+)
+
 var (
 	input       = flag.String("i", "", "Templates to annotate")
 	templateDir = flag.String("d", "", "Custom template directory for update")
@@ -63,6 +71,8 @@ var (
 	severityRegex = regexp.MustCompile(`severity: ([a-z]+)`)
 )
 
+const maxReferenceCount = 5
+
 func getCVEData(client *nvd.Client, filePath, data string) {
 	matches := idRegex.FindAllStringSubmatch(data, 1)
 	if len(matches) == 0 {
@@ -76,10 +86,6 @@ func getCVEData(client *nvd.Client, filePath, data string) {
 	}
 	severityValue := severityMatches[0][1]
 
-	// Skip if there's classification data already
-	if strings.Contains(data, "classification:") {
-		return
-	}
 	cveItem, err := client.FetchCVE(cveName)
 	if err != nil {
 		log.Printf("Could not fetch cve %s: %s\n", cveName, err)
@@ -98,43 +104,84 @@ func getCVEData(client *nvd.Client, filePath, data string) {
 	infoBlockIndexData := data[strings.Index(data, "info:"):]
 	requestsIndex := strings.Index(infoBlockIndexData, "requests:")
 	networkIndex := strings.Index(infoBlockIndexData, "network:")
-	if requestsIndex == -1 && networkIndex == -1 {
+	variablesIndex := strings.Index(infoBlockIndexData, "variables:")
+	if requestsIndex == -1 && networkIndex == -1 && variablesIndex == -1 {
 		return
 	}
 	if networkIndex != -1 {
 		requestsIndex = networkIndex
 	}
+	if variablesIndex != -1 {
+		requestsIndex = variablesIndex
+	}
 	infoBlockData := infoBlockIndexData[:requestsIndex]
 	infoBlockClean := strings.TrimRight(infoBlockData, "\n")
 
-	newInfoBlock := infoBlockClean
-	var changed bool
+	infoBlock := InfoBlock{}
+	err = yaml.Unmarshal([]byte(data), &infoBlock)
+	if err != nil {
+		log.Printf("Could not unmarshal info block: %s\n", err)
+	}
 
+	var changed bool
 	if newSeverity := isSeverityMatchingCvssScore(severityValue, cvssScore); newSeverity != "" {
 		changed = true
-		newInfoBlock = strings.ReplaceAll(newInfoBlock, severityMatches[0][0], "severity: "+newSeverity)
+		infoBlock.Info.Severity = newSeverity
 		fmt.Printf("Adjusting severity for %s from %s=>%s (%.2f)\n", filePath, severityValue, newSeverity, cvssScore)
 	}
-	if !strings.Contains(infoBlockClean, "classification") && (cvssScore != 0 && cvssMetrics != "") {
+	isCvssEmpty := cvssScore == 0 || cvssMetrics == ""
+	hasCvssChanged := infoBlock.Info.Classification.CvssScore != cvssScore || cvssMetrics != infoBlock.Info.Classification.CvssMetrics
+	if !isCvssEmpty && hasCvssChanged {
 		changed = true
-		newInfoBlock += fmt.Sprintf("\n  classification:\n    cvss-metrics: %s\n    cvss-score: %.2f\n    cve-id: %s", cvssMetrics, cvssScore, cveName)
+		infoBlock.Info.Classification.CvssMetrics = cvssMetrics
+		infoBlock.Info.Classification.CvssScore = cvssScore
+		infoBlock.Info.Classification.CveId = cveName
 		if len(cweID) > 0 && (cweID[0] != "NVD-CWE-Other" && cweID[0] != "NVD-CWE-noinfo") {
-			newInfoBlock += fmt.Sprintf("\n    cwe-id: %s", strings.Join(cweID, ","))
+			infoBlock.Info.Classification.CweId = strings.Join(cweID, ",")
 		}
 	}
 	// If there is no description field, fill the description from CVE information
-	if !strings.Contains(infoBlockClean, "description:") && len(cveItem.CVE.Description.DescriptionData) > 0 {
+	hasDescriptionData := len(cveItem.CVE.Description.DescriptionData) > 0
+	isDescriptionEmpty := infoBlock.Info.Description == ""
+	if isDescriptionEmpty && hasDescriptionData {
 		changed = true
-		newInfoBlock += fmt.Sprintf("\n  description: %s", fmt.Sprintf("%q", cveItem.CVE.Description.DescriptionData[0].Value))
+		// removes all new lines
+		description := stringsutil.ReplaceAny(cveItem.CVE.Description.DescriptionData[0].Value, "", "\n", "\\", "'", "\t")
+		description += "\n"
+		infoBlock.Info.Description = description
+	}
+
+	// we are unmarshaling info block to have valid data
+	var referenceDataURLs []string
+	for _, reference := range cveItem.CVE.References.ReferenceData {
+		referenceDataURLs = append(referenceDataURLs, reference.URL)
 	}
-	if !strings.Contains(infoBlockClean, "reference:") && len(cveItem.CVE.References.ReferenceData) > 0 {
+	hasReferenceData := len(cveItem.CVE.References.ReferenceData) > 0
+	areCveReferencesContained := sliceutil.ContainsItems(infoBlock.Info.Reference, referenceDataURLs)
+	referencesCount := len(infoBlock.Info.Reference)
+	if hasReferenceData && !areCveReferencesContained {
 		changed = true
-		newInfoBlock += "\n  reference:"
 		for _, reference := range cveItem.CVE.References.ReferenceData {
-			newInfoBlock += fmt.Sprintf("\n    - %s", reference.URL)
+			referencesCount++
+			if referencesCount >= maxReferenceCount {
+				break
+			}
+			infoBlock.Info.Reference = append(infoBlock.Info.Reference, reference.URL)
 		}
+		infoBlock.Info.Reference = sliceutil.PruneEmptyStrings(sliceutil.Dedupe(infoBlock.Info.Reference))
 	}
-	newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlock)
+
+	var newInfoBlock bytes.Buffer
+	yamlEncoder := yaml.NewEncoder(&newInfoBlock)
+	yamlEncoder.SetIndent(yamlIndentSpaces)
+	err = yamlEncoder.Encode(infoBlock)
+	if err != nil {
+		log.Printf("Could not marshal info block: %s\n", err)
+		return
+	}
+	newInfoBlockData := strings.TrimSuffix(newInfoBlock.String(), "\n")
+
+	newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlockData)
 	if changed {
 		_ = ioutil.WriteFile(filePath, []byte(newTemplate), 0644)
 		fmt.Printf("Wrote updated template to %s\n", filePath)
@@ -161,3 +208,27 @@ func isSeverityMatchingCvssScore(severity string, score float64) string {
 	}
 	return ""
 }
+
+// Cloning struct from nuclei as we don't want any validation
+type InfoBlock struct {
+	Info TemplateInfo `yaml:"info"`
+}
+
+type TemplateClassification struct {
+	CvssMetrics string  `yaml:"cvss-metrics,omitempty"`
+	CvssScore   float64 `yaml:"cvss-score,omitempty"`
+	CveId       string  `yaml:"cve-id,omitempty"`
+	CweId       string  `yaml:"cwe-id,omitempty"`
+}
+
+type TemplateInfo struct {
+	Name           string                 `yaml:"name"`
+	Author         string                 `yaml:"author"`
+	Severity       string                 `yaml:"severity"`
+	Description    string                 `yaml:"description,omitempty"`
+	Reference      []string               `yaml:"reference,omitempty"`
+	Remediation    string                 `yaml:"remediation,omitempty"`
+	Classification TemplateClassification `yaml:"classification,omitempty"`
+	Metadata       map[string]string      `yaml:"metadata,omitempty"`
+	Tags           string                 `yaml:"tags,omitempty"`
+}

v2/cmd/integration-test/http.go

@@ -48,6 +48,8 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/stop-at-first-match.yaml":                 &httpStopAtFirstMatch{},
 	"http/stop-at-first-match-with-extractors.yaml": &httpStopAtFirstMatchWithExtractors{},
 	"http/variables.yaml":                           &httpVariables{},
+	"http/get-override-sni.yaml":                    &httpSniAnnotation{},
+	"http/get-sni.yaml":                             &customCLISNI{},
 }
 
 type httpInteractshRequest struct{}
@@ -810,3 +812,47 @@ func (h *httpVariables) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type customCLISNI struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *customCLISNI) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		if r.TLS.ServerName == "test" {
+			_, _ = w.Write([]byte("test-ok"))
+		} else {
+			_, _ = w.Write([]byte("test-ko"))
+		}
+	})
+	ts := httptest.NewTLSServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-sni", "test")
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}
+
+type httpSniAnnotation struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpSniAnnotation) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		if r.TLS.ServerName == "test" {
+			_, _ = w.Write([]byte("test-ok"))
+		} else {
+			_, _ = w.Write([]byte("test-ko"))
+		}
+	})
+	ts := httptest.NewTLSServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}

v2/cmd/nuclei/main.go

@@ -119,6 +119,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	createGroup(flagSet, "output", "Output",
 		flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"),
+		flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"),
+		flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"),
 		flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
 		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
 		flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"),
@@ -146,7 +148,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.ClientCertFile, "client-cert", "cc", "", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"),
 		flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"),
 		flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"),
-		flagSet.BoolVar(&options.ZTLS, "ztls", false, "Use ztls library with autofallback to standard one for tls13"),
+		flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"),
+		flagSet.StringVar(&options.SNI, "sni", "", "tls sni hostname to use (default: input domain name)"),
 	)
 
 	createGroup(flagSet, "interactsh", "interactsh",
@@ -190,8 +193,6 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"),
 		flagSet.BoolVarP(&options.DebugRequests, "debug-req", "dreq", false, "show all sent requests"),
 		flagSet.BoolVarP(&options.DebugResponse, "debug-resp", "dresp", false, "show all received responses"),
-		flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"),
-		flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"),
 		flagSet.NormalizedOriginalStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "list of http/socks5 proxy to use (comma separated or file input)"),
 		flagSet.BoolVarP(&options.ProxyInternal, "proxy-internal", "pi", false, "proxy all internal requests"),
 		flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),