Merge pull request #1526 from projectdiscovery/dev

v2.5.9 Release
projectdiscovery · Jan 23, 2022 · 4d4d722 · 4d4d722
2 parents 1956dab + aa695c4
commit 4d4d722
Show file tree

Hide file tree

Showing 33 changed files with 376 additions and 110 deletions.
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
@@ -2172,6 +2172,8 @@ Enum Values:
   - <code>PATCH</code>
 
   - <code>PURGE</code>
+
+  - <code>Debug</code>
 </div>
 
 <hr />

diff --git a/integration_tests/dns/ptr.yaml b/integration_tests/dns/ptr.yaml
@@ -0,0 +1,22 @@
+id: ptr-fingerprint
+
+info:
+  name: PTR Fingerprint
+  author: pdteam
+  severity: info
+  tags: dns,ptr
+
+dns:
+  - name: "{{FQDN}}"
+    type: PTR
+
+    matchers:
+      - type: word
+        words:
+          - "IN\tPTR"
+
+    extractors:
+      - type: regex
+        group: 1
+        regex:
+          - "IN\tPTR\t(.+)"
diff --git a/integration_tests/http/dsl-functions.yaml b/integration_tests/http/dsl-functions.yaml
@@ -14,55 +14,56 @@ requests:
         02: {{base64(1234)}}
         03: {{base64_decode("SGVsbG8=")}}
         04: {{base64_py("Hello")}}
-        05: {{contains("Hello", "lo")}}
-        06: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}}
-        07: {{gzip("Hello")}}
-        08: {{hex_decode("6161")}}
-        09: {{hex_encode("aa")}}
-        10: {{html_escape("<body>test</body>")}}
-        11: {{html_unescape("&lt;body&gt;test&lt;/body&gt;")}}
-        12: {{len("Hello")}}
-        13: {{len(5555)}}
-        14: {{md5("Hello")}}
-        15: {{md5(1234)}}
-        16: {{mmh3("Hello")}}
-        17: {{print_debug(1+2, "Hello")}}
-        18: {{rand_base(5, "abc")}}
-        19: {{rand_base(5, "")}}
-        20: {{rand_base(5)}}
-        21: {{rand_char("abc")}}
-        22: {{rand_char("")}}
-        23: {{rand_char()}}
-        24: {{rand_int(1, 10)}}
-        25: {{rand_int(10)}}
-        26: {{rand_int()}}
-        27: {{rand_text_alpha(10, "abc")}}
-        28: {{rand_text_alpha(10, "")}}
-        29: {{rand_text_alpha(10)}}
-        30: {{rand_text_alphanumeric(10, "ab12")}}
-        31: {{rand_text_alphanumeric(10)}}
-        32: {{rand_text_numeric(10, 123)}}
-        33: {{rand_text_numeric(10)}}
-        34: {{regex("H([a-z]+)o", "Hello")}}
-        35: {{remove_bad_chars("abcd", "bc")}}
-        36: {{repeat("a", 5)}}
-        37: {{replace("Hello", "He", "Ha")}}
-        38: {{replace_regex("He123llo", "(\\d+)", "")}}
-        39: {{reverse("abc")}}
-        40: {{sha1("Hello")}}
-        41: {{sha256("Hello")}}
-        42: {{to_lower("HELLO")}}
-        43: {{to_upper("hello")}}
-        44: {{trim("aaaHelloddd", "ad")}}
-        45: {{trim_left("aaaHelloddd", "ad")}}
-        46: {{trim_prefix("aaHelloaa", "aa")}}
-        47: {{trim_right("aaaHelloddd", "ad")}}
-        48: {{trim_space("  Hello  ")}}
-        49: {{trim_suffix("aaHelloaa", "aa")}}
-        50: {{unix_time(10)}}
-        51: {{url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")}}
-        52: {{url_encode("https://projectdiscovery.io/test?a=1")}}
-        53: {{wait_for(1)}}
+        05: {{concat("Hello", "world")}}
+        06: {{contains("Hello", "lo")}}
+        07: {{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}}
+        08: {{gzip("Hello")}}
+        09: {{hex_decode("6161")}}
+        10: {{hex_encode("aa")}}
+        11: {{html_escape("<body>test</body>")}}
+        12: {{html_unescape("&lt;body&gt;test&lt;/body&gt;")}}
+        13: {{len("Hello")}}
+        14: {{len(5555)}}
+        15: {{md5("Hello")}}
+        16: {{md5(1234)}}
+        17: {{mmh3("Hello")}}
+        18: {{print_debug(1+2, "Hello")}}
+        19: {{rand_base(5, "abc")}}
+        20: {{rand_base(5, "")}}
+        21: {{rand_base(5)}}
+        22: {{rand_char("abc")}}
+        23: {{rand_char("")}}
+        24: {{rand_char()}}
+        25: {{rand_int(1, 10)}}
+        26: {{rand_int(10)}}
+        27: {{rand_int()}}
+        28: {{rand_text_alpha(10, "abc")}}
+        29: {{rand_text_alpha(10, "")}}
+        30: {{rand_text_alpha(10)}}
+        31: {{rand_text_alphanumeric(10, "ab12")}}
+        32: {{rand_text_alphanumeric(10)}}
+        33: {{rand_text_numeric(10, 123)}}
+        34: {{rand_text_numeric(10)}}
+        35: {{regex("H([a-z]+)o", "Hello")}}
+        36: {{remove_bad_chars("abcd", "bc")}}
+        37: {{repeat("a", 5)}}
+        38: {{replace("Hello", "He", "Ha")}}
+        39: {{replace_regex("He123llo", "(\\d+)", "")}}
+        40: {{reverse("abc")}}
+        41: {{sha1("Hello")}}
+        42: {{sha256("Hello")}}
+        43: {{to_lower("HELLO")}}
+        44: {{to_upper("hello")}}
+        45: {{trim("aaaHelloddd", "ad")}}
+        46: {{trim_left("aaaHelloddd", "ad")}}
+        47: {{trim_prefix("aaHelloaa", "aa")}}
+        48: {{trim_right("aaaHelloddd", "ad")}}
+        49: {{trim_space("  Hello  ")}}
+        50: {{trim_suffix("aaHelloaa", "aa")}}
+        51: {{unix_time(10)}}
+        52: {{url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1")}}
+        53: {{url_encode("https://projectdiscovery.io/test?a=1")}}
+        54: {{wait_for(1)}}
 
     extractors:
       - type: regex

diff --git a/integration_tests/http/race-multiple.yaml b/integration_tests/http/race-multiple.yaml
@@ -0,0 +1,46 @@
+id: race-condition-testing
+
+info:
+  name: Race condition testing with multiple requests
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:  
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+
+        id=1
+
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+
+        id=2
+
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+
+        id=3
+
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+
+        id=4
+
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+
+        id=5
+
+    threads: 5
+    race: true
+
+    matchers:
+      - type: status
+        status:
+          - 200
diff --git a/integration_tests/http/race-simple.yaml b/integration_tests/http/race-simple.yaml
@@ -0,0 +1,23 @@
+id: race-condition-testing
+
+info:
+  name: Race Condition testing
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET / HTTP/1.1
+        Host: {{Hostname}}
+
+        test      
+
+    race: true
+    race_count: 10
+
+    matchers:
+      - type: status
+        part: header
+        status:
+          - 200
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
@@ -316,7 +316,7 @@
       ],
       "type": "string",
       "title": "type of the matcher",
-      "description": "Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"
+      "description": "Type of the matcher"
     },
     "generators.AttackTypeHolder": {
       "enum": [
@@ -342,7 +342,7 @@
       ],
       "type": "string",
       "title": "type of DNS request to make",
-      "description": "Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"
+      "description": "Type is the type of DNS request to make"
     },
     "dns.Request": {
       "properties": {
@@ -601,7 +601,7 @@
       ],
       "type": "string",
       "title": "action to perform",
-      "description": "Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"
+      "description": "Type of actions to perform"
     },
     "http.HTTPMethodTypeHolder": {
       "enum": [
@@ -614,11 +614,12 @@
         "OPTIONS",
         "TRACE",
         "PATCH",
-        "PURGE"
+        "PURGE",
+        "DEBUG"
       ],
       "type": "string",
       "title": "method is the HTTP request method",
-      "description": "Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"
+      "description": "Method is the HTTP Request Method"
     },
     "http.Request": {
       "properties": {
@@ -838,7 +839,7 @@
       ],
       "type": "string",
       "title": "type is the type of input data",
-      "description": "description=Type of input specified in data field,enum=hex,enum=text"
+      "description": "description=Type of input specified in data field"
     },
     "network.Request": {
       "properties": {

diff --git a/v2/cmd/integration-test/dns.go b/v2/cmd/integration-test/dns.go
@@ -6,6 +6,7 @@ import (
 
 var dnsTestCases = map[string]testutils.TestCase{
 	"dns/basic.yaml": &dnsBasic{},
+	"dns/ptr.yaml":   &dnsPtr{},
 }
 
 type dnsBasic struct{}
@@ -23,3 +24,19 @@ func (h *dnsBasic) Execute(filePath string) error {
 	}
 	return expectResultsCount(results, 1)
 }
+
+type dnsPtr struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *dnsPtr) Execute(filePath string) error {
+	var routerErr error
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "1.1.1.1", debug)
+	if err != nil {
+		return err
+	}
+	if routerErr != nil {
+		return routerErr
+	}
+	return expectResultsCount(results, 1)
+}
diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go
@@ -42,6 +42,8 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/get-redirects-chain-headers.yaml":        &httpGetRedirectsChainHeaders{},
 	"http/dsl-matcher-variable.yaml":               &httpDSLVariable{},
 	"http/dsl-functions.yaml":                      &httpDSLFunctions{},
+	"http/race-simple.yaml":                        &httpRaceSimple{},
+	"http/race-multiple.yaml":                      &httpRaceMultiple{},
 }
 
 type httpInteractshRequest struct{}
@@ -224,7 +226,7 @@ func (h *httpDSLFunctions) Execute(filePath string) error {
 	}
 
 	totalExtracted := strings.Split(submatch[1], ",")
-	numberOfDslFunctions := 53
+	numberOfDslFunctions := 54
 	if len(totalExtracted) != numberOfDslFunctions {
 		return errors.New("incorrect number of results")
 	}
@@ -689,3 +691,39 @@ func (h *httpGetRedirectsChainHeaders) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type httpRaceSimple struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpRaceSimple) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 10)
+}
+
+type httpRaceMultiple struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpRaceMultiple) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 5)
+}
diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
@@ -13,6 +13,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/internal/runner"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
@@ -157,6 +158,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	createGroup(flagSet, "optimization", "Optimizations",
 		flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"),
 		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
+		flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "Leave default HTTP/HTTPS ports (eg. host:80,host:443"),
 		flagSet.IntVarP(&options.MaxHostError, "max-host-error", "mhe", 30, "max errors for a host before skipping from scan"),
 		flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
 		flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),
@@ -201,6 +203,10 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	_ = flagSet.Parse()
 
+	if options.LeaveDefaultPorts {
+		http.LeaveDefaultPorts = true
+	}
+
 	if cfgFile != "" {
 		if err := flagSet.MergeConfigFile(cfgFile); err != nil {
 			gologger.Fatal().Msgf("Could not read config: %s\n", err)