Merge remote-tracking branch 'origin'

cmd/integration-test/headless.go

@@ -12,6 +12,7 @@ import (
 
 var headlessTestcases = []TestCaseInfo{
 	{Path: "protocols/headless/headless-basic.yaml", TestCase: &headlessBasic{}},
+	{Path: "protocols/headless/headless-waitevent.yaml", TestCase: &headlessBasic{}},
 	{Path: "protocols/headless/headless-self-contained.yaml", TestCase: &headlessSelfContained{}},
 	{Path: "protocols/headless/headless-header-action.yaml", TestCase: &headlessHeaderActions{}},
 	{Path: "protocols/headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}},

go.mod

@@ -20,13 +20,13 @@ require (
 	github.com/miekg/dns v1.1.57
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/clistats v0.0.19
+	github.com/projectdiscovery/clistats v0.0.20
 	github.com/projectdiscovery/fastdialer v0.0.46
 	github.com/projectdiscovery/hmap v0.0.27
 	github.com/projectdiscovery/interactsh v1.1.7
-	github.com/projectdiscovery/rawhttp v0.1.27
-	github.com/projectdiscovery/retryabledns v1.0.44
-	github.com/projectdiscovery/retryablehttp-go v1.0.36
+	github.com/projectdiscovery/rawhttp v0.1.28
+	github.com/projectdiscovery/retryabledns v1.0.45
+	github.com/projectdiscovery/retryablehttp-go v1.0.38
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -76,21 +76,21 @@ require (
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/praetorian-inc/fingerprintx v1.1.9
-	github.com/projectdiscovery/dsl v0.0.32
+	github.com/projectdiscovery/dsl v0.0.33
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/goflags v0.1.28
 	github.com/projectdiscovery/gologger v1.1.11
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.1
 	github.com/projectdiscovery/httpx v1.3.7
-	github.com/projectdiscovery/mapcidr v1.1.15
+	github.com/projectdiscovery/mapcidr v1.1.16
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
 	github.com/projectdiscovery/ratelimit v0.0.17
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6-0.20231116215000-e842dc367a74
 	github.com/projectdiscovery/uncover v1.0.7
-	github.com/projectdiscovery/utils v0.0.64
+	github.com/projectdiscovery/utils v0.0.65
 	github.com/projectdiscovery/wappalyzergo v0.0.109
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02

go.sum

@@ -791,10 +791,10 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k
 github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
 github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg=
 github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
-github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw=
-github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c=
-github.com/projectdiscovery/dsl v0.0.32 h1:hhZcqu+FgrF6RrikPdwc1cmYhkCyYXYKNNZL4vcGWEc=
-github.com/projectdiscovery/dsl v0.0.32/go.mod h1:0dxtOiPv0UXD8YpuPieVFN/zisQaL5Sp8OGTRzKVH0A=
+github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
+github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
+github.com/projectdiscovery/dsl v0.0.33 h1:2iSTfi5s1nKTdG76nq3hb5f5oSEg2K0t4V1Ndy0bWZ8=
+github.com/projectdiscovery/dsl v0.0.33/go.mod h1:62LGScwUP02fKQ+j/mBFjoB/zej1In+AS6ZO8P9+Rx0=
 github.com/projectdiscovery/fastdialer v0.0.46 h1:XoP41jyOBupz4w+hz9WRnvBrml3E0OiNsmfhDPsXiFM=
 github.com/projectdiscovery/fastdialer v0.0.46/go.mod h1:qe6hd6PDUoJhmkAkFGmWIfGWSuezvpClxN80pUAtVfE=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
@@ -815,22 +815,22 @@ github.com/projectdiscovery/httpx v1.3.7 h1:g/ZQIBdWWPQLF+niv39b7jRhAkyrcroJJfqb
 github.com/projectdiscovery/httpx v1.3.7/go.mod h1:FqEmL2zWZArgD1vSQ+tqHvmUItPqxYhOgKyfN8GyWMQ=
 github.com/projectdiscovery/interactsh v1.1.7 h1:rK+eKklyM+4qaLS+1MgtHDvrHdAnSTMx/Ezd62pWNL8=
 github.com/projectdiscovery/interactsh v1.1.7/go.mod h1:WYxbcV0fz3LMf83mugCYo5VUsBb4nfIdAVK6GVJhobs=
-github.com/projectdiscovery/mapcidr v1.1.15 h1:rYAgxLvMyxPU0JunE/Y3uSK1n/TcNJHK839d6YM0ms4=
-github.com/projectdiscovery/mapcidr v1.1.15/go.mod h1:s9erRsoZqWcLGhJW+WT1SnbscqzhHRRnSX916xBw5ZM=
+github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo=
+github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
 github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o=
 github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk=
 github.com/projectdiscovery/ratelimit v0.0.17 h1:+EaC/4GWg7hwK1XmTaudXucjHImrT9059UG2CEpY4hs=
 github.com/projectdiscovery/ratelimit v0.0.17/go.mod h1:kj2GIhHJ2YZzlGxpaKXRihRkiDSTLPe/I+vgqfkMsO4=
-github.com/projectdiscovery/rawhttp v0.1.27 h1:39tFKjwHf8c3VxweSpaBXsYpnanYCJVcsbylinvfAcs=
-github.com/projectdiscovery/rawhttp v0.1.27/go.mod h1:EHM1Ld29YOFK+2lqhQAmQi70xdzHc0EIfx77oxNK2UE=
+github.com/projectdiscovery/rawhttp v0.1.28 h1:6cR6JpjzEMjtyXHOWKwfFUNdmo0CXtUbOn6w6RsBYf4=
+github.com/projectdiscovery/rawhttp v0.1.28/go.mod h1:VfGWfefvtSzixCdsst+gMRYVMMnOvrLieW1l9xDdO0U=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.44 h1:JnknActEHbk3C95YHZReRUDYy4tZ113QCTIKIejqbD0=
-github.com/projectdiscovery/retryabledns v1.0.44/go.mod h1:0yE6BI+BAvZjKE2vNTphU/2gJStQZW8T5fKqANi5ehM=
-github.com/projectdiscovery/retryablehttp-go v1.0.36 h1:t83R3W9WkvkeLx4BQaR9LSzsOTHycKeBzZEVrvchIWQ=
-github.com/projectdiscovery/retryablehttp-go v1.0.36/go.mod h1:5Hen3FD4xznAJAf9Uk8tOC6wkYaMi5zv9+sjyEkCba0=
+github.com/projectdiscovery/retryabledns v1.0.45 h1:D30X3SdsJ7TOFlWMh80xYrzdjaVZcL5rksrFQ27X/Cw=
+github.com/projectdiscovery/retryabledns v1.0.45/go.mod h1:ammxRdvW5SHvbc1XIoHY/rtrA2BfYJp9TuqTJvY7dh0=
+github.com/projectdiscovery/retryablehttp-go v1.0.38 h1:MvXTiqL58+HKNL0fxvGEXUWuTrYENtrNklxxOIGFh0o=
+github.com/projectdiscovery/retryablehttp-go v1.0.38/go.mod h1:vTDxZuLKrAots/HUBM6g+E5tc64XaFCTOpxPXp+F12E=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -839,8 +839,8 @@ github.com/projectdiscovery/tlsx v1.1.6-0.20231116215000-e842dc367a74 h1:G0gw+3z
 github.com/projectdiscovery/tlsx v1.1.6-0.20231116215000-e842dc367a74/go.mod h1:YH8el7/6pyZbNed1IibjzbGpeigiCVyvE28g5+LsPAw=
 github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
-github.com/projectdiscovery/utils v0.0.64 h1:umTmYC9srbLgtJcMbHK+I77oT2MjrW6SjCt7P/F9puA=
-github.com/projectdiscovery/utils v0.0.64/go.mod h1:buA3AXoT1TVb21YWp6QM/Ks2bq/UgQU20s2E9TE9L5Y=
+github.com/projectdiscovery/utils v0.0.65 h1:QanNhSFhOfkUjniSlsV2Exm3m5MDyTGmpKbJDMcAVhg=
+github.com/projectdiscovery/utils v0.0.65/go.mod h1:qs8T5JWLIXZmoc1DbUkihFVTjPtuYgJ3WOmaNQedhwg=
 github.com/projectdiscovery/wappalyzergo v0.0.109 h1:BERfwTRn1dvB1tbhyc5m67R8VkC9zbVuPsEq4VEm07k=
 github.com/projectdiscovery/wappalyzergo v0.0.109/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

integration_tests/protocols/headless/headless-waitevent.yaml

@@ -0,0 +1,24 @@
+id: headless-waitevent
+
+info:
+    name: WaitEvent
+    severity: info
+    author: pdteam
+
+headless:
+  - steps:
+      # note waitevent must be used before navigating to any page
+      # unlike waitload
+      - action: waitevent
+        args:
+            event: 'Page.loadEventFired'
+            max-duration: 15s
+
+      - action: navigate  
+        args:
+          url: "{{BaseURL}}/"
+
+    matchers:
+      - type: word
+        words:
+          - "<html>"

internal/pdcp/writer.go

@@ -1,7 +1,6 @@
 package pdcp
 
 import (
-	"bufio"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -10,7 +9,6 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
-	"strings"
 	"sync/atomic"
 	"time"
 
@@ -84,14 +82,15 @@ type uploadResponse struct {
 func (u *UploadWriter) Upload() {
 	defer u.done.Store(true)
 
-	// start from beginning
-	_, _ = u.tempFile.Seek(0, 0)
-	// skip if file is empty
-	scanner := bufio.NewScanner(u.tempFile)
-	if !scanner.Scan() || (scanner.Scan() && strings.TrimSpace(scanner.Text()) == "") {
+	_ = u.tempFile.Sync()
+	info, err := u.tempFile.Stat()
+	if err != nil {
+		gologger.Error().Msgf("Failed to upload scan results on cloud: %v", err)
+		return
+	}
+	if info.Size() == 0 {
 		gologger.Verbose().Msgf("Scan results upload to cloud skipped, no results found to upload")
 		return
-
 	}
 	_, _ = u.tempFile.Seek(0, 0)
 

pkg/catalog/config/constants.go

@@ -17,7 +17,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.1.0`
+	Version = `v3.1.1`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

pkg/js/generated/js/libssh/ssh.js

@@ -1,4 +1,4 @@
-/** @module ssh */
+/**@module ssh */
 
 /**
  * @typedef {object} HandshakeLog
@@ -13,28 +13,28 @@ const HandshakeLog = {};
 class SSHClient {
     /**
     @method
-    @description Close closes the SSH connection and destroys the client. Returns the success state and error. If error is not nil, state will be false.
+    @description Close closes the SSH connection and destroys the client
     @returns {boolean} - The success state of the operation.
     @throws {error} - The error encountered during the operation.
     @example
     let m = require('nuclei/ssh');
     let c = m.SSHClient();
     let state = c.Connect('localhost', 22, 'user', 'password');
-    c.Close();
+    let result = c.Close();
     */
     Close() {
         // implemented in go
     };
 
     /**
     @method
-    @description Connect tries to connect to provided host and port with provided username and password with ssh. Returns state of connection and error. If error is not nil, state will be false.
+    @description Connect tries to connect to provided host and port with provided username and password with ssh.
     @param {string} host - The host to connect to.
     @param {number} port - The port to connect to.
     @param {string} username - The username for the connection.
     @param {string} password - The password for the connection.
-    @returns {boolean} - The state of the connection.
-    @throws {error} - The error encountered during the connection.
+    @returns {boolean} - The success state of the operation.
+    @throws {error} - The error encountered during the operation.
     @example
     let m = require('nuclei/ssh');
     let c = m.SSHClient();
@@ -46,11 +46,11 @@ class SSHClient {
 
     /**
     @method
-    @description ConnectSSHInfoMode tries to connect to provided host and port with provided host and port. Returns HandshakeLog and error. If error is not nil, state will be false.
+    @description ConnectSSHInfoMode tries to connect to provided host and port with provided host and port
     @param {string} host - The host to connect to.
     @param {number} port - The port to connect to.
     @returns {HandshakeLog} - The HandshakeLog object containing information about the ssh connection.
-    @throws {error} - The error encountered during the connection.
+    @throws {error} - The error encountered during the operation.
     @example
     let m = require('nuclei/ssh');
     let c = m.SSHClient();
@@ -62,28 +62,28 @@ class SSHClient {
 
     /**
     @method
-    @description ConnectWithKey tries to connect to provided host and port with provided username and private_key. Returns state of connection and error. If error is not nil, state will be false.
+    @description ConnectWithKey tries to connect to provided host and port with provided username and private_key.
     @param {string} host - The host to connect to.
     @param {number} port - The port to connect to.
     @param {string} username - The username for the connection.
     @param {string} key - The private key for the connection.
-    @returns {boolean} - The state of the connection.
-    @throws {error} - The error encountered during the connection.
+    @returns {boolean} - The success state of the operation.
+    @throws {error} - The error encountered during the operation.
     @example
     let m = require('nuclei/ssh');
     let c = m.SSHClient();
-    let result = c.ConnectWithKey('localhost', 22, 'user', 'private_key');
+    let result = c.ConnectWithKey('localhost', 22, 'user', 'key');
     */
     ConnectWithKey(host, port, username, key) {
         // implemented in go
     };
 
     /**
     @method
-    @description Run tries to open a new SSH session, then tries to execute the provided command in said session. Returns string and error. If error is not nil, state will be false. The string contains the command output.
+    @description Run tries to open a new SSH session, then tries to execute the provided command in said session
     @param {string} cmd - The command to execute.
     @returns {string} - The output of the command.
-    @throws {error} - The error encountered during the execution of the command.
+    @throws {error} - The error encountered during the operation.
     @example
     let m = require('nuclei/ssh');
     let c = m.SSHClient();
@@ -92,6 +92,19 @@ class SSHClient {
     Run(cmd) {
         // implemented in go
     };
+
+    /**
+    @method
+    @description SetTimeout sets the timeout for the SSH connection in seconds
+    @param {number} sec - The number of seconds for the timeout.
+    @example
+    let m = require('nuclei/ssh');
+    let c = m.SSHClient();
+    c.SetTimeout(30);
+    */
+    SetTimeout(sec) {
+        // implemented in go
+    };
 };
 
 module.exports = {