From c125df4ef6ec9027bf444bd640e525c3e8cfc785 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Fri, 5 Nov 2021 09:21:52 -0500 Subject: [PATCH 1/6] feat: Checking socks5 proxy before launching a scan #1001 (#1169) * "#issue1001" * changes for #issue1001 * minor changes * minor * flag consolidation and proxy file #issue1001 * readme changes * review changes * enviroment variable changes * review comment changes * review changes * removed commented out code --- README.md | 5 +- README_CN.md | 3 +- v2/cmd/nuclei/main.go | 8 +- v2/internal/runner/options.go | 25 +--- v2/internal/runner/proxy.go | 123 ++++++++++++++++++ v2/internal/testutils/testutils.go | 3 +- v2/pkg/protocols/headless/engine/engine.go | 4 +- .../protocols/headless/engine/http_client.go | 5 +- .../http/httpclientpool/clientpool.go | 42 +++--- v2/pkg/types/proxy.go | 15 +++ v2/pkg/types/types.go | 6 +- 11 files changed, 172 insertions(+), 67 deletions(-) create mode 100644 v2/internal/runner/proxy.go create mode 100644 v2/pkg/types/proxy.go diff --git a/README.md b/README.md index bfa68972e1..3fc76ec7a9 100644 --- a/README.md +++ b/README.md @@ -162,9 +162,8 @@ DEBUG: -debug show all requests and responses -debug-req show all sent requests -debug-resp show all received responses - -proxy, -proxy-url string URL of the HTTP proxy server - -proxy-socks-url string URL of the SOCKS proxy server - -tlog, -trace-log string file to write sent requests trace log + -proxy string List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list + -trace-log string file to write sent requests trace log -version show nuclei version -v, -verbose show verbose output -vv display extra verbose information diff --git a/README_CN.md b/README_CN.md index 04138acd21..0c4aa69c03 100644 --- a/README_CN.md +++ b/README_CN.md @@ -121,8 +121,7 @@ nuclei -h |templates-version|显示已安装的模板版本|nuclei -templates-version| |v|显示发送请求的详细信息|nuclei -v| |version|显示nuclei的版本号|nuclei -version| -|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080| -|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080| +|proxy|输入代理地址|nuclei -proxy ./proxy.txt| |random-agent|使用随机的UA|nuclei -random-agent| |H|自定义请求头|nuclei -H “x-bug-bounty:hacker”| diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9f3..f958ec169b 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -134,11 +134,9 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), - /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc - TODO should auto-set the HTTP_PROXY variable for the process? */ - flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), - flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), - flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), + /* TODO should auto-set the HTTP_PROXY variable for the process? */ + flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list"), + flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 59791fe633..1529466fbf 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -3,7 +3,6 @@ package runner import ( "bufio" "errors" - "net/url" "os" "path/filepath" "strings" @@ -24,7 +23,6 @@ func ParseOptions(options *types.Options) { // Read the inputs and configure the logging configureOutput(options) - // Show the user the banner showBanner() @@ -89,15 +87,10 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - - if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { + //loading the proxy server list from file or cli and test the connectivity + if err := loadProxyServers(options); err != nil { return err } - - if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil { - return err - } - if options.Validate { options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) @@ -106,19 +99,6 @@ func validateOptions(options *types.Options) error { return nil } -func validateProxyURL(proxyURL, message string) error { - if proxyURL != "" && !isValidURL(proxyURL) { - return errors.New(message) - } - - return nil -} - -func isValidURL(urlString string) bool { - _, err := url.Parse(urlString) - return err == nil -} - // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -164,7 +144,6 @@ func loadResolvers(options *types.Options) { func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) { allGivenTemplatePaths := append(templatePaths, workflowPaths...) - for _, templatePath := range allGivenTemplatePaths { if templatesDirectory != templatePath && filepath.IsAbs(templatePath) { fileInfo, err := os.Stat(templatePath) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go new file mode 100644 index 0000000000..e254472a7e --- /dev/null +++ b/v2/internal/runner/proxy.go @@ -0,0 +1,123 @@ +package runner + +import ( + "bufio" + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "time" + + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +var proxyURLList []url.URL + +// loadProxyServers load list of proxy servers from file or comma seperated +func loadProxyServers(options *types.Options) error { + if len(options.Proxy) == 0 { + return nil + } + for _, p := range options.Proxy { + if proxyURL, err := validateProxyURL(p); err == nil { + proxyURLList = append(proxyURLList, proxyURL) + } else if fileutil.FileExists(p) { + file, err := os.Open(p) + if err != nil { + return fmt.Errorf("could not open proxy file: %s", err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + proxy := scanner.Text() + if strings.TrimSpace(proxy) == "" { + continue + } + if proxyURL, err := validateProxyURL(proxy); err != nil { + return err + } else { + proxyURLList = append(proxyURLList, proxyURL) + } + } + } else { + return fmt.Errorf("invalid proxy file or URL provided for %s", p) + } + } + return processProxyList(options) +} + +func processProxyList(options *types.Options) error { + if len(proxyURLList) == 0 { + return fmt.Errorf("could not find any valid proxy") + } else { + done := make(chan bool) + exitCounter := make(chan bool) + counter := 0 + for _, url := range proxyURLList { + go runProxyConnectivity(url, options, done, exitCounter) + } + for { + select { + case <-done: + { + close(done) + return nil + } + case <-exitCounter: + { + if counter += 1; counter == len(proxyURLList) { + return errors.New("no reachable proxy found") + } + } + } + } + } +} + +func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { + if err := testProxyConnection(proxyURL, options.Timeout); err == nil { + if types.ProxyURL == "" && types.ProxySocksURL == "" { + assignProxyURL(proxyURL, options) + done <- true + } + } + exitCounter <- true +} + +func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { + timeout := time.Duration(timeoutDelay) * time.Second + _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout) + if err != nil { + return err + } + return nil +} + +func assignProxyURL(proxyURL url.URL, options *types.Options) { + os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) + if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { + types.ProxyURL = proxyURL.String() + types.ProxySocksURL = "" + gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String()) + } else if proxyURL.Scheme == types.SOCKS5 { + types.ProxyURL = "" + types.ProxySocksURL = proxyURL.String() + gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String()) + } +} + +func validateProxyURL(proxy string) (url.URL, error) { + if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) { + return *url, nil + } + return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") +} + +//isSupportedProtocol checks given protocols are supported +func isSupportedProtocol(value string) bool { + return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 +} diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index d62b1d630b..029f8bb734 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -51,8 +51,7 @@ var DefaultOptions = &types.Options{ Targets: []string{}, TargetsFilePath: "", Output: "", - ProxyURL: "", - ProxySocksURL: "", + Proxy: []string{}, TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a506cbda46..a425958016 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) { } else { chromeLauncher = chromeLauncher.Headless(true) } - if options.ProxyURL != "" { - chromeLauncher = chromeLauncher.Proxy(options.ProxyURL) + if types.ProxyURL != "" { + chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) } launcherURL, err := chromeLauncher.Launch() if err != nil { diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index b4747d5419..71a9421cc3 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -23,9 +23,8 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, }, } - - if options.ProxyURL != "" { - if proxyURL, err := url.Parse(options.ProxyURL); err == nil { + if types.ProxyURL != "" { + if proxyURL, err := url.Parse(types.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index a4faef07d8..b3d953e7e3 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -128,9 +128,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return client, nil } poolMutex.RUnlock() - - if options.ProxyURL != "" { - proxyURL, err = url.Parse(options.ProxyURL) + if types.ProxyURL != "" { + proxyURL, err = url.Parse(types.ProxyURL) } if err != nil { return nil, err @@ -172,27 +171,24 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl }, DisableKeepAlives: disableKeepAlives, } - - // Attempts to overwrite the dial function with the socks proxied version - if options.ProxySocksURL != "" { - var proxyAuth *proxy.Auth - - socksURL, proxyErr := url.Parse(options.ProxySocksURL) - if proxyErr == nil { - proxyAuth = &proxy.Auth{} - proxyAuth.User = socksURL.User.Username() - proxyAuth.Password, _ = socksURL.User.Password() - } - dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) - dc := dialer.(interface { - DialContext(ctx context.Context, network, addr string) (net.Conn, error) - }) - if proxyErr == nil { - transport.DialContext = dc.DialContext - } - } if proxyURL != nil { - transport.Proxy = http.ProxyURL(proxyURL) + // Attempts to overwrite the dial function with the socks proxied version + if proxyURL.Scheme == types.SOCKS5 { + var proxyAuth *proxy.Auth = &proxy.Auth{} + proxyAuth.User = proxyURL.User.Username() + proxyAuth.Password, _ = proxyURL.User.Password() + + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct) + + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } + } else { + transport.Proxy = http.ProxyURL(proxyURL) + } } var jar *cookiejar.Jar diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go new file mode 100644 index 0000000000..a45b4eca67 --- /dev/null +++ b/v2/pkg/types/proxy.go @@ -0,0 +1,15 @@ +package types + +const ( + HTTP_PROXY_ENV = "HTTP_PROXY" + SOCKS5 = "socks5" + HTTP = "http" + HTTPS = "https" +) + +var ( + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string +) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f4245b..5494828904 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -49,10 +49,8 @@ type Options struct { TargetsFilePath string // Output is the file to write found results to. Output string - // ProxyURL is the URL for the proxy server - ProxyURL string - // ProxySocksURL is the URL for the proxy socks server - ProxySocksURL string + // List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list + Proxy goflags.NormalizedStringSlice // TemplatesDirectory is the directory to use for storing templates TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests From 89c6e7677201001b093ef1f792331bfe7562fe15 Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 5 Nov 2021 20:09:00 +0530 Subject: [PATCH 2/6] Revert "feat: Checking socks5 proxy before launching a scan #1001 (#1169)" This reverts commit c125df4ef6ec9027bf444bd640e525c3e8cfc785. --- README.md | 5 +- README_CN.md | 3 +- v2/cmd/nuclei/main.go | 8 +- v2/internal/runner/options.go | 25 +++- v2/internal/runner/proxy.go | 123 ------------------ v2/internal/testutils/testutils.go | 3 +- v2/pkg/protocols/headless/engine/engine.go | 4 +- .../protocols/headless/engine/http_client.go | 5 +- .../http/httpclientpool/clientpool.go | 42 +++--- v2/pkg/types/proxy.go | 15 --- v2/pkg/types/types.go | 6 +- 11 files changed, 67 insertions(+), 172 deletions(-) delete mode 100644 v2/internal/runner/proxy.go delete mode 100644 v2/pkg/types/proxy.go diff --git a/README.md b/README.md index 3fc76ec7a9..bfa68972e1 100644 --- a/README.md +++ b/README.md @@ -162,8 +162,9 @@ DEBUG: -debug show all requests and responses -debug-req show all sent requests -debug-resp show all received responses - -proxy string List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list - -trace-log string file to write sent requests trace log + -proxy, -proxy-url string URL of the HTTP proxy server + -proxy-socks-url string URL of the SOCKS proxy server + -tlog, -trace-log string file to write sent requests trace log -version show nuclei version -v, -verbose show verbose output -vv display extra verbose information diff --git a/README_CN.md b/README_CN.md index 0c4aa69c03..04138acd21 100644 --- a/README_CN.md +++ b/README_CN.md @@ -121,7 +121,8 @@ nuclei -h |templates-version|显示已安装的模板版本|nuclei -templates-version| |v|显示发送请求的详细信息|nuclei -v| |version|显示nuclei的版本号|nuclei -version| -|proxy|输入代理地址|nuclei -proxy ./proxy.txt| +|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080| +|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080| |random-agent|使用随机的UA|nuclei -random-agent| |H|自定义请求头|nuclei -H “x-bug-bounty:hacker”| diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index f958ec169b..d2f8ebb9f3 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -134,9 +134,11 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), - /* TODO should auto-set the HTTP_PROXY variable for the process? */ - flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list"), - flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), + /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc + TODO should auto-set the HTTP_PROXY variable for the process? */ + flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), + flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), + flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 1529466fbf..59791fe633 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -3,6 +3,7 @@ package runner import ( "bufio" "errors" + "net/url" "os" "path/filepath" "strings" @@ -23,6 +24,7 @@ func ParseOptions(options *types.Options) { // Read the inputs and configure the logging configureOutput(options) + // Show the user the banner showBanner() @@ -87,10 +89,15 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - //loading the proxy server list from file or cli and test the connectivity - if err := loadProxyServers(options); err != nil { + + if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { return err } + + if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil { + return err + } + if options.Validate { options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) @@ -99,6 +106,19 @@ func validateOptions(options *types.Options) error { return nil } +func validateProxyURL(proxyURL, message string) error { + if proxyURL != "" && !isValidURL(proxyURL) { + return errors.New(message) + } + + return nil +} + +func isValidURL(urlString string) bool { + _, err := url.Parse(urlString) + return err == nil +} + // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -144,6 +164,7 @@ func loadResolvers(options *types.Options) { func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) { allGivenTemplatePaths := append(templatePaths, workflowPaths...) + for _, templatePath := range allGivenTemplatePaths { if templatesDirectory != templatePath && filepath.IsAbs(templatePath) { fileInfo, err := os.Stat(templatePath) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go deleted file mode 100644 index e254472a7e..0000000000 --- a/v2/internal/runner/proxy.go +++ /dev/null @@ -1,123 +0,0 @@ -package runner - -import ( - "bufio" - "errors" - "fmt" - "net" - "net/url" - "os" - "strings" - "time" - - "github.com/projectdiscovery/fileutil" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/types" -) - -var proxyURLList []url.URL - -// loadProxyServers load list of proxy servers from file or comma seperated -func loadProxyServers(options *types.Options) error { - if len(options.Proxy) == 0 { - return nil - } - for _, p := range options.Proxy { - if proxyURL, err := validateProxyURL(p); err == nil { - proxyURLList = append(proxyURLList, proxyURL) - } else if fileutil.FileExists(p) { - file, err := os.Open(p) - if err != nil { - return fmt.Errorf("could not open proxy file: %s", err) - } - defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - proxy := scanner.Text() - if strings.TrimSpace(proxy) == "" { - continue - } - if proxyURL, err := validateProxyURL(proxy); err != nil { - return err - } else { - proxyURLList = append(proxyURLList, proxyURL) - } - } - } else { - return fmt.Errorf("invalid proxy file or URL provided for %s", p) - } - } - return processProxyList(options) -} - -func processProxyList(options *types.Options) error { - if len(proxyURLList) == 0 { - return fmt.Errorf("could not find any valid proxy") - } else { - done := make(chan bool) - exitCounter := make(chan bool) - counter := 0 - for _, url := range proxyURLList { - go runProxyConnectivity(url, options, done, exitCounter) - } - for { - select { - case <-done: - { - close(done) - return nil - } - case <-exitCounter: - { - if counter += 1; counter == len(proxyURLList) { - return errors.New("no reachable proxy found") - } - } - } - } - } -} - -func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { - if err := testProxyConnection(proxyURL, options.Timeout); err == nil { - if types.ProxyURL == "" && types.ProxySocksURL == "" { - assignProxyURL(proxyURL, options) - done <- true - } - } - exitCounter <- true -} - -func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { - timeout := time.Duration(timeoutDelay) * time.Second - _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout) - if err != nil { - return err - } - return nil -} - -func assignProxyURL(proxyURL url.URL, options *types.Options) { - os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) - if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { - types.ProxyURL = proxyURL.String() - types.ProxySocksURL = "" - gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String()) - } else if proxyURL.Scheme == types.SOCKS5 { - types.ProxyURL = "" - types.ProxySocksURL = proxyURL.String() - gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String()) - } -} - -func validateProxyURL(proxy string) (url.URL, error) { - if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) { - return *url, nil - } - return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") -} - -//isSupportedProtocol checks given protocols are supported -func isSupportedProtocol(value string) bool { - return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 -} diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index 029f8bb734..d62b1d630b 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -51,7 +51,8 @@ var DefaultOptions = &types.Options{ Targets: []string{}, TargetsFilePath: "", Output: "", - Proxy: []string{}, + ProxyURL: "", + ProxySocksURL: "", TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a425958016..a506cbda46 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) { } else { chromeLauncher = chromeLauncher.Headless(true) } - if types.ProxyURL != "" { - chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) + if options.ProxyURL != "" { + chromeLauncher = chromeLauncher.Proxy(options.ProxyURL) } launcherURL, err := chromeLauncher.Launch() if err != nil { diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 71a9421cc3..b4747d5419 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -23,8 +23,9 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, }, } - if types.ProxyURL != "" { - if proxyURL, err := url.Parse(types.ProxyURL); err == nil { + + if options.ProxyURL != "" { + if proxyURL, err := url.Parse(options.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index b3d953e7e3..a4faef07d8 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -128,8 +128,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return client, nil } poolMutex.RUnlock() - if types.ProxyURL != "" { - proxyURL, err = url.Parse(types.ProxyURL) + + if options.ProxyURL != "" { + proxyURL, err = url.Parse(options.ProxyURL) } if err != nil { return nil, err @@ -171,25 +172,28 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl }, DisableKeepAlives: disableKeepAlives, } - if proxyURL != nil { - // Attempts to overwrite the dial function with the socks proxied version - if proxyURL.Scheme == types.SOCKS5 { - var proxyAuth *proxy.Auth = &proxy.Auth{} - proxyAuth.User = proxyURL.User.Username() - proxyAuth.Password, _ = proxyURL.User.Password() - - dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct) - - dc := dialer.(interface { - DialContext(ctx context.Context, network, addr string) (net.Conn, error) - }) - if proxyErr == nil { - transport.DialContext = dc.DialContext - } - } else { - transport.Proxy = http.ProxyURL(proxyURL) + + // Attempts to overwrite the dial function with the socks proxied version + if options.ProxySocksURL != "" { + var proxyAuth *proxy.Auth + + socksURL, proxyErr := url.Parse(options.ProxySocksURL) + if proxyErr == nil { + proxyAuth = &proxy.Auth{} + proxyAuth.User = socksURL.User.Username() + proxyAuth.Password, _ = socksURL.User.Password() + } + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext } } + if proxyURL != nil { + transport.Proxy = http.ProxyURL(proxyURL) + } var jar *cookiejar.Jar if configuration.CookieReuse { diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go deleted file mode 100644 index a45b4eca67..0000000000 --- a/v2/pkg/types/proxy.go +++ /dev/null @@ -1,15 +0,0 @@ -package types - -const ( - HTTP_PROXY_ENV = "HTTP_PROXY" - SOCKS5 = "socks5" - HTTP = "http" - HTTPS = "https" -) - -var ( - // ProxyURL is the URL for the proxy server - ProxyURL string - // ProxySocksURL is the URL for the proxy socks server - ProxySocksURL string -) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 5494828904..d258f4245b 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -49,8 +49,10 @@ type Options struct { TargetsFilePath string // Output is the file to write found results to. Output string - // List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list - Proxy goflags.NormalizedStringSlice + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string // TemplatesDirectory is the directory to use for storing templates TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests From 348d86d330614ba6720f52bdc9e4cbc27fc88daf Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Fri, 5 Nov 2021 20:10:21 +0530 Subject: [PATCH 3/6] Revert "feat: Checking socks5 proxy before launching a scan #1001 (#1169)" This reverts commit c125df4ef6ec9027bf444bd640e525c3e8cfc785. --- README.md | 5 +- README_CN.md | 3 +- v2/cmd/nuclei/main.go | 8 +- v2/internal/runner/options.go | 25 +++- v2/internal/runner/proxy.go | 123 ------------------ v2/internal/testutils/testutils.go | 3 +- v2/pkg/protocols/headless/engine/engine.go | 4 +- .../protocols/headless/engine/http_client.go | 5 +- .../http/httpclientpool/clientpool.go | 42 +++--- v2/pkg/types/proxy.go | 15 --- v2/pkg/types/types.go | 6 +- 11 files changed, 67 insertions(+), 172 deletions(-) delete mode 100644 v2/internal/runner/proxy.go delete mode 100644 v2/pkg/types/proxy.go diff --git a/README.md b/README.md index 3fc76ec7a9..bfa68972e1 100644 --- a/README.md +++ b/README.md @@ -162,8 +162,9 @@ DEBUG: -debug show all requests and responses -debug-req show all sent requests -debug-resp show all received responses - -proxy string List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list - -trace-log string file to write sent requests trace log + -proxy, -proxy-url string URL of the HTTP proxy server + -proxy-socks-url string URL of the SOCKS proxy server + -tlog, -trace-log string file to write sent requests trace log -version show nuclei version -v, -verbose show verbose output -vv display extra verbose information diff --git a/README_CN.md b/README_CN.md index 0c4aa69c03..04138acd21 100644 --- a/README_CN.md +++ b/README_CN.md @@ -121,7 +121,8 @@ nuclei -h |templates-version|显示已安装的模板版本|nuclei -templates-version| |v|显示发送请求的详细信息|nuclei -v| |version|显示nuclei的版本号|nuclei -version| -|proxy|输入代理地址|nuclei -proxy ./proxy.txt| +|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080| +|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080| |random-agent|使用随机的UA|nuclei -random-agent| |H|自定义请求头|nuclei -H “x-bug-bounty:hacker”| diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index f958ec169b..d2f8ebb9f3 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -134,9 +134,11 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), - /* TODO should auto-set the HTTP_PROXY variable for the process? */ - flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list"), - flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), + /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc + TODO should auto-set the HTTP_PROXY variable for the process? */ + flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), + flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), + flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 1529466fbf..59791fe633 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -3,6 +3,7 @@ package runner import ( "bufio" "errors" + "net/url" "os" "path/filepath" "strings" @@ -23,6 +24,7 @@ func ParseOptions(options *types.Options) { // Read the inputs and configure the logging configureOutput(options) + // Show the user the banner showBanner() @@ -87,10 +89,15 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - //loading the proxy server list from file or cli and test the connectivity - if err := loadProxyServers(options); err != nil { + + if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { return err } + + if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil { + return err + } + if options.Validate { options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) @@ -99,6 +106,19 @@ func validateOptions(options *types.Options) error { return nil } +func validateProxyURL(proxyURL, message string) error { + if proxyURL != "" && !isValidURL(proxyURL) { + return errors.New(message) + } + + return nil +} + +func isValidURL(urlString string) bool { + _, err := url.Parse(urlString) + return err == nil +} + // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -144,6 +164,7 @@ func loadResolvers(options *types.Options) { func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) { allGivenTemplatePaths := append(templatePaths, workflowPaths...) + for _, templatePath := range allGivenTemplatePaths { if templatesDirectory != templatePath && filepath.IsAbs(templatePath) { fileInfo, err := os.Stat(templatePath) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go deleted file mode 100644 index e254472a7e..0000000000 --- a/v2/internal/runner/proxy.go +++ /dev/null @@ -1,123 +0,0 @@ -package runner - -import ( - "bufio" - "errors" - "fmt" - "net" - "net/url" - "os" - "strings" - "time" - - "github.com/projectdiscovery/fileutil" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/types" -) - -var proxyURLList []url.URL - -// loadProxyServers load list of proxy servers from file or comma seperated -func loadProxyServers(options *types.Options) error { - if len(options.Proxy) == 0 { - return nil - } - for _, p := range options.Proxy { - if proxyURL, err := validateProxyURL(p); err == nil { - proxyURLList = append(proxyURLList, proxyURL) - } else if fileutil.FileExists(p) { - file, err := os.Open(p) - if err != nil { - return fmt.Errorf("could not open proxy file: %s", err) - } - defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - proxy := scanner.Text() - if strings.TrimSpace(proxy) == "" { - continue - } - if proxyURL, err := validateProxyURL(proxy); err != nil { - return err - } else { - proxyURLList = append(proxyURLList, proxyURL) - } - } - } else { - return fmt.Errorf("invalid proxy file or URL provided for %s", p) - } - } - return processProxyList(options) -} - -func processProxyList(options *types.Options) error { - if len(proxyURLList) == 0 { - return fmt.Errorf("could not find any valid proxy") - } else { - done := make(chan bool) - exitCounter := make(chan bool) - counter := 0 - for _, url := range proxyURLList { - go runProxyConnectivity(url, options, done, exitCounter) - } - for { - select { - case <-done: - { - close(done) - return nil - } - case <-exitCounter: - { - if counter += 1; counter == len(proxyURLList) { - return errors.New("no reachable proxy found") - } - } - } - } - } -} - -func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { - if err := testProxyConnection(proxyURL, options.Timeout); err == nil { - if types.ProxyURL == "" && types.ProxySocksURL == "" { - assignProxyURL(proxyURL, options) - done <- true - } - } - exitCounter <- true -} - -func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { - timeout := time.Duration(timeoutDelay) * time.Second - _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout) - if err != nil { - return err - } - return nil -} - -func assignProxyURL(proxyURL url.URL, options *types.Options) { - os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) - if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { - types.ProxyURL = proxyURL.String() - types.ProxySocksURL = "" - gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String()) - } else if proxyURL.Scheme == types.SOCKS5 { - types.ProxyURL = "" - types.ProxySocksURL = proxyURL.String() - gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String()) - } -} - -func validateProxyURL(proxy string) (url.URL, error) { - if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) { - return *url, nil - } - return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") -} - -//isSupportedProtocol checks given protocols are supported -func isSupportedProtocol(value string) bool { - return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 -} diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index 029f8bb734..d62b1d630b 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -51,7 +51,8 @@ var DefaultOptions = &types.Options{ Targets: []string{}, TargetsFilePath: "", Output: "", - Proxy: []string{}, + ProxyURL: "", + ProxySocksURL: "", TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a425958016..a506cbda46 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) { } else { chromeLauncher = chromeLauncher.Headless(true) } - if types.ProxyURL != "" { - chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) + if options.ProxyURL != "" { + chromeLauncher = chromeLauncher.Proxy(options.ProxyURL) } launcherURL, err := chromeLauncher.Launch() if err != nil { diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 71a9421cc3..b4747d5419 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -23,8 +23,9 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, }, } - if types.ProxyURL != "" { - if proxyURL, err := url.Parse(types.ProxyURL); err == nil { + + if options.ProxyURL != "" { + if proxyURL, err := url.Parse(options.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index b3d953e7e3..a4faef07d8 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -128,8 +128,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return client, nil } poolMutex.RUnlock() - if types.ProxyURL != "" { - proxyURL, err = url.Parse(types.ProxyURL) + + if options.ProxyURL != "" { + proxyURL, err = url.Parse(options.ProxyURL) } if err != nil { return nil, err @@ -171,25 +172,28 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl }, DisableKeepAlives: disableKeepAlives, } - if proxyURL != nil { - // Attempts to overwrite the dial function with the socks proxied version - if proxyURL.Scheme == types.SOCKS5 { - var proxyAuth *proxy.Auth = &proxy.Auth{} - proxyAuth.User = proxyURL.User.Username() - proxyAuth.Password, _ = proxyURL.User.Password() - - dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct) - - dc := dialer.(interface { - DialContext(ctx context.Context, network, addr string) (net.Conn, error) - }) - if proxyErr == nil { - transport.DialContext = dc.DialContext - } - } else { - transport.Proxy = http.ProxyURL(proxyURL) + + // Attempts to overwrite the dial function with the socks proxied version + if options.ProxySocksURL != "" { + var proxyAuth *proxy.Auth + + socksURL, proxyErr := url.Parse(options.ProxySocksURL) + if proxyErr == nil { + proxyAuth = &proxy.Auth{} + proxyAuth.User = socksURL.User.Username() + proxyAuth.Password, _ = socksURL.User.Password() + } + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext } } + if proxyURL != nil { + transport.Proxy = http.ProxyURL(proxyURL) + } var jar *cookiejar.Jar if configuration.CookieReuse { diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go deleted file mode 100644 index a45b4eca67..0000000000 --- a/v2/pkg/types/proxy.go +++ /dev/null @@ -1,15 +0,0 @@ -package types - -const ( - HTTP_PROXY_ENV = "HTTP_PROXY" - SOCKS5 = "socks5" - HTTP = "http" - HTTPS = "https" -) - -var ( - // ProxyURL is the URL for the proxy server - ProxyURL string - // ProxySocksURL is the URL for the proxy socks server - ProxySocksURL string -) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 5494828904..d258f4245b 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -49,8 +49,10 @@ type Options struct { TargetsFilePath string // Output is the file to write found results to. Output string - // List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list - Proxy goflags.NormalizedStringSlice + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string // TemplatesDirectory is the directory to use for storing templates TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests From 6c819d7917f2779f32cce266ed6e56ea783df1e2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 23 Nov 2021 10:44:31 +0530 Subject: [PATCH 4/6] Added additional variables for network + simplified logic --- v2/pkg/protocols/network/network.go | 16 ++------- v2/pkg/protocols/network/network_test.go | 15 ++------- v2/pkg/protocols/network/request.go | 41 ++++++++++++++---------- v2/pkg/protocols/network/request_test.go | 9 ++---- 4 files changed, 32 insertions(+), 49 deletions(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index a5274eff68..4495d0b033 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -2,7 +2,6 @@ package network import ( "fmt" - "net" "strings" "github.com/pkg/errors" @@ -78,9 +77,8 @@ type Request struct { } type addressKV struct { - ip string - port string - tls bool + address string + tls bool } // Input is the input to send on the network @@ -136,15 +134,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { shouldUseTLS = true address = strings.TrimPrefix(address, "tls://") } - if strings.Contains(address, ":") { - addressHost, addressPort, portErr := net.SplitHostPort(address) - if portErr != nil { - return errors.Wrap(portErr, "could not parse address") - } - request.addresses = append(request.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS}) - } else { - request.addresses = append(request.addresses, addressKV{ip: address, tls: shouldUseTLS}) - } + request.addresses = append(request.addresses, addressKV{address: address, tls: shouldUseTLS}) } // Pre-compile any input dsl functions before executing the request. for _, input := range request.Inputs { diff --git a/v2/pkg/protocols/network/network_test.go b/v2/pkg/protocols/network/network_test.go index 180b34192c..733211a9e5 100644 --- a/v2/pkg/protocols/network/network_test.go +++ b/v2/pkg/protocols/network/network_test.go @@ -17,7 +17,7 @@ func TestNetworkCompileMake(t *testing.T) { templateID := "testing-network" request := &Request{ ID: templateID, - Address: []string{"{{Hostname}}", "{{Hostname}}:8082", "tls://{{Hostname}}:443"}, + Address: []string{"tls://{{Hostname}}:443"}, ReadSize: 1024, Inputs: []*Input{{Data: "test-data"}}, } @@ -28,17 +28,8 @@ func TestNetworkCompileMake(t *testing.T) { err := request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") - require.Equal(t, 3, len(request.addresses), "could not get correct number of input address") - t.Run("check-host", func(t *testing.T) { - require.Equal(t, "{{Hostname}}", request.addresses[0].ip, "could not get correct host") - }) - t.Run("check-host-with-port", func(t *testing.T) { - require.Equal(t, "{{Hostname}}", request.addresses[1].ip, "could not get correct host with port") - require.Equal(t, "8082", request.addresses[1].port, "could not get correct port for host") - }) + require.Equal(t, 1, len(request.addresses), "could not get correct number of input address") t.Run("check-tls-with-port", func(t *testing.T) { - require.Equal(t, "{{Hostname}}", request.addresses[2].ip, "could not get correct host with port") - require.Equal(t, "443", request.addresses[2].port, "could not get correct port for host") - require.True(t, request.addresses[2].tls, "could not get correct port for host") + require.True(t, request.addresses[0].tls, "could not get correct port for host") }) } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index acbbac9af1..2446bc4db1 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -41,18 +41,10 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } for _, kv := range request.addresses { - actualAddress := replacer.Replace(kv.ip, map[string]interface{}{"Hostname": address}) - if kv.port != "" { - if strings.Contains(address, ":") { - actualAddress, _, _ = net.SplitHostPort(actualAddress) - } - actualAddress = net.JoinHostPort(actualAddress, kv.port) - } - if input != "" { - input = actualAddress - } + variables := generateNetworkVariables(address) + actualAddress := replacer.Replace(kv.address, variables) - if err := request.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { + if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } @@ -61,7 +53,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } // executeAddress executes the request for an address -func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") request.options.Output.Request(request.options.TemplateID, address, "network", err) @@ -80,27 +72,27 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho break } value = generators.MergeMaps(value, payloads) - if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } } else { - value := generators.MergeMaps(map[string]interface{}{}, payloads) - if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + value := generators.CopyMap(payloads) + if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } return nil } -func (request *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var ( hostname string conn net.Conn err error ) - request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) + request.dynamicValues = generators.MergeMaps(payloads, variables) if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host @@ -257,3 +249,18 @@ func getAddress(toTest string) (string, error) { } return toTest, nil } + +func generateNetworkVariables(input string) map[string]interface{} { + if !strings.Contains(input, ":") { + return map[string]interface{}{"Hostname": input, "Host": input} + } + host, port, err := net.SplitHostPort(input) + if err != nil { + return map[string]interface{}{"Hostname": input} + } + return map[string]interface{}{ + "Host": host, + "Port": port, + "Hostname": input, + } +} diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index d1f266b48e..4cc578919f 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -50,7 +50,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { parsed, err := url.Parse(ts.URL) require.Nil(t, err, "could not parse url") - request.Address[0] = "{{Hostname}}:" + parsed.Port() + request.Address[0] = "{{Hostname}}" request.Inputs = append(request.Inputs, &Input{Data: fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host)}) executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ @@ -84,12 +84,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { }) require.Nil(t, err, "could not execute network request") }) - require.NotNil(t, finalEvent, "could not get event output from request") - require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") - require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") - require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results") - require.Equal(t, "

Example Domain

", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") - finalEvent = nil + require.Nil(t, finalEvent, "could not get event output from request") request.Inputs[0].Type = "hex" request.Inputs[0].Data = hex.EncodeToString([]byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host))) From 5a1c7a62b2d6041895050c48bc000ab54e3c6ec4 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 28 Nov 2021 04:55:18 +0530 Subject: [PATCH 5/6] missing goimports --- v2/pkg/protocols/network/network.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 82eb4aa50a..ef03cc92b5 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -1,8 +1,6 @@ package network import ( - "fmt" - "net" "strings" "github.com/pkg/errors" From 684f332599f674210512003e3bdef545578eb73f Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 30 Nov 2021 12:04:47 +0530 Subject: [PATCH 6/6] fix: {{Hostname}} to {{Host}} in test --- v2/pkg/protocols/network/network_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/network/network_test.go b/v2/pkg/protocols/network/network_test.go index 730f9584ad..27a64d592a 100644 --- a/v2/pkg/protocols/network/network_test.go +++ b/v2/pkg/protocols/network/network_test.go @@ -17,7 +17,7 @@ func TestNetworkCompileMake(t *testing.T) { templateID := "testing-network" request := &Request{ ID: templateID, - Address: []string{"tls://{{Hostname}}:443"}, + Address: []string{"tls://{{Host}}:443"}, ReadSize: 1024, Inputs: []*Input{{Data: "test-data"}}, }