From f53b945a5c3d88807a6797db66f732f961fc03f9 Mon Sep 17 00:00:00 2001 From: ayuxsec <249073442+ayuxsec@users.noreply.github.com> Date: Fri, 26 Dec 2025 17:37:29 +0530 Subject: [PATCH 01/39] Add Markdown output support - Introduce `MarkDownOutput` flag and CLI option (`-markdown` / `-md`). - Implement `Result.MarkdownOutput` to generate a formatted Markdown table with optional title and CDN columns, and a response body preview. - Add markdown escaping helper. - Update runner to create `.md` output file, handle markdown generation per result, and write to file or stdout. - Adjust option parsing and related logic to include markdown handling. --- runner/md_output.go | 67 +++++++++++++++++++++++++++++++++++++++++++++ runner/options.go | 2 ++ runner/runner.go | 24 ++++++++++++++-- 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 runner/md_output.go diff --git a/runner/md_output.go b/runner/md_output.go new file mode 100644 index 00000000..76e24218 --- /dev/null +++ b/runner/md_output.go @@ -0,0 +1,67 @@ +package runner + +import ( + "fmt" + "net/http" + "strings" +) + +func (r Result) MarkdownOutput(scanopts *ScanOptions) string { + var b strings.Builder + + // Table Header + b.WriteString("| URL | Status | Method | IP | Size | Words | Lines |") + if r.Title != "" { + b.WriteString(" Title |") + } + if r.CDNName != "" { + b.WriteString(" CDN |") + } + b.WriteString("\n") + + // Table Separator + b.WriteString("|---|---|---|---|---|---|---|") + if r.Title != "" { + b.WriteString("---|") + } + if r.CDNName != "" { + b.WriteString("---|") + } + b.WriteString("\n") + + // Table Data Row + fmt.Fprintf(&b, "| %s | `%d %s` | `%s` | `%s` | %d | %d | %d |", + r.URL, + r.StatusCode, http.StatusText(r.StatusCode), + r.Method, + r.HostIP, + r.ContentLength, + r.Words, + r.Lines) + + if r.Title != "" { + fmt.Fprintf(&b, " %s |", escapeMarkdown(r.Title)) + } + if r.CDNName != "" { + fmt.Fprintf(&b, " `%s` |", r.CDNName) + } + b.WriteString("\n\n") + + // Response Body Code Block + if r.BodyPreview != "" { + b.WriteString("**Response Body Preview:**\n") + b.WriteString("```text\n") + b.WriteString(r.BodyPreview) + b.WriteString("\n```\n") + } + + return b.String() +} + +func escapeMarkdown(s string) string { + replacer := strings.NewReplacer( + "|", "\\|", + "\n", " ", + ) + return strings.TrimSpace(replacer.Replace(s)) +} diff --git a/runner/options.go b/runner/options.go index c5c98d7d..e2b7a4c8 100644 --- a/runner/options.go +++ b/runner/options.go @@ -222,6 +222,7 @@ type Options struct { RespectHSTS bool StoreResponse bool JSONOutput bool + MarkDownOutput bool CSVOutput bool CSVOutputEncoding string PdcpAuth string @@ -478,6 +479,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"), flagSet.StringVarP(&options.CSVOutputEncoding, "csv-output-encoding", "csvo", "", "define output encoding"), flagSet.BoolVarP(&options.JSONOutput, "json", "j", false, "store output in JSONL(ines) format"), + flagSet.BoolVarP(&options.MarkDownOutput, "markdown", "md", false, "store output in Markdown table format"), flagSet.BoolVarP(&options.ResponseHeadersInStdout, "include-response-header", "irh", false, "include http response (headers) in JSON output (-json only)"), flagSet.BoolVarP(&options.ResponseInStdout, "include-response", "irr", false, "include http request/response (headers + body) in JSON output (-json only)"), flagSet.BoolVarP(&options.Base64ResponseInStdout, "include-response-base64", "irrb", false, "include base64 encoded http request/response in JSON output (-json only)"), diff --git a/runner/runner.go b/runner/runner.go index 2bf4dd44..1d707f16 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -815,7 +815,7 @@ func (r *Runner) RunEnumeration() { } }() - var plainFile, jsonFile, csvFile, indexFile, indexScreenshotFile *os.File + var plainFile, jsonFile, csvFile, mdFile, indexFile, indexScreenshotFile *os.File if r.options.Output != "" && r.options.OutputAll { plainFile = openOrCreateFile(r.options.Resume, r.options.Output) @@ -832,7 +832,17 @@ func (r *Runner) RunEnumeration() { }() } - jsonOrCsv := (r.options.JSONOutput || r.options.CSVOutput) + if r.options.Output != "" && r.options.MarkDownOutput { + mdFile = openOrCreateFile( + r.options.Resume, + r.options.Output+".md", + ) + defer func() { + _ = mdFile.Close() + }() + } + + jsonOrCsv := (r.options.JSONOutput || r.options.CSVOutput || r.options.MarkDownOutput) jsonAndCsv := (r.options.JSONOutput && r.options.CSVOutput) if r.options.Output != "" && plainFile == nil && !jsonOrCsv { plainFile = openOrCreateFile(r.options.Resume, r.options.Output) @@ -1228,6 +1238,16 @@ func (r *Runner) RunEnumeration() { } } + if r.options.MarkDownOutput { + row := resp.MarkdownOutput(&r.scanopts) + if !r.options.OutputAll { + gologger.Silent().Msgf("%s\n", row) + } + if mdFile != nil { + mdFile.WriteString(row + "\n") + } + } + for _, nextStep := range nextSteps { nextStep <- resp } From 15177106a4e2ab2c9fcc59f03fbaa5b1b55743f6 Mon Sep 17 00:00:00 2001 From: ayuxsec <249073442+ayuxsec@users.noreply.github.com> Date: Sat, 27 Dec 2025 00:31:56 +0530 Subject: [PATCH 02/39] fix(md output): emit markdown table header only once per run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Split Markdown generation into `MarkdownHeader` and `MarkdownRow` for clearer separation. - Remove body‑preview block from Markdown rows. - Add guard in runner to write the header only once per run. --- runner/md_output.go | 21 ++++++++------------- runner/runner.go | 19 ++++++++++++++++--- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/runner/md_output.go b/runner/md_output.go index 76e24218..4b577774 100644 --- a/runner/md_output.go +++ b/runner/md_output.go @@ -6,10 +6,9 @@ import ( "strings" ) -func (r Result) MarkdownOutput(scanopts *ScanOptions) string { +func MarkdownHeader(r Result) string { var b strings.Builder - // Table Header b.WriteString("| URL | Status | Method | IP | Size | Words | Lines |") if r.Title != "" { b.WriteString(" Title |") @@ -19,7 +18,6 @@ func (r Result) MarkdownOutput(scanopts *ScanOptions) string { } b.WriteString("\n") - // Table Separator b.WriteString("|---|---|---|---|---|---|---|") if r.Title != "" { b.WriteString("---|") @@ -29,7 +27,12 @@ func (r Result) MarkdownOutput(scanopts *ScanOptions) string { } b.WriteString("\n") - // Table Data Row + return b.String() +} + +func (r Result) MarkdownRow(scanopts *ScanOptions) string { + var b strings.Builder + fmt.Fprintf(&b, "| %s | `%d %s` | `%s` | `%s` | %d | %d | %d |", r.URL, r.StatusCode, http.StatusText(r.StatusCode), @@ -45,15 +48,7 @@ func (r Result) MarkdownOutput(scanopts *ScanOptions) string { if r.CDNName != "" { fmt.Fprintf(&b, " `%s` |", r.CDNName) } - b.WriteString("\n\n") - - // Response Body Code Block - if r.BodyPreview != "" { - b.WriteString("**Response Body Preview:**\n") - b.WriteString("```text\n") - b.WriteString(r.BodyPreview) - b.WriteString("\n```\n") - } + b.WriteString("\n") return b.String() } diff --git a/runner/runner.go b/runner/runner.go index 1d707f16..f66c8488 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -816,6 +816,7 @@ func (r *Runner) RunEnumeration() { }() var plainFile, jsonFile, csvFile, mdFile, indexFile, indexScreenshotFile *os.File + markdownHeaderWritten := false // guard to prevent writing the header multiple times if r.options.Output != "" && r.options.OutputAll { plainFile = openOrCreateFile(r.options.Resume, r.options.Output) @@ -1239,12 +1240,24 @@ func (r *Runner) RunEnumeration() { } if r.options.MarkDownOutput { - row := resp.MarkdownOutput(&r.scanopts) + if !markdownHeaderWritten { + header := MarkdownHeader(resp) + if !r.options.OutputAll { + gologger.Silent().Msgf("%s", header) + } + if mdFile != nil { + mdFile.WriteString(header) + } + markdownHeaderWritten = true + } + + row := resp.MarkdownRow(&r.scanopts) + if !r.options.OutputAll { - gologger.Silent().Msgf("%s\n", row) + gologger.Silent().Msgf("%s", row) } if mdFile != nil { - mdFile.WriteString(row + "\n") + mdFile.WriteString(row) } } From cabc1aef20fa120e6d32ab04f510eeefc7356837 Mon Sep 17 00:00:00 2001 From: ayuxsec <249073442+ayuxsec@users.noreply.github.com> Date: Sat, 27 Dec 2025 00:44:57 +0530 Subject: [PATCH 03/39] =?UTF-8?q?fix(md=20output):=20always=20include=20Ti?= =?UTF-8?q?tle=E2=80=AF&=E2=80=AFCDN=20columns=20and=20escape=20URLs.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- runner/md_output.go | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/runner/md_output.go b/runner/md_output.go index 4b577774..3428210e 100644 --- a/runner/md_output.go +++ b/runner/md_output.go @@ -8,23 +8,9 @@ import ( func MarkdownHeader(r Result) string { var b strings.Builder - - b.WriteString("| URL | Status | Method | IP | Size | Words | Lines |") - if r.Title != "" { - b.WriteString(" Title |") - } - if r.CDNName != "" { - b.WriteString(" CDN |") - } + b.WriteString("| URL | Status | Method | IP | Size | Words | Lines | Title | CDN |") b.WriteString("\n") - - b.WriteString("|---|---|---|---|---|---|---|") - if r.Title != "" { - b.WriteString("---|") - } - if r.CDNName != "" { - b.WriteString("---|") - } + b.WriteString("|---|---|---|---|---|---|---|---|---|") b.WriteString("\n") return b.String() @@ -34,7 +20,7 @@ func (r Result) MarkdownRow(scanopts *ScanOptions) string { var b strings.Builder fmt.Fprintf(&b, "| %s | `%d %s` | `%s` | `%s` | %d | %d | %d |", - r.URL, + escapeMarkdown(r.URL), r.StatusCode, http.StatusText(r.StatusCode), r.Method, r.HostIP, @@ -44,12 +30,17 @@ func (r Result) MarkdownRow(scanopts *ScanOptions) string { if r.Title != "" { fmt.Fprintf(&b, " %s |", escapeMarkdown(r.Title)) + } else { + b.WriteString(" |") } + if r.CDNName != "" { fmt.Fprintf(&b, " `%s` |", r.CDNName) + } else { + b.WriteString(" |") } - b.WriteString("\n") + b.WriteString("\n") return b.String() } From b1f4809c2cc3c11cd56a445b0abc6935c45a14f0 Mon Sep 17 00:00:00 2001 From: ayuxsec <249073442+ayuxsec@users.noreply.github.com> Date: Sat, 27 Dec 2025 04:54:56 +0530 Subject: [PATCH 04/39] =?UTF-8?q?fix(mdoutput):=20Fix=20duplicate=20?= =?UTF-8?q?=E2=80=9C.md=E2=80=9D=20extension=20when=20using=20`-o`=20with?= =?UTF-8?q?=20`-md`=20flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Consolidate output‑type checks into a single `jsonOrCsvOrMD` variable. * Adjust file‑opening logic to add the “.md” suffix only when needed. * Update related conditionals to respect the new combined flag handling. * Ensure markdown output is correctly written without creating “.md.md” files. --- runner/runner.go | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index f66c8488..e5c5414f 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -831,21 +831,15 @@ func (r *Runner) RunEnumeration() { defer func() { _ = csvFile.Close() }() - } - - if r.options.Output != "" && r.options.MarkDownOutput { - mdFile = openOrCreateFile( - r.options.Resume, - r.options.Output+".md", - ) + mdFile = openOrCreateFile(r.options.Resume, r.options.Output+".md") defer func() { _ = mdFile.Close() }() } - jsonOrCsv := (r.options.JSONOutput || r.options.CSVOutput || r.options.MarkDownOutput) - jsonAndCsv := (r.options.JSONOutput && r.options.CSVOutput) - if r.options.Output != "" && plainFile == nil && !jsonOrCsv { + jsonOrCsvOrMD := (r.options.JSONOutput || r.options.CSVOutput || r.options.MarkDownOutput) + jsonAndCsvAndMD := (r.options.JSONOutput && r.options.CSVOutput && r.options.MarkDownOutput) + if r.options.Output != "" && plainFile == nil && !jsonOrCsvOrMD { plainFile = openOrCreateFile(r.options.Resume, r.options.Output) defer func() { _ = plainFile.Close() @@ -854,7 +848,7 @@ func (r *Runner) RunEnumeration() { if r.options.Output != "" && r.options.JSONOutput && jsonFile == nil { ext := "" - if jsonAndCsv { + if jsonAndCsvAndMD { ext = ".json" } jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+ext) @@ -865,7 +859,7 @@ func (r *Runner) RunEnumeration() { if r.options.Output != "" && r.options.CSVOutput && csvFile == nil { ext := "" - if jsonAndCsv { + if jsonAndCsvAndMD { ext = ".csv" } csvFile = openOrCreateFile(r.options.Resume, r.options.Output+ext) @@ -874,6 +868,17 @@ func (r *Runner) RunEnumeration() { }() } + if r.options.Output != "" && r.options.MarkDownOutput && mdFile == nil { + ext := "" + if jsonAndCsvAndMD { + ext = ".md" + } + mdFile = openOrCreateFile(r.options.Resume, r.options.Output+ext) + defer func() { + _ = mdFile.Close() + }() + } + if r.options.CSVOutput { outEncoding := strings.ToLower(r.options.CSVOutputEncoding) switch outEncoding { @@ -888,7 +893,7 @@ func (r *Runner) RunEnumeration() { gologger.Fatal().Msgf("unknown csv output encoding: %s\n", r.options.CSVOutputEncoding) } headers := Result{}.CSVHeader() - if !r.options.OutputAll && !jsonAndCsv { + if !r.options.OutputAll && !jsonAndCsvAndMD { gologger.Silent().Msgf("%s\n", headers) } @@ -1103,7 +1108,7 @@ func (r *Runner) RunEnumeration() { } } - if !r.options.DisableStdout && (!jsonOrCsv || jsonAndCsv || r.options.OutputAll) { + if !r.options.DisableStdout && (!jsonOrCsvOrMD || jsonAndCsvAndMD || r.options.OutputAll) { gologger.Silent().Msgf("%s\n", resp.str) } @@ -1216,7 +1221,7 @@ func (r *Runner) RunEnumeration() { if r.options.JSONOutput { row := resp.JSON(&r.scanopts) - if !r.options.OutputAll && !jsonAndCsv { + if !r.options.OutputAll && !jsonAndCsvAndMD { gologger.Silent().Msgf("%s\n", row) } @@ -1229,7 +1234,7 @@ func (r *Runner) RunEnumeration() { if r.options.CSVOutput { row := resp.CSVRow(&r.scanopts) - if !r.options.OutputAll && !jsonAndCsv { + if !r.options.OutputAll && !jsonAndCsvAndMD { gologger.Silent().Msgf("%s\n", row) } @@ -1239,7 +1244,7 @@ func (r *Runner) RunEnumeration() { } } - if r.options.MarkDownOutput { + if r.options.MarkDownOutput || r.options.OutputAll { if !markdownHeaderWritten { header := MarkdownHeader(resp) if !r.options.OutputAll { From 3bcbe1b3b1a0b83e08418c5e2f5bc00da15b08f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Sat, 3 Jan 2026 17:44:30 +0300 Subject: [PATCH 05/39] fix: handle errcheck lint errors --- runner/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index e5c5414f..50cf0a56 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1251,7 +1251,7 @@ func (r *Runner) RunEnumeration() { gologger.Silent().Msgf("%s", header) } if mdFile != nil { - mdFile.WriteString(header) + _, _ = mdFile.WriteString(header) } markdownHeaderWritten = true } @@ -1262,7 +1262,7 @@ func (r *Runner) RunEnumeration() { gologger.Silent().Msgf("%s", row) } if mdFile != nil { - mdFile.WriteString(row) + _, _ = mdFile.WriteString(row) } } From bad939e139b9e8028d83a89d2f3bf0b2a558a002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Sat, 3 Jan 2026 17:51:16 +0300 Subject: [PATCH 06/39] refactor: make MarkdownHeader a method on Result --- runner/md_output.go | 2 +- runner/runner.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/md_output.go b/runner/md_output.go index 3428210e..0eca258d 100644 --- a/runner/md_output.go +++ b/runner/md_output.go @@ -6,7 +6,7 @@ import ( "strings" ) -func MarkdownHeader(r Result) string { +func (r Result) MarkdownHeader() string { //nolint var b strings.Builder b.WriteString("| URL | Status | Method | IP | Size | Words | Lines | Title | CDN |") b.WriteString("\n") diff --git a/runner/runner.go b/runner/runner.go index 50cf0a56..b541bb12 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1246,7 +1246,7 @@ func (r *Runner) RunEnumeration() { if r.options.MarkDownOutput || r.options.OutputAll { if !markdownHeaderWritten { - header := MarkdownHeader(resp) + header := resp.MarkdownHeader() if !r.options.OutputAll { gologger.Silent().Msgf("%s", header) } From 6399f7808019c7b9172c7342182515f73683e3c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Sat, 3 Jan 2026 18:05:13 +0300 Subject: [PATCH 07/39] refactor: dynamic markdown header/row generation from struct tags --- runner/md_output.go | 89 ++++++++++++++++++++-------- runner/types.go | 138 ++++++++++++++++++++++---------------------- 2 files changed, 135 insertions(+), 92 deletions(-) diff --git a/runner/md_output.go b/runner/md_output.go index 0eca258d..6aac73f6 100644 --- a/runner/md_output.go +++ b/runner/md_output.go @@ -2,48 +2,91 @@ package runner import ( "fmt" - "net/http" + "reflect" "strings" ) func (r Result) MarkdownHeader() string { //nolint + var headers []string + + t := reflect.TypeOf(r) + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + tag := field.Tag.Get("md") + if tag == "" || tag == "-" { + continue + } + headers = append(headers, tag) + } + var b strings.Builder - b.WriteString("| URL | Status | Method | IP | Size | Words | Lines | Title | CDN |") + b.WriteString("|") + for _, h := range headers { + fmt.Fprintf(&b, " %s |", h) + } b.WriteString("\n") - b.WriteString("|---|---|---|---|---|---|---|---|---|") + + b.WriteString("|") + for range headers { + b.WriteString("---|") + } b.WriteString("\n") return b.String() } -func (r Result) MarkdownRow(scanopts *ScanOptions) string { - var b strings.Builder +func (r Result) MarkdownRow(scanopts *ScanOptions) string { //nolint + var values []string - fmt.Fprintf(&b, "| %s | `%d %s` | `%s` | `%s` | %d | %d | %d |", - escapeMarkdown(r.URL), - r.StatusCode, http.StatusText(r.StatusCode), - r.Method, - r.HostIP, - r.ContentLength, - r.Words, - r.Lines) - - if r.Title != "" { - fmt.Fprintf(&b, " %s |", escapeMarkdown(r.Title)) - } else { - b.WriteString(" |") - } + v := reflect.ValueOf(r) + t := v.Type() - if r.CDNName != "" { - fmt.Fprintf(&b, " `%s` |", r.CDNName) - } else { - b.WriteString(" |") + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + tag := field.Tag.Get("md") + if tag == "" || tag == "-" { + continue + } + + fieldValue := v.Field(i) + values = append(values, formatMarkdownValue(fieldValue)) } + var b strings.Builder + b.WriteString("|") + for _, val := range values { + fmt.Fprintf(&b, " %s |", val) + } b.WriteString("\n") + return b.String() } +func formatMarkdownValue(v reflect.Value) string { + switch v.Kind() { + case reflect.String: + return escapeMarkdown(v.String()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return fmt.Sprintf("%d", v.Int()) + case reflect.Bool: + return fmt.Sprintf("%t", v.Bool()) + case reflect.Slice: + if v.Len() == 0 { + return "" + } + var items []string + for i := 0; i < v.Len(); i++ { + items = append(items, fmt.Sprintf("%v", v.Index(i).Interface())) + } + return escapeMarkdown(strings.Join(items, ", ")) + default: + if v.CanInterface() { + return escapeMarkdown(fmt.Sprintf("%v", v.Interface())) + } + return "" + } +} + func escapeMarkdown(s string) string { replacer := strings.NewReplacer( "|", "\\|", diff --git a/runner/types.go b/runner/types.go index ab013a70..9acd0e7d 100644 --- a/runner/types.go +++ b/runner/types.go @@ -33,75 +33,75 @@ func (o AsnResponse) String() string { // Result of a scan type Result struct { - Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" mapstructure:"timestamp"` - LinkRequest []NetworkRequest `json:"link_request,omitempty" csv:"link_request" mapstructure:"link_request"` - ASN *AsnResponse `json:"asn,omitempty" csv:"-" mapstructure:"asn"` - Err error `json:"-" csv:"-" mapstructure:"-"` - CSPData *httpx.CSPData `json:"csp,omitempty" csv:"-" mapstructure:"csp"` - TLSData *clients.Response `json:"tls,omitempty" csv:"-" mapstructure:"tls"` - Hashes map[string]interface{} `json:"hash,omitempty" csv:"-" mapstructure:"hash"` - ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex" mapstructure:"extract_regex"` - CDNName string `json:"cdn_name,omitempty" csv:"cdn_name" mapstructure:"cdn_name"` - CDNType string `json:"cdn_type,omitempty" csv:"cdn_type" mapstructure:"cdn_type"` - SNI string `json:"sni,omitempty" csv:"sni" mapstructure:"sni"` - Port string `json:"port,omitempty" csv:"port" mapstructure:"port"` - Raw string `json:"-" csv:"-" mapstructure:"-"` - URL string `json:"url,omitempty" csv:"url" mapstructure:"url"` - Input string `json:"input,omitempty" csv:"input" mapstructure:"input"` - Location string `json:"location,omitempty" csv:"location" mapstructure:"location"` - Title string `json:"title,omitempty" csv:"title" mapstructure:"title"` - str string `json:"-" csv:"-" mapstructure:"-"` - Scheme string `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"` - Error string `json:"error,omitempty" csv:"error" mapstructure:"error"` - WebServer string `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"` - ResponseBody string `json:"body,omitempty" csv:"-" mapstructure:"body"` - BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"` - ContentType string `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"` - Method string `json:"method,omitempty" csv:"method" mapstructure:"method"` - Host string `json:"host,omitempty" csv:"host" mapstructure:"host"` - HostIP string `json:"host_ip,omitempty" csv:"host_ip" mapstructure:"host_ip"` - Path string `json:"path,omitempty" csv:"path" mapstructure:"path"` - FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"` - FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon_md5" mapstructure:"favicon_md5"` - FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"` - FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"` - FinalURL string `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"` - ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"-" mapstructure:"header"` - RawHeaders string `json:"raw_header,omitempty" csv:"-" mapstructure:"raw_header"` - Request string `json:"request,omitempty" csv:"-" mapstructure:"request"` - ResponseTime string `json:"time,omitempty" csv:"time" mapstructure:"time"` - JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"` - ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"` - A []string `json:"a,omitempty" csv:"a" mapstructure:"a"` - AAAA []string `json:"aaaa,omitempty" csv:"aaaa" mapstructure:"aaaa"` - CNAMEs []string `json:"cname,omitempty" csv:"cname" mapstructure:"cname"` - Technologies []string `json:"tech,omitempty" csv:"tech" mapstructure:"tech"` - Extracts map[string][]string `json:"extracts,omitempty" csv:"-" mapstructure:"extracts"` - Chain []httpx.ChainItem `json:"chain,omitempty" csv:"-" mapstructure:"chain"` - Words int `json:"words" csv:"words" mapstructure:"words"` - Lines int `json:"lines" csv:"lines" mapstructure:"lines"` - StatusCode int `json:"status_code" csv:"status_code" mapstructure:"status_code"` - ContentLength int `json:"content_length" csv:"content_length" mapstructure:"content_length"` - Failed bool `json:"failed" csv:"failed" mapstructure:"failed"` - VHost bool `json:"vhost,omitempty" csv:"vhost" mapstructure:"vhost"` - WebSocket bool `json:"websocket,omitempty" csv:"websocket" mapstructure:"websocket"` - CDN bool `json:"cdn,omitempty" csv:"cdn" mapstructure:"cdn"` - HTTP2 bool `json:"http2,omitempty" csv:"http2" mapstructure:"http2"` - Pipeline bool `json:"pipeline,omitempty" csv:"pipeline" mapstructure:"pipeline"` - HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body" mapstructure:"headless_body"` - ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" mapstructure:"screenshot_bytes"` - StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path" mapstructure:"stored_response_path"` - ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path" mapstructure:"screenshot_path"` - ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" mapstructure:"screenshot_path_rel"` - KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"-" mapstructure:"knowledgebase"` - Resolvers []string `json:"resolvers,omitempty" csv:"resolvers" mapstructure:"resolvers"` - Fqdns []string `json:"body_fqdn,omitempty" csv:"body_fqdn" mapstructure:"body_fqdn"` - Domains []string `json:"body_domains,omitempty" csv:"body_domains" mapstructure:"body_domains"` - TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-" mapstructure:"-"` - RequestRaw []byte `json:"-" csv:"-" mapstructure:"-"` - Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"` - FaviconData []byte `json:"-" csv:"-" mapstructure:"-"` - Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" mapstructure:"trace"` + Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" md:"timestamp" mapstructure:"timestamp"` + LinkRequest []NetworkRequest `json:"link_request,omitempty" csv:"link_request" md:"link_request" mapstructure:"link_request"` + ASN *AsnResponse `json:"asn,omitempty" csv:"-" md:"-" mapstructure:"asn"` + Err error `json:"-" csv:"-" md:"-" mapstructure:"-"` + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"-" md:"-" mapstructure:"csp"` + TLSData *clients.Response `json:"tls,omitempty" csv:"-" md:"-" mapstructure:"tls"` + Hashes map[string]interface{} `json:"hash,omitempty" csv:"-" md:"-" mapstructure:"hash"` + ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex" md:"extract_regex" mapstructure:"extract_regex"` + CDNName string `json:"cdn_name,omitempty" csv:"cdn_name" md:"cdn_name" mapstructure:"cdn_name"` + CDNType string `json:"cdn_type,omitempty" csv:"cdn_type" md:"cdn_type" mapstructure:"cdn_type"` + SNI string `json:"sni,omitempty" csv:"sni" md:"sni" mapstructure:"sni"` + Port string `json:"port,omitempty" csv:"port" md:"port" mapstructure:"port"` + Raw string `json:"-" csv:"-" md:"-" mapstructure:"-"` + URL string `json:"url,omitempty" csv:"url" md:"url" mapstructure:"url"` + Input string `json:"input,omitempty" csv:"input" md:"input" mapstructure:"input"` + Location string `json:"location,omitempty" csv:"location" md:"location" mapstructure:"location"` + Title string `json:"title,omitempty" csv:"title" md:"title" mapstructure:"title"` + str string `json:"-" csv:"-" md:"-" mapstructure:"-"` + Scheme string `json:"scheme,omitempty" csv:"scheme" md:"scheme" mapstructure:"scheme"` + Error string `json:"error,omitempty" csv:"error" md:"error" mapstructure:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver" md:"webserver" mapstructure:"webserver"` + ResponseBody string `json:"body,omitempty" csv:"-" md:"-" mapstructure:"body"` + BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" md:"body_preview" mapstructure:"body_preview"` + ContentType string `json:"content_type,omitempty" csv:"content_type" md:"content_type" mapstructure:"content_type"` + Method string `json:"method,omitempty" csv:"method" md:"method" mapstructure:"method"` + Host string `json:"host,omitempty" csv:"host" md:"host" mapstructure:"host"` + HostIP string `json:"host_ip,omitempty" csv:"host_ip" md:"host_ip" mapstructure:"host_ip"` + Path string `json:"path,omitempty" csv:"path" md:"path" mapstructure:"path"` + FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" md:"favicon" mapstructure:"favicon"` + FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon_md5" md:"favicon_md5" mapstructure:"favicon_md5"` + FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" md:"favicon_path" mapstructure:"favicon_path"` + FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" md:"favicon_url" mapstructure:"favicon_url"` + FinalURL string `json:"final_url,omitempty" csv:"final_url" md:"final_url" mapstructure:"final_url"` + ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"-" md:"-" mapstructure:"header"` + RawHeaders string `json:"raw_header,omitempty" csv:"-" md:"-" mapstructure:"raw_header"` + Request string `json:"request,omitempty" csv:"-" md:"-" mapstructure:"request"` + ResponseTime string `json:"time,omitempty" csv:"time" md:"time" mapstructure:"time"` + JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" md:"jarm_hash" mapstructure:"jarm_hash"` + ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" md:"chain_status_codes" mapstructure:"chain_status_codes"` + A []string `json:"a,omitempty" csv:"a" md:"a" mapstructure:"a"` + AAAA []string `json:"aaaa,omitempty" csv:"aaaa" md:"aaaa" mapstructure:"aaaa"` + CNAMEs []string `json:"cname,omitempty" csv:"cname" md:"cname" mapstructure:"cname"` + Technologies []string `json:"tech,omitempty" csv:"tech" md:"tech" mapstructure:"tech"` + Extracts map[string][]string `json:"extracts,omitempty" csv:"-" md:"-" mapstructure:"extracts"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"-" md:"-" mapstructure:"chain"` + Words int `json:"words" csv:"words" md:"words" mapstructure:"words"` + Lines int `json:"lines" csv:"lines" md:"lines" mapstructure:"lines"` + StatusCode int `json:"status_code" csv:"status_code" md:"status_code" mapstructure:"status_code"` + ContentLength int `json:"content_length" csv:"content_length" md:"content_length" mapstructure:"content_length"` + Failed bool `json:"failed" csv:"failed" md:"failed" mapstructure:"failed"` + VHost bool `json:"vhost,omitempty" csv:"vhost" md:"vhost" mapstructure:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket" md:"websocket" mapstructure:"websocket"` + CDN bool `json:"cdn,omitempty" csv:"cdn" md:"cdn" mapstructure:"cdn"` + HTTP2 bool `json:"http2,omitempty" csv:"http2" md:"http2" mapstructure:"http2"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline" md:"pipeline" mapstructure:"pipeline"` + HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body" md:"headless_body" mapstructure:"headless_body"` + ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" md:"screenshot_bytes" mapstructure:"screenshot_bytes"` + StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path" md:"stored_response_path" mapstructure:"stored_response_path"` + ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path" md:"screenshot_path" mapstructure:"screenshot_path"` + ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" md:"screenshot_path_rel" mapstructure:"screenshot_path_rel"` + KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"-" md:"-" mapstructure:"knowledgebase"` + Resolvers []string `json:"resolvers,omitempty" csv:"resolvers" md:"resolvers" mapstructure:"resolvers"` + Fqdns []string `json:"body_fqdn,omitempty" csv:"body_fqdn" md:"body_fqdn" mapstructure:"body_fqdn"` + Domains []string `json:"body_domains,omitempty" csv:"body_domains" md:"body_domains" mapstructure:"body_domains"` + TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-" md:"-" mapstructure:"-"` + RequestRaw []byte `json:"-" csv:"-" md:"-" mapstructure:"-"` + Response *httpx.Response `json:"-" csv:"-" md:"-" mapstructure:"-"` + FaviconData []byte `json:"-" csv:"-" md:"-" mapstructure:"-"` + Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" md:"-" mapstructure:"trace"` } type Trace struct { From 822257198b229c5a0f979aab2da1b0b60f65c4fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Sat, 3 Jan 2026 18:45:09 +0300 Subject: [PATCH 08/39] fix resolver parsing --- runner/options.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index c5c98d7d..58183df6 100644 --- a/runner/options.go +++ b/runner/options.go @@ -735,7 +735,17 @@ func (options *Options) ValidateOptions() error { return errors.Wrapf(err, "Couldn't process resolver file \"%s\"", resolver) } for line := range chFile { - resolvers = append(resolvers, line) + line = strings.TrimSpace(line) + if line != "" && strings.Contains(line, ",") { + for item := range strings.SplitSeq(line, ",") { + item = strings.TrimSpace(item) + if item != "" { + resolvers = append(resolvers, item) + } + } + } else { + resolvers = append(resolvers, line) + } } } else { resolvers = append(resolvers, resolver) From 2ea576f175325f584b871b52f418facb94b5c248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 6 Jan 2026 12:53:09 +0300 Subject: [PATCH 09/39] feat: add database output support for storing scan results Add support for storing httpx scan results directly to databases with both CLI flags and YAML config file options. Supported databases: - MongoDB - PostgreSQL - MySQL Features: - Batched writes for performance (configurable batch size) - Auto-flush with configurable interval - Individual columns for each Result field (not JSON blob) - Support for environment variable HTTPX_DB_CONNECTION_STRING - Option to omit raw request/response data (-rdbor) New CLI flags under OUTPUT group: - -rdb, -result-db: enable database storage - -rdbc, -result-db-config: path to YAML config file - -rdbt, -result-db-type: database type (mongodb, postgres, mysql) - -rdbcs, -result-db-conn: connection string - -rdbn, -result-db-name: database name (default: httpx) - -rdbtb, -result-db-table: table/collection name (default: results) - -rdbbs, -result-db-batch-size: batch size (default: 100) - -rdbor, -result-db-omit-raw: omit raw data Closes #1973 Closes #2360 Closes #2361 Closes #2362 --- README.md | 98 +++++++-------- cmd/httpx/httpx.go | 63 ++++++++++ go.mod | 10 +- go.sum | 17 +++ internal/db/config.go | 125 ++++++++++++++++++++ internal/db/db.go | 70 +++++++++++ internal/db/mongodb.go | 127 ++++++++++++++++++++ internal/db/mysql.go | 256 ++++++++++++++++++++++++++++++++++++++++ internal/db/postgres.go | 256 ++++++++++++++++++++++++++++++++++++++++ internal/db/writer.go | 148 +++++++++++++++++++++++ runner/options.go | 17 +++ 11 files changed, 1140 insertions(+), 47 deletions(-) create mode 100644 internal/db/config.go create mode 100644 internal/db/db.go create mode 100644 internal/db/mongodb.go create mode 100644 internal/db/mysql.go create mode 100644 internal/db/postgres.go create mode 100644 internal/db/writer.go diff --git a/README.md b/README.md index 283e6691..af2dd799 100644 --- a/README.md +++ b/README.md @@ -97,42 +97,40 @@ INPUT: -u, -target string[] input target host(s) to probe PROBES: - -sc, -status-code display response status-code - -cl, -content-length display response content-length - -ct, -content-type display response content-type - -location display response redirect location - -favicon display mmh3 hash for '/favicon.ico' file - -hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512) - -jarm display jarm fingerprint hash - -rt, -response-time display response time - -lc, -line-count display response body line count - -wc, -word-count display response body word count - -title display page title - -bp, -body-preview display first N characters of response body (default 100) - -server, -web-server display server name + -sc, -status-code display response status-code + -cl, -content-length display response content-length + -ct, -content-type display response content-type + -location display response redirect location + -favicon display mmh3 hash for '/favicon.ico' file + -hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512) + -jarm display jarm fingerprint hash + -rt, -response-time display response time + -lc, -line-count display response body line count + -wc, -word-count display response body word count + -title display page title + -bp, -body-preview display first N characters of response body (default 100) + -server, -web-server display server name -td, -tech-detect display technology in use based on wappalyzer dataset -cff, -custom-fingerprint-file string path to a custom fingerprint file for technology detection - -cpe display CPE (Common Platform Enumeration) based on awesome-search-queries - -wp, -wordpress display WordPress plugins and themes -method display http request method - -ws, -websocket display server using websocket - -ip display host ip - -cname display host cname - -extract-fqdn, -efqdn get domain and subdomains from response body and header in jsonl/csv output - -asn display host asn information - -cdn display cdn/waf in use (default true) - -probe display probe status + -ws, -websocket display server using websocket + -ip display host ip + -cname display host cname + -extract-fqdn, -efqdn get domain and subdomains from response body and header in jsonl/csv output + -asn display host asn information + -cdn display cdn/waf in use (default true) + -probe display probe status HEADLESS: -ss, -screenshot enable saving screenshot of the page using headless browser -system-chrome enable using local installed chrome for screenshot -ho, -headless-options string[] start headless chrome with additional options -esb, -exclude-screenshot-bytes enable excluding screenshot bytes from json output - -no-screenshot-full-page disable saving full page screenshot -ehb, -exclude-headless-body enable excluding headless header from json output + -no-screenshot-full-page disable saving full page screenshot -st, -screenshot-timeout value set timeout for screenshot in seconds (default 10s) -sid, -screenshot-idle value set idle time before taking screenshot in seconds (default 1s) - -jsc, -javascript-code string[] execute JavaScript code after navigation + -jsc, -javascript-code string[] execute JavaScript code after navigation MATCHERS: -mc, -match-code string match response with specified status code (-mc 200,302) @@ -142,7 +140,7 @@ MATCHERS: -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) -ms, -match-string string[] match response with specified string (-ms admin) -mr, -match-regex string[] match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google) + -mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google, etc.) -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') -mdc, -match-condition string match response with dsl expression condition @@ -151,19 +149,21 @@ EXTRACTOR: -ep, -extract-preset string[] display response content matched by a pre-defined regex (url,ipv4,mail) FILTERS: - -fc, -filter-code string filter response with specified status code (-fc 403,401) - -fep, -filter-error-page filter response with ML based error page detection - -fd, -filter-duplicates filter out near-duplicate responses (only first response is retained) - -fl, -filter-length string filter response with specified content length (-fl 23,33) - -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) - -fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532) - -ffc, -filter-favicon string[] filter response with specified favicon hash (-ffc 1494302000) - -fs, -filter-string string[] filter response with specified string (-fs admin) - -fe, -filter-regex string[] filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google) - -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') - -fdc, -filter-condition string filter response with dsl expression condition - -strip strips all tags in response. supported formats: html,xml (default html) + -fc, -filter-code string filter response with specified status code (-fc 403,401) + -fep, -filter-error-page filter response with ML based error page detection + -fd, -filter-duplicates filter out near-duplicate responses (only first response is retained) + -fl, -filter-length string filter response with specified content length (-fl 23,33) + -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) + -fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532) + -ffc, -filter-favicon string[] filter response with specified favicon hash (-ffc 1494302000) + -fs, -filter-string string[] filter response with specified string (-fs admin) + -fe, -filter-regex string[] filter response with specified regex (-fe admin) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google, etc.) + -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') + -fdc, -filter-condition string filter response with dsl expression condition + -strip strips all tags in response. supported formats: html,xml (default html) + -lof, -list-output-fields list of fields to output (comma separated) + -eof, -exclude-output-fields string[] exclude output fields output based on a condition RATE-LIMIT: -t, -threads int number of threads to use (default 50) @@ -201,10 +201,16 @@ OUTPUT: -include-chain include redirect http chain in JSON output (-json only) -store-chain include http redirect chain in responses (-sr only) -svrc, -store-vision-recon-cluster include visual recon clusters (-ss and -sr only) - -pr, -protocol string protocol to use (unknown, http11) + -pr, -protocol string protocol to use (unknown, http11, http2, http3) -fepp, -filter-error-page-path string path to store filtered error pages (default "filtered_error_page.json") - -lof, -list-output-fields list available output field names for filtering - -eof, -exclude-output-fields string[] exclude specified output fields from results + -rdb, -result-db store results in database + -rdbc, -result-db-config string path to database config file + -rdbt, -result-db-type string database type (mongodb, postgres, mysql) + -rdbcs, -result-db-conn string database connection string (env: HTTPX_DB_CONNECTION_STRING) + -rdbn, -result-db-name string database name (default "httpx") + -rdbtb, -result-db-table string table/collection name (default "results") + -rdbbs, -result-db-batch-size int batch size for database inserts (default 100) + -rdbor, -result-db-omit-raw omit raw request/response data from database CONFIGURATIONS: -config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml) @@ -213,9 +219,9 @@ CONFIGURATIONS: -deny string[] denied list of IP/CIDR's to process (file or comma separated) -sni, -sni-name string custom TLS SNI name -random-agent enable Random User-Agent to use (default true) - -auto-referer set the Referer header to the current URL (default false) + -auto-referer set the Referer header to the current URL -H, -header string[] custom http headers to send with request - -http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080) + -http-proxy, -proxy string proxy (http|socks) to use (eg http://127.0.0.1:8080) -unsafe send raw requests skipping golang normalization -resume resume scan using resume.cfg -fr, -follow-redirects follow http redirects @@ -250,14 +256,14 @@ DEBUG: OPTIMIZATIONS: -nf, -no-fallback display both probed protocol (HTTPS and HTTP) - -nfs, -no-fallback-scheme probe with protocol scheme specified in input + -nfs, -no-fallback-scheme probe with protocol scheme specified in input -maxhr, -max-host-error int max error count per host before skipping remaining path/s (default 30) -e, -exclude string[] exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex) -retries int number of retries -timeout int timeout in seconds (default 10) -delay value duration between each http request (eg: 200ms, 1s) (default -1ns) - -rsts, -response-size-to-save int max response size to save in bytes (default 2147483647) - -rstr, -response-size-to-read int max response size to read in bytes (default 2147483647) + -rsts, -response-size-to-save int max response size to save in bytes (default 512000000) + -rstr, -response-size-to-read int max response size to read in bytes (default 512000000) CLOUD: -auth configure projectdiscovery cloud (pdcp) api key (default true) diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 77e54f4c..f7b26810 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -10,6 +10,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/httpx/internal/db" "github.com/projectdiscovery/httpx/internal/pdcp" "github.com/projectdiscovery/httpx/runner" pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" @@ -64,6 +65,9 @@ func main() { // setup optional asset upload _ = setupOptionalAssetUpload(options) + // setup optional database output + _ = setupDatabaseOutput(options) + httpxRunner, err := runner.New(options) if err != nil { gologger.Fatal().Msgf("Could not create runner: %s\n", err) @@ -143,3 +147,62 @@ func setupOptionalAssetUpload(opts *runner.Options) *pdcp.UploadWriter { } return writer } + +// setupDatabaseOutput sets up database output for storing results +// This is optional and only initialized when explicitly enabled via -rdb flag +func setupDatabaseOutput(opts *runner.Options) *db.Writer { + if !opts.ResultDatabase { + return nil + } + + var cfg *db.Config + var err error + + if opts.ResultDatabaseConfig != "" { + // Load configuration from file + cfg, err = db.LoadConfigFromFile(opts.ResultDatabaseConfig) + if err != nil { + gologger.Fatal().Msgf("Could not load database config: %s\n", err) + } + } else { + // Build configuration from CLI options + dbOpts := &db.Options{ + Enabled: opts.ResultDatabase, + Type: opts.ResultDatabaseType, + ConnectionString: opts.ResultDatabaseConnStr, + DatabaseName: opts.ResultDatabaseName, + TableName: opts.ResultDatabaseTable, + BatchSize: opts.ResultDatabaseBatchSize, + OmitRaw: opts.ResultDatabaseOmitRaw, + } + cfg, err = dbOpts.ToConfig() + if err != nil { + gologger.Fatal().Msgf("Invalid database configuration: %s\n", err) + } + } + + writer, err := db.NewWriter(context.Background(), cfg) + if err != nil { + gologger.Fatal().Msgf("Could not setup database output: %s\n", err) + } + + // Chain with existing OnResult callback if present + existingCallback := opts.OnResult + opts.OnResult = func(r runner.Result) { + if existingCallback != nil { + existingCallback(r) + } + writer.GetWriterCallback()(r) + } + + // Chain with existing OnClose callback if present + existingClose := opts.OnClose + opts.OnClose = func() { + writer.Close() + if existingClose != nil { + existingClose() + } + } + + return writer +} diff --git a/go.mod b/go.mod index 1ac2fdd6..97ebc341 100644 --- a/go.mod +++ b/go.mod @@ -52,13 +52,18 @@ require ( require ( github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 github.com/dustin/go-humanize v1.0.1 + github.com/go-sql-driver/mysql v1.9.3 github.com/go-viper/mapstructure/v2 v2.4.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 + github.com/lib/pq v1.10.9 github.com/weppos/publicsuffix-go v0.50.2 + go.mongodb.org/mongo-driver v1.17.6 + gopkg.in/yaml.v3 v3.0.1 ) require ( aead.dev/minisign v0.2.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect github.com/JohannesKaufmann/dom v0.2.0 // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect @@ -121,6 +126,7 @@ require ( github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect @@ -157,6 +163,9 @@ require ( github.com/ulikunitz/xz v0.5.15 // indirect github.com/vulncheck-oss/go-exploit v1.51.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/ysmood/fetchup v0.2.3 // indirect github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/got v0.40.0 // indirect @@ -176,5 +185,4 @@ require ( golang.org/x/time v0.11.0 // indirect golang.org/x/tools v0.39.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 0188b44c..163c35d7 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= @@ -141,6 +143,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA= github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg= +github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= +github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= @@ -252,6 +256,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= @@ -288,6 +294,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= @@ -462,10 +470,16 @@ github.com/weppos/publicsuffix-go v0.50.2 h1:KsJFc8IEKTJovM46SRCnGNsM+rFShxcs6VE github.com/weppos/publicsuffix-go v0.50.2/go.mod h1:CbQCKDtXF8UcT7hrxeMa0MDjwhpOI9iYOU7cfq+yo8k= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= @@ -502,6 +516,8 @@ github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db/go.mod h1:mo/07mo6reD github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= +go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= +go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -684,6 +700,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= diff --git a/internal/db/config.go b/internal/db/config.go new file mode 100644 index 00000000..c72dcac3 --- /dev/null +++ b/internal/db/config.go @@ -0,0 +1,125 @@ +package db + +import ( + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + DefaultBatchSize = 100 + + DefaultFlushInterval = time.Minute + + DefaultTableName = "results" + + DefaultDatabaseName = "httpx" + + EnvConnectionString = "HTTPX_DB_CONNECTION_STRING" +) + +type Config struct { + Type DatabaseType `yaml:"type"` + + ConnectionString string `yaml:"connection-string"` + + DatabaseName string `yaml:"database-name"` + + TableName string `yaml:"table-name"` + + BatchSize int `yaml:"batch-size"` + + FlushInterval time.Duration `yaml:"flush-interval"` + + OmitRaw bool `yaml:"omit-raw"` +} + +func (c *Config) Validate() error { + if !c.Type.IsValid() { + return fmt.Errorf("invalid database type: %s (supported: %v)", c.Type, SupportedDatabases()) + } + + if c.ConnectionString == "" { + return fmt.Errorf("connection string is required") + } + + return nil +} + +func (c *Config) ApplyDefaults() { + if c.DatabaseName == "" { + c.DatabaseName = DefaultDatabaseName + } + + if c.TableName == "" { + c.TableName = DefaultTableName + } + + if c.BatchSize <= 0 { + c.BatchSize = DefaultBatchSize + } + + if c.FlushInterval <= 0 { + c.FlushInterval = DefaultFlushInterval + } +} + +func LoadConfigFromFile(configPath string) (*Config, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + var cfg Config + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + + if cfg.ConnectionString == "" { + cfg.ConnectionString = os.Getenv(EnvConnectionString) + } + + cfg.ApplyDefaults() + + if err := cfg.Validate(); err != nil { + return nil, err + } + + return &cfg, nil +} + +type Options struct { + Enabled bool + ConfigFile string + Type string + ConnectionString string + DatabaseName string + TableName string + BatchSize int + OmitRaw bool +} + +func (o *Options) ToConfig() (*Config, error) { + cfg := &Config{ + Type: DatabaseType(o.Type), + ConnectionString: o.ConnectionString, + DatabaseName: o.DatabaseName, + TableName: o.TableName, + BatchSize: o.BatchSize, + OmitRaw: o.OmitRaw, + } + + if cfg.ConnectionString == "" { + cfg.ConnectionString = os.Getenv(EnvConnectionString) + } + + cfg.ApplyDefaults() + + if err := cfg.Validate(); err != nil { + return nil, err + } + + return cfg, nil +} diff --git a/internal/db/db.go b/internal/db/db.go new file mode 100644 index 00000000..a4f7e717 --- /dev/null +++ b/internal/db/db.go @@ -0,0 +1,70 @@ +package db + +import ( + "context" + "fmt" + + "github.com/projectdiscovery/httpx/runner" +) + +type DatabaseType string + +const ( + MongoDB DatabaseType = "mongodb" + PostgreSQL DatabaseType = "postgres" + MySQL DatabaseType = "mysql" +) + +func (d DatabaseType) String() string { + return string(d) +} + +func (d DatabaseType) IsValid() bool { + switch d { + case MongoDB, PostgreSQL, MySQL: + return true + default: + return false + } +} + +type Database interface { + Connect(ctx context.Context) error + + Close() error + + InsertBatch(ctx context.Context, results []runner.Result) error + + EnsureSchema(ctx context.Context) error + + Type() DatabaseType +} + +type databaseFactory func(cfg *Config) (Database, error) + +var registry = make(map[DatabaseType]databaseFactory) + +func Register(dbType DatabaseType, factory databaseFactory) { + registry[dbType] = factory +} + +func NewDatabase(cfg *Config) (Database, error) { + if cfg == nil { + return nil, fmt.Errorf("database configuration is required") + } + + if !cfg.Type.IsValid() { + return nil, fmt.Errorf("unsupported database type: %s", cfg.Type) + } + + factory, ok := registry[cfg.Type] + if !ok { + return nil, fmt.Errorf("database type %s is not registered", cfg.Type) + } + + return factory(cfg) +} + +func SupportedDatabases() []DatabaseType { + return []DatabaseType{MongoDB, PostgreSQL, MySQL} +} diff --git a/internal/db/mongodb.go b/internal/db/mongodb.go new file mode 100644 index 00000000..5db69729 --- /dev/null +++ b/internal/db/mongodb.go @@ -0,0 +1,127 @@ +package db + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/projectdiscovery/httpx/runner" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func init() { + Register(MongoDB, newMongoDatabase) +} + +type mongoDatabase struct { + cfg *Config + client *mongo.Client + database *mongo.Database + collection *mongo.Collection +} + +func newMongoDatabase(cfg *Config) (Database, error) { + return &mongoDatabase{cfg: cfg}, nil +} + +func (m *mongoDatabase) Connect(ctx context.Context) error { + clientOpts := options.Client(). + ApplyURI(m.cfg.ConnectionString). + SetConnectTimeout(10 * time.Second). + SetServerSelectionTimeout(10 * time.Second) + + client, err := mongo.Connect(ctx, clientOpts) + if err != nil { + return fmt.Errorf("failed to connect to MongoDB: %w", err) + } + + if err := client.Ping(ctx, nil); err != nil { + return fmt.Errorf("failed to ping MongoDB: %w", err) + } + + m.client = client + m.database = client.Database(m.cfg.DatabaseName) + m.collection = m.database.Collection(m.cfg.TableName) + + return nil +} + +func (m *mongoDatabase) Close() error { + if m.client != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return m.client.Disconnect(ctx) + } + return nil +} + +func (m *mongoDatabase) EnsureSchema(ctx context.Context) error { + indexes := []mongo.IndexModel{ + { + Keys: bson.D{{Key: "timestamp", Value: -1}}, + }, + { + Keys: bson.D{{Key: "url", Value: 1}}, + }, + { + Keys: bson.D{{Key: "host", Value: 1}}, + }, + { + Keys: bson.D{{Key: "status_code", Value: 1}}, + }, + { + Keys: bson.D{{Key: "tech", Value: 1}}, + Options: options.Index().SetSparse(true), + }, + } + + _, err := m.collection.Indexes().CreateMany(ctx, indexes) + if err != nil { + return fmt.Errorf("failed to create indexes: %w", err) + } + + return nil +} + +func (m *mongoDatabase) InsertBatch(ctx context.Context, results []runner.Result) error { + if len(results) == 0 { + return nil + } + + documents := make([]interface{}, len(results)) + for i, r := range results { + doc, err := m.resultToDocument(r) + if err != nil { + return fmt.Errorf("failed to convert result to document: %w", err) + } + documents[i] = doc + } + + _, err := m.collection.InsertMany(ctx, documents) + if err != nil { + return fmt.Errorf("failed to insert batch: %w", err) + } + + return nil +} + +func (m *mongoDatabase) Type() DatabaseType { + return MongoDB +} + +func (m *mongoDatabase) resultToDocument(r runner.Result) (bson.M, error) { + jsonBytes, err := json.Marshal(r) + if err != nil { + return nil, fmt.Errorf("failed to marshal result to JSON: %w", err) + } + + var doc bson.M + if err := json.Unmarshal(jsonBytes, &doc); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON to BSON: %w", err) + } + + return doc, nil +} diff --git a/internal/db/mysql.go b/internal/db/mysql.go new file mode 100644 index 00000000..50434985 --- /dev/null +++ b/internal/db/mysql.go @@ -0,0 +1,256 @@ +package db + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + + _ "github.com/go-sql-driver/mysql" + "github.com/projectdiscovery/httpx/runner" +) + +func init() { + Register(MySQL, newMySQLDatabase) +} + +type mysqlDatabase struct { + cfg *Config + db *sql.DB +} + +func newMySQLDatabase(cfg *Config) (Database, error) { + return &mysqlDatabase{cfg: cfg}, nil +} + +func (m *mysqlDatabase) Connect(ctx context.Context) error { + db, err := sql.Open("mysql", m.cfg.ConnectionString) + if err != nil { + return fmt.Errorf("failed to open MySQL connection: %w", err) + } + + if err := db.PingContext(ctx); err != nil { + return fmt.Errorf("failed to ping MySQL: %w", err) + } + + m.db = db + return nil +} + +func (m *mysqlDatabase) Close() error { + if m.db != nil { + return m.db.Close() + } + return nil +} + +func (m *mysqlDatabase) EnsureSchema(ctx context.Context) error { + schema := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + + -- Basic info + timestamp DATETIME(6), + url TEXT, + input TEXT, + host VARCHAR(255), + port VARCHAR(10), + scheme VARCHAR(10), + path TEXT, + method VARCHAR(10), + final_url TEXT, + + -- Response data + status_code INT, + content_length INT, + content_type VARCHAR(255), + title TEXT, + webserver VARCHAR(255), + response_time VARCHAR(50), + location TEXT, + body LONGTEXT, + body_preview TEXT, + raw_header LONGTEXT, + request LONGTEXT, + + -- Network info + host_ip VARCHAR(45), + a JSON, + aaaa JSON, + cname JSON, + resolvers JSON, + body_fqdn JSON, + body_domains JSON, + sni TEXT, + + -- Technology detection + tech JSON, + + -- Hashes and fingerprints + hash JSON, + favicon VARCHAR(100), + favicon_md5 VARCHAR(32), + favicon_path TEXT, + favicon_url TEXT, + jarm_hash VARCHAR(62), + + -- CDN info + cdn BOOLEAN, + cdn_name VARCHAR(100), + cdn_type VARCHAR(50), + + -- ASN info + asn JSON, + + -- TLS data + tls JSON, + + -- CSP data + csp JSON, + + -- Status flags + failed BOOLEAN, + error TEXT, + websocket BOOLEAN, + http2 BOOLEAN, + pipeline BOOLEAN, + vhost BOOLEAN, + + -- Metrics + words INT, + ` + "`lines`" + ` INT, + + -- Headers and extracts + header JSON, + extracts JSON, + extract_regex JSON, + + -- Chain data + chain JSON, + chain_status_codes JSON, + + -- Headless/Screenshot + headless_body LONGTEXT, + screenshot_bytes LONGBLOB, + screenshot_path TEXT, + screenshot_path_rel TEXT, + stored_response_path TEXT, + + -- Knowledge base + knowledgebase JSON, + + -- Link requests + link_request JSON, + + -- Trace + trace JSON, + + INDEX idx_timestamp (timestamp), + INDEX idx_host (host), + INDEX idx_status_code (status_code) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + `, m.cfg.TableName) + + _, err := m.db.ExecContext(ctx, schema) + if err != nil { + return fmt.Errorf("failed to create schema: %w", err) + } + + return nil +} + +func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result) error { + if len(results) == 0 { + return nil + } + + tx, err := m.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer func() { + _ = tx.Rollback() + }() + + query := fmt.Sprintf(` + INSERT INTO %s ( + timestamp, url, input, host, port, scheme, path, method, final_url, + status_code, content_length, content_type, title, webserver, response_time, + location, body, body_preview, raw_header, request, + host_ip, a, aaaa, cname, resolvers, body_fqdn, body_domains, sni, + tech, hash, favicon, favicon_md5, favicon_path, favicon_url, jarm_hash, + cdn, cdn_name, cdn_type, asn, tls, csp, + failed, error, websocket, http2, pipeline, vhost, + words, `+"`lines`"+`, header, extracts, extract_regex, + chain, chain_status_codes, + headless_body, screenshot_bytes, screenshot_path, screenshot_path_rel, stored_response_path, + knowledgebase, link_request, trace + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, + ?, ?, + ?, ?, ?, ?, ?, + ?, ?, ? + )`, m.cfg.TableName) + + stmt, err := tx.PrepareContext(ctx, query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, r := range results { + aJSON, _ := json.Marshal(r.A) + aaaaJSON, _ := json.Marshal(r.AAAA) + cnameJSON, _ := json.Marshal(r.CNAMEs) + resolversJSON, _ := json.Marshal(r.Resolvers) + fqdnJSON, _ := json.Marshal(r.Fqdns) + domainsJSON, _ := json.Marshal(r.Domains) + techJSON, _ := json.Marshal(r.Technologies) + hashJSON, _ := json.Marshal(r.Hashes) + asnJSON, _ := json.Marshal(r.ASN) + tlsJSON, _ := json.Marshal(r.TLSData) + cspJSON, _ := json.Marshal(r.CSPData) + headerJSON, _ := json.Marshal(r.ResponseHeaders) + extractsJSON, _ := json.Marshal(r.Extracts) + extractRegexJSON, _ := json.Marshal(r.ExtractRegex) + chainJSON, _ := json.Marshal(r.Chain) + chainStatusJSON, _ := json.Marshal(r.ChainStatusCodes) + kbJSON, _ := json.Marshal(r.KnowledgeBase) + linkReqJSON, _ := json.Marshal(r.LinkRequest) + traceJSON, _ := json.Marshal(r.Trace) + + _, err = stmt.ExecContext(ctx, + r.Timestamp, r.URL, r.Input, r.Host, r.Port, r.Scheme, r.Path, r.Method, r.FinalURL, + r.StatusCode, r.ContentLength, r.ContentType, r.Title, r.WebServer, r.ResponseTime, + r.Location, r.ResponseBody, r.BodyPreview, r.RawHeaders, r.Request, + r.HostIP, aJSON, aaaaJSON, cnameJSON, resolversJSON, fqdnJSON, domainsJSON, r.SNI, + techJSON, hashJSON, r.FavIconMMH3, r.FavIconMD5, r.FaviconPath, r.FaviconURL, r.JarmHash, + r.CDN, r.CDNName, r.CDNType, asnJSON, tlsJSON, cspJSON, + r.Failed, r.Error, r.WebSocket, r.HTTP2, r.Pipeline, r.VHost, + r.Words, r.Lines, headerJSON, extractsJSON, extractRegexJSON, + chainJSON, chainStatusJSON, + r.HeadlessBody, r.ScreenshotBytes, r.ScreenshotPath, r.ScreenshotPathRel, r.StoredResponsePath, + kbJSON, linkReqJSON, traceJSON, + ) + if err != nil { + return fmt.Errorf("failed to insert result: %w", err) + } + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + return nil +} + +func (m *mysqlDatabase) Type() DatabaseType { + return MySQL +} diff --git a/internal/db/postgres.go b/internal/db/postgres.go new file mode 100644 index 00000000..80ecf7dc --- /dev/null +++ b/internal/db/postgres.go @@ -0,0 +1,256 @@ +package db + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + + "github.com/lib/pq" + "github.com/projectdiscovery/httpx/runner" +) + +func init() { + Register(PostgreSQL, newPostgresDatabase) +} + +type postgresDatabase struct { + cfg *Config + db *sql.DB +} + +func newPostgresDatabase(cfg *Config) (Database, error) { + return &postgresDatabase{cfg: cfg}, nil +} + +func (p *postgresDatabase) Connect(ctx context.Context) error { + db, err := sql.Open("postgres", p.cfg.ConnectionString) + if err != nil { + return fmt.Errorf("failed to open PostgreSQL connection: %w", err) + } + + if err := db.PingContext(ctx); err != nil { + return fmt.Errorf("failed to ping PostgreSQL: %w", err) + } + + p.db = db + return nil +} + +func (p *postgresDatabase) Close() error { + if p.db != nil { + return p.db.Close() + } + return nil +} + +func (p *postgresDatabase) EnsureSchema(ctx context.Context) error { + schema := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id BIGSERIAL PRIMARY KEY, + + -- Basic info + timestamp TIMESTAMP WITH TIME ZONE, + url TEXT, + input TEXT, + host TEXT, + port TEXT, + scheme TEXT, + path TEXT, + method TEXT, + final_url TEXT, + + -- Response data + status_code INTEGER, + content_length INTEGER, + content_type TEXT, + title TEXT, + webserver TEXT, + response_time TEXT, + location TEXT, + body TEXT, + body_preview TEXT, + raw_header TEXT, + request TEXT, + + -- Network info + host_ip TEXT, + a TEXT[], + aaaa TEXT[], + cname TEXT[], + resolvers TEXT[], + body_fqdn TEXT[], + body_domains TEXT[], + sni TEXT, + + -- Technology detection + tech TEXT[], + + -- Hashes and fingerprints + hash JSONB, + favicon TEXT, + favicon_md5 TEXT, + favicon_path TEXT, + favicon_url TEXT, + jarm_hash TEXT, + + -- CDN info + cdn BOOLEAN, + cdn_name TEXT, + cdn_type TEXT, + + -- ASN info + asn JSONB, + + -- TLS data + tls JSONB, + + -- CSP data + csp JSONB, + + -- Status flags + failed BOOLEAN, + error TEXT, + websocket BOOLEAN, + http2 BOOLEAN, + pipeline BOOLEAN, + vhost BOOLEAN, + + -- Metrics + words INTEGER, + lines INTEGER, + + -- Headers and extracts + header JSONB, + extracts JSONB, + extract_regex TEXT[], + + -- Chain data + chain JSONB, + chain_status_codes INTEGER[], + + -- Headless/Screenshot + headless_body TEXT, + screenshot_bytes BYTEA, + screenshot_path TEXT, + screenshot_path_rel TEXT, + stored_response_path TEXT, + + -- Knowledge base + knowledgebase JSONB, + + -- Link requests + link_request JSONB, + + -- Trace + trace JSONB + ); + + CREATE INDEX IF NOT EXISTS idx_%s_timestamp ON %s(timestamp DESC); + CREATE INDEX IF NOT EXISTS idx_%s_url ON %s(url); + CREATE INDEX IF NOT EXISTS idx_%s_host ON %s(host); + CREATE INDEX IF NOT EXISTS idx_%s_status_code ON %s(status_code); + CREATE INDEX IF NOT EXISTS idx_%s_tech ON %s USING GIN(tech); + `, + p.cfg.TableName, + p.cfg.TableName, p.cfg.TableName, + p.cfg.TableName, p.cfg.TableName, + p.cfg.TableName, p.cfg.TableName, + p.cfg.TableName, p.cfg.TableName, + p.cfg.TableName, p.cfg.TableName, + ) + + _, err := p.db.ExecContext(ctx, schema) + if err != nil { + return fmt.Errorf("failed to create schema: %w", err) + } + + return nil +} + +func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Result) error { + if len(results) == 0 { + return nil + } + + tx, err := p.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer func() { + _ = tx.Rollback() + }() + + query := fmt.Sprintf(` + INSERT INTO %s ( + timestamp, url, input, host, port, scheme, path, method, final_url, + status_code, content_length, content_type, title, webserver, response_time, + location, body, body_preview, raw_header, request, + host_ip, a, aaaa, cname, resolvers, body_fqdn, body_domains, sni, + tech, hash, favicon, favicon_md5, favicon_path, favicon_url, jarm_hash, + cdn, cdn_name, cdn_type, asn, tls, csp, + failed, error, websocket, http2, pipeline, vhost, + words, lines, header, extracts, extract_regex, + chain, chain_status_codes, + headless_body, screenshot_bytes, screenshot_path, screenshot_path_rel, stored_response_path, + knowledgebase, link_request, trace + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, + $10, $11, $12, $13, $14, $15, + $16, $17, $18, $19, $20, + $21, $22, $23, $24, $25, $26, $27, $28, + $29, $30, $31, $32, $33, $34, $35, + $36, $37, $38, $39, $40, $41, + $42, $43, $44, $45, $46, $47, + $48, $49, $50, $51, $52, + $53, $54, + $55, $56, $57, $58, $59, + $60, $61, $62 + )`, p.cfg.TableName) + + stmt, err := tx.PrepareContext(ctx, query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, r := range results { + hashJSON, _ := json.Marshal(r.Hashes) + asnJSON, _ := json.Marshal(r.ASN) + tlsJSON, _ := json.Marshal(r.TLSData) + cspJSON, _ := json.Marshal(r.CSPData) + headerJSON, _ := json.Marshal(r.ResponseHeaders) + extractsJSON, _ := json.Marshal(r.Extracts) + chainJSON, _ := json.Marshal(r.Chain) + kbJSON, _ := json.Marshal(r.KnowledgeBase) + linkReqJSON, _ := json.Marshal(r.LinkRequest) + traceJSON, _ := json.Marshal(r.Trace) + + _, err = stmt.ExecContext(ctx, + r.Timestamp, r.URL, r.Input, r.Host, r.Port, r.Scheme, r.Path, r.Method, r.FinalURL, + r.StatusCode, r.ContentLength, r.ContentType, r.Title, r.WebServer, r.ResponseTime, + r.Location, r.ResponseBody, r.BodyPreview, r.RawHeaders, r.Request, + r.HostIP, pq.Array(r.A), pq.Array(r.AAAA), pq.Array(r.CNAMEs), pq.Array(r.Resolvers), pq.Array(r.Fqdns), pq.Array(r.Domains), r.SNI, + pq.Array(r.Technologies), hashJSON, r.FavIconMMH3, r.FavIconMD5, r.FaviconPath, r.FaviconURL, r.JarmHash, + r.CDN, r.CDNName, r.CDNType, asnJSON, tlsJSON, cspJSON, + r.Failed, r.Error, r.WebSocket, r.HTTP2, r.Pipeline, r.VHost, + r.Words, r.Lines, headerJSON, extractsJSON, pq.Array(r.ExtractRegex), + chainJSON, pq.Array(r.ChainStatusCodes), + r.HeadlessBody, r.ScreenshotBytes, r.ScreenshotPath, r.ScreenshotPathRel, r.StoredResponsePath, + kbJSON, linkReqJSON, traceJSON, + ) + if err != nil { + return fmt.Errorf("failed to insert result: %w", err) + } + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + return nil +} + +func (p *postgresDatabase) Type() DatabaseType { + return PostgreSQL +} diff --git a/internal/db/writer.go b/internal/db/writer.go new file mode 100644 index 00000000..ebc894a2 --- /dev/null +++ b/internal/db/writer.go @@ -0,0 +1,148 @@ +package db + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/httpx/runner" +) + +type Writer struct { + db Database + cfg *Config + data chan runner.Result + done chan struct{} + counter atomic.Int64 + closed atomic.Bool + wg sync.WaitGroup + ctx context.Context + cancel context.CancelFunc + omitRaw bool +} + +func NewWriter(ctx context.Context, cfg *Config) (*Writer, error) { + db, err := NewDatabase(cfg) + if err != nil { + return nil, err + } + + if err := db.Connect(ctx); err != nil { + return nil, err + } + + if err := db.EnsureSchema(ctx); err != nil { + _ = db.Close() + return nil, err + } + + writerCtx, cancel := context.WithCancel(ctx) + + w := &Writer{ + db: db, + cfg: cfg, + data: make(chan runner.Result, cfg.BatchSize), + done: make(chan struct{}), + ctx: writerCtx, + cancel: cancel, + omitRaw: cfg.OmitRaw, + } + + w.wg.Add(1) + go w.run() + + gologger.Info().Msgf("Database output enabled: %s (%s/%s)", cfg.Type, cfg.DatabaseName, cfg.TableName) + + return w, nil +} + +func (w *Writer) GetWriterCallback() runner.OnResultCallback { + return func(r runner.Result) { + if r.Err != nil { + return + } + + if w.omitRaw { + r.Raw = "" + r.Request = "" + r.ResponseBody = "" + } + + select { + case w.data <- r: + case <-w.ctx.Done(): + } + } +} + +func (w *Writer) run() { + defer w.wg.Done() + defer close(w.done) + + batch := make([]runner.Result, 0, w.cfg.BatchSize) + ticker := time.NewTicker(w.cfg.FlushInterval) + defer ticker.Stop() + + flush := func() { + if len(batch) == 0 { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := w.db.InsertBatch(ctx, batch); err != nil { + gologger.Error().Msgf("Failed to insert batch to database: %v", err) + } else { + w.counter.Add(int64(len(batch))) + gologger.Verbose().Msgf("Inserted %d results to database (total: %d)", len(batch), w.counter.Load()) + } + + batch = batch[:0] + } + + for { + select { + case <-w.ctx.Done(): + flush() + return + + case <-ticker.C: + flush() + + case result, ok := <-w.data: + if !ok { + flush() + return + } + + batch = append(batch, result) + + if len(batch) >= w.cfg.BatchSize { + flush() + } + } + } +} + +func (w *Writer) Close() { + if !w.closed.CompareAndSwap(false, true) { + return + } + + w.cancel() + close(w.data) + <-w.done + + if err := w.db.Close(); err != nil { + gologger.Error().Msgf("Error closing database connection: %v", err) + } + + gologger.Info().Msgf("Database writer closed. Total results stored: %d", w.counter.Load()) +} + +func (w *Writer) Stats() int64 { + return w.counter.Load() +} diff --git a/runner/options.go b/runner/options.go index 46e9bc80..1862164b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -357,6 +357,15 @@ type Options struct { Trace bool + ResultDatabase bool + ResultDatabaseConfig string + ResultDatabaseType string + ResultDatabaseConnStr string + ResultDatabaseName string + ResultDatabaseTable string + ResultDatabaseBatchSize int + ResultDatabaseOmitRaw bool + // Optional pre-created objects to reduce allocations Wappalyzer *wappalyzer.Wappalyze Networkpolicy *networkpolicy.NetworkPolicy @@ -494,6 +503,14 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.StoreVisionReconClusters, "store-vision-recon-cluster", "svrc", false, "include visual recon clusters (-ss and -sr only)"), flagSet.StringVarP(&options.Protocol, "protocol", "pr", "", "protocol to use (unknown, http11, http2 [experimental], http3 [experimental])"), flagSet.StringVarP(&options.OutputFilterErrorPagePath, "filter-error-page-path", "fepp", "filtered_error_page.json", "path to store filtered error pages"), + flagSet.BoolVarP(&options.ResultDatabase, "result-db", "rdb", false, "store results in database"), + flagSet.StringVarP(&options.ResultDatabaseConfig, "result-db-config", "rdbc", "", "path to database config file"), + flagSet.StringVarP(&options.ResultDatabaseType, "result-db-type", "rdbt", "", "database type (mongodb, postgres, mysql)"), + flagSet.StringVarP(&options.ResultDatabaseConnStr, "result-db-conn", "rdbcs", "", "database connection string (env: HTTPX_DB_CONNECTION_STRING)"), + flagSet.StringVarP(&options.ResultDatabaseName, "result-db-name", "rdbn", "httpx", "database name"), + flagSet.StringVarP(&options.ResultDatabaseTable, "result-db-table", "rdbtb", "results", "table/collection name"), + flagSet.IntVarP(&options.ResultDatabaseBatchSize, "result-db-batch-size", "rdbbs", 100, "batch size for database inserts"), + flagSet.BoolVarP(&options.ResultDatabaseOmitRaw, "result-db-omit-raw", "rdbor", false, "omit raw request/response data from database"), ) flagSet.CreateGroup("configs", "Configurations", From f2690f50afbf13b8b2a577606ad43ca1e1e38869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 6 Jan 2026 13:31:04 +0300 Subject: [PATCH 10/39] fix: address CodeRabbitAI review comments for database output - Fix SQL injection in postgres.go using pq.QuoteIdentifier for table/index names - Fix SQL injection in mysql.go using custom quoteIdentifier function - Fix race condition in writer.go by checking closed state before channel send - Add missing idx_url index in mysql.go for parity with PostgreSQL schema - Include RawHeaders in omitRaw check for consistency --- internal/db/mysql.go | 15 ++++++++++++--- internal/db/postgres.go | 32 ++++++++++++++++++++------------ internal/db/writer.go | 5 +++++ 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/internal/db/mysql.go b/internal/db/mysql.go index 50434985..6ebc0011 100644 --- a/internal/db/mysql.go +++ b/internal/db/mysql.go @@ -5,11 +5,16 @@ import ( "database/sql" "encoding/json" "fmt" + "strings" _ "github.com/go-sql-driver/mysql" "github.com/projectdiscovery/httpx/runner" ) +func quoteIdentifier(name string) string { + return "`" + strings.ReplaceAll(name, "`", "``") + "`" +} + func init() { Register(MySQL, newMySQLDatabase) } @@ -45,6 +50,7 @@ func (m *mysqlDatabase) Close() error { } func (m *mysqlDatabase) EnsureSchema(ctx context.Context) error { + tableName := quoteIdentifier(m.cfg.TableName) schema := fmt.Sprintf(` CREATE TABLE IF NOT EXISTS %s ( id BIGINT AUTO_INCREMENT PRIMARY KEY, @@ -118,7 +124,7 @@ func (m *mysqlDatabase) EnsureSchema(ctx context.Context) error { -- Metrics words INT, - ` + "`lines`" + ` INT, + `+"`lines`"+` INT, -- Headers and extracts header JSON, @@ -146,10 +152,11 @@ func (m *mysqlDatabase) EnsureSchema(ctx context.Context) error { trace JSON, INDEX idx_timestamp (timestamp), + INDEX idx_url (url(255)), INDEX idx_host (host), INDEX idx_status_code (status_code) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - `, m.cfg.TableName) + `, tableName) _, err := m.db.ExecContext(ctx, schema) if err != nil { @@ -172,6 +179,8 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result _ = tx.Rollback() }() + // Use quoteIdentifier to safely quote table name to prevent SQL injection + tableName := quoteIdentifier(m.cfg.TableName) query := fmt.Sprintf(` INSERT INTO %s ( timestamp, url, input, host, port, scheme, path, method, final_url, @@ -197,7 +206,7 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result ?, ?, ?, ?, ?, ?, ?, ?, ?, ? - )`, m.cfg.TableName) + )`, tableName) stmt, err := tx.PrepareContext(ctx, query) if err != nil { diff --git a/internal/db/postgres.go b/internal/db/postgres.go index 80ecf7dc..b53d5aaf 100644 --- a/internal/db/postgres.go +++ b/internal/db/postgres.go @@ -45,6 +45,13 @@ func (p *postgresDatabase) Close() error { } func (p *postgresDatabase) EnsureSchema(ctx context.Context) error { + tableName := pq.QuoteIdentifier(p.cfg.TableName) + idxTimestamp := pq.QuoteIdentifier("idx_" + p.cfg.TableName + "_timestamp") + idxURL := pq.QuoteIdentifier("idx_" + p.cfg.TableName + "_url") + idxHost := pq.QuoteIdentifier("idx_" + p.cfg.TableName + "_host") + idxStatusCode := pq.QuoteIdentifier("idx_" + p.cfg.TableName + "_status_code") + idxTech := pq.QuoteIdentifier("idx_" + p.cfg.TableName + "_tech") + schema := fmt.Sprintf(` CREATE TABLE IF NOT EXISTS %s ( id BIGSERIAL PRIMARY KEY, @@ -146,18 +153,18 @@ func (p *postgresDatabase) EnsureSchema(ctx context.Context) error { trace JSONB ); - CREATE INDEX IF NOT EXISTS idx_%s_timestamp ON %s(timestamp DESC); - CREATE INDEX IF NOT EXISTS idx_%s_url ON %s(url); - CREATE INDEX IF NOT EXISTS idx_%s_host ON %s(host); - CREATE INDEX IF NOT EXISTS idx_%s_status_code ON %s(status_code); - CREATE INDEX IF NOT EXISTS idx_%s_tech ON %s USING GIN(tech); + CREATE INDEX IF NOT EXISTS %s ON %s(timestamp DESC); + CREATE INDEX IF NOT EXISTS %s ON %s(url); + CREATE INDEX IF NOT EXISTS %s ON %s(host); + CREATE INDEX IF NOT EXISTS %s ON %s(status_code); + CREATE INDEX IF NOT EXISTS %s ON %s USING GIN(tech); `, - p.cfg.TableName, - p.cfg.TableName, p.cfg.TableName, - p.cfg.TableName, p.cfg.TableName, - p.cfg.TableName, p.cfg.TableName, - p.cfg.TableName, p.cfg.TableName, - p.cfg.TableName, p.cfg.TableName, + tableName, + idxTimestamp, tableName, + idxURL, tableName, + idxHost, tableName, + idxStatusCode, tableName, + idxTech, tableName, ) _, err := p.db.ExecContext(ctx, schema) @@ -181,6 +188,7 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res _ = tx.Rollback() }() + tableName := pq.QuoteIdentifier(p.cfg.TableName) query := fmt.Sprintf(` INSERT INTO %s ( timestamp, url, input, host, port, scheme, path, method, final_url, @@ -206,7 +214,7 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res $53, $54, $55, $56, $57, $58, $59, $60, $61, $62 - )`, p.cfg.TableName) + )`, tableName) stmt, err := tx.PrepareContext(ctx, query) if err != nil { diff --git a/internal/db/writer.go b/internal/db/writer.go index ebc894a2..62ba3a71 100644 --- a/internal/db/writer.go +++ b/internal/db/writer.go @@ -60,6 +60,10 @@ func NewWriter(ctx context.Context, cfg *Config) (*Writer, error) { func (w *Writer) GetWriterCallback() runner.OnResultCallback { return func(r runner.Result) { + if w.closed.Load() { + return + } + if r.Err != nil { return } @@ -68,6 +72,7 @@ func (w *Writer) GetWriterCallback() runner.OnResultCallback { r.Raw = "" r.Request = "" r.ResponseBody = "" + r.RawHeaders = "" } select { From c1c1fd0a76653985adc1297e36f9bc1d4a9784a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 12 Jan 2026 12:29:38 +0300 Subject: [PATCH 11/39] refactor: use batcher utility and StringVarEnv --- internal/db/config.go | 4 -- internal/db/writer.go | 109 +++++++++++++----------------------------- runner/options.go | 2 +- 3 files changed, 34 insertions(+), 81 deletions(-) diff --git a/internal/db/config.go b/internal/db/config.go index c72dcac3..c057b80b 100644 --- a/internal/db/config.go +++ b/internal/db/config.go @@ -111,10 +111,6 @@ func (o *Options) ToConfig() (*Config, error) { OmitRaw: o.OmitRaw, } - if cfg.ConnectionString == "" { - cfg.ConnectionString = os.Getenv(EnvConnectionString) - } - cfg.ApplyDefaults() if err := cfg.Validate(); err != nil { diff --git a/internal/db/writer.go b/internal/db/writer.go index 62ba3a71..ec0042df 100644 --- a/internal/db/writer.go +++ b/internal/db/writer.go @@ -2,25 +2,21 @@ package db import ( "context" - "sync" "sync/atomic" "time" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/httpx/runner" + "github.com/projectdiscovery/utils/batcher" ) type Writer struct { - db Database - cfg *Config - data chan runner.Result - done chan struct{} - counter atomic.Int64 - closed atomic.Bool - wg sync.WaitGroup - ctx context.Context - cancel context.CancelFunc - omitRaw bool + db Database + cfg *Config + batcher *batcher.Batcher[runner.Result] + counter atomic.Int64 + closed atomic.Bool + omitRaw bool } func NewWriter(ctx context.Context, cfg *Config) (*Writer, error) { @@ -38,26 +34,41 @@ func NewWriter(ctx context.Context, cfg *Config) (*Writer, error) { return nil, err } - writerCtx, cancel := context.WithCancel(ctx) - w := &Writer{ db: db, cfg: cfg, - data: make(chan runner.Result, cfg.BatchSize), - done: make(chan struct{}), - ctx: writerCtx, - cancel: cancel, omitRaw: cfg.OmitRaw, } - w.wg.Add(1) - go w.run() + w.batcher = batcher.New( + batcher.WithMaxCapacity[runner.Result](cfg.BatchSize), + batcher.WithFlushInterval[runner.Result](cfg.FlushInterval), + batcher.WithFlushCallback(w.flush), + ) + + w.batcher.Run() gologger.Info().Msgf("Database output enabled: %s (%s/%s)", cfg.Type, cfg.DatabaseName, cfg.TableName) return w, nil } +func (w *Writer) flush(batch []runner.Result) { + if len(batch) == 0 { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := w.db.InsertBatch(ctx, batch); err != nil { + gologger.Error().Msgf("Failed to insert batch to database: %v", err) + } else { + w.counter.Add(int64(len(batch))) + gologger.Verbose().Msgf("Inserted %d results to database (total: %d)", len(batch), w.counter.Load()) + } +} + func (w *Writer) GetWriterCallback() runner.OnResultCallback { return func(r runner.Result) { if w.closed.Load() { @@ -75,60 +86,7 @@ func (w *Writer) GetWriterCallback() runner.OnResultCallback { r.RawHeaders = "" } - select { - case w.data <- r: - case <-w.ctx.Done(): - } - } -} - -func (w *Writer) run() { - defer w.wg.Done() - defer close(w.done) - - batch := make([]runner.Result, 0, w.cfg.BatchSize) - ticker := time.NewTicker(w.cfg.FlushInterval) - defer ticker.Stop() - - flush := func() { - if len(batch) == 0 { - return - } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - if err := w.db.InsertBatch(ctx, batch); err != nil { - gologger.Error().Msgf("Failed to insert batch to database: %v", err) - } else { - w.counter.Add(int64(len(batch))) - gologger.Verbose().Msgf("Inserted %d results to database (total: %d)", len(batch), w.counter.Load()) - } - - batch = batch[:0] - } - - for { - select { - case <-w.ctx.Done(): - flush() - return - - case <-ticker.C: - flush() - - case result, ok := <-w.data: - if !ok { - flush() - return - } - - batch = append(batch, result) - - if len(batch) >= w.cfg.BatchSize { - flush() - } - } + w.batcher.Append(r) } } @@ -137,9 +95,8 @@ func (w *Writer) Close() { return } - w.cancel() - close(w.data) - <-w.done + w.batcher.Stop() + w.batcher.WaitDone() if err := w.db.Close(); err != nil { gologger.Error().Msgf("Error closing database connection: %v", err) diff --git a/runner/options.go b/runner/options.go index 1862164b..9af8e541 100644 --- a/runner/options.go +++ b/runner/options.go @@ -506,7 +506,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.ResultDatabase, "result-db", "rdb", false, "store results in database"), flagSet.StringVarP(&options.ResultDatabaseConfig, "result-db-config", "rdbc", "", "path to database config file"), flagSet.StringVarP(&options.ResultDatabaseType, "result-db-type", "rdbt", "", "database type (mongodb, postgres, mysql)"), - flagSet.StringVarP(&options.ResultDatabaseConnStr, "result-db-conn", "rdbcs", "", "database connection string (env: HTTPX_DB_CONNECTION_STRING)"), + flagSet.StringVarEnv(&options.ResultDatabaseConnStr, "result-db-conn", "rdbcs", "", "HTTPX_DB_CONNECTION_STRING", "database connection string"), flagSet.StringVarP(&options.ResultDatabaseName, "result-db-name", "rdbn", "httpx", "database name"), flagSet.StringVarP(&options.ResultDatabaseTable, "result-db-table", "rdbtb", "results", "table/collection name"), flagSet.IntVarP(&options.ResultDatabaseBatchSize, "result-db-batch-size", "rdbbs", 100, "batch size for database inserts"), From 12e36246c04ecba17818589d4df356393d4419c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jan 2026 19:50:51 +0700 Subject: [PATCH 12/39] chore(deps): bump the modules group with 5 updates (#2383) Bumps the modules group with 5 updates: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.2.19` | `1.2.20` | | [github.com/projectdiscovery/networkpolicy](https://github.com/projectdiscovery/networkpolicy) | `0.1.33` | `0.1.34` | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.3.4` | `1.3.5` | | [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) | `0.0.106` | `0.0.107` | | [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) | `0.2.64` | `0.2.65` | Updates `github.com/projectdiscovery/cdncheck` from 1.2.19 to 1.2.20 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.2.19...v1.2.20) Updates `github.com/projectdiscovery/networkpolicy` from 0.1.33 to 0.1.34 - [Release notes](https://github.com/projectdiscovery/networkpolicy/releases) - [Commits](https://github.com/projectdiscovery/networkpolicy/compare/v0.1.33...v0.1.34) Updates `github.com/projectdiscovery/retryablehttp-go` from 1.3.4 to 1.3.5 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.3.4...v1.3.5) Updates `github.com/projectdiscovery/useragent` from 0.0.106 to 0.0.107 - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.106...v0.0.107) Updates `github.com/projectdiscovery/wappalyzergo` from 0.2.64 to 0.2.65 - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.2.64...v0.2.65) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.2.20 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/networkpolicy dependency-version: 0.1.34 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-version: 1.3.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/useragent dependency-version: 0.0.107 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-version: 0.2.65 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index da7280df..96e41eaf 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/miekg/dns v1.1.68 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 - github.com/projectdiscovery/cdncheck v1.2.19 + github.com/projectdiscovery/cdncheck v1.2.20 github.com/projectdiscovery/clistats v0.1.1 github.com/projectdiscovery/dsl v0.8.12 github.com/projectdiscovery/fastdialer v0.5.3 @@ -29,14 +29,14 @@ require ( github.com/projectdiscovery/gologger v1.1.67 github.com/projectdiscovery/hmap v0.0.99 github.com/projectdiscovery/mapcidr v1.1.97 - github.com/projectdiscovery/networkpolicy v0.1.33 + github.com/projectdiscovery/networkpolicy v0.1.34 github.com/projectdiscovery/ratelimit v0.0.83 github.com/projectdiscovery/rawhttp v0.1.90 - github.com/projectdiscovery/retryablehttp-go v1.3.4 + github.com/projectdiscovery/retryablehttp-go v1.3.5 github.com/projectdiscovery/tlsx v1.2.2 - github.com/projectdiscovery/useragent v0.0.106 + github.com/projectdiscovery/useragent v0.0.107 github.com/projectdiscovery/utils v0.9.0 - github.com/projectdiscovery/wappalyzergo v0.2.64 + github.com/projectdiscovery/wappalyzergo v0.2.65 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index f5833c7d..0b54d815 100644 --- a/go.sum +++ b/go.sum @@ -324,8 +324,8 @@ github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.2.19 h1:UU0ii1z8WZFsFODD89MYJ4i9h1EXhBJSZt/rzIH94JY= -github.com/projectdiscovery/cdncheck v1.2.19/go.mod h1:RRA4KOiUTBhkk2tImdoxqPpD0fB5C9rBP7W0r+ji9Cg= +github.com/projectdiscovery/cdncheck v1.2.20 h1:sMzoCi5TR7qQsH4LW0NF219PmX/lYjWUeoB2Iiddwcs= +github.com/projectdiscovery/cdncheck v1.2.20/go.mod h1:gpeX5OrzaC4DmeUGDcKrC7cPUXQvRGTY/Ui0XrVfdzU= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= github.com/projectdiscovery/dsl v0.8.12 h1:gQL8k5zPok+5JGc7poiXzHCElNY/WnaTKoRB2wI3CYA= @@ -350,26 +350,26 @@ github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0 github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8QuWs2puwy+2w= github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA= -github.com/projectdiscovery/networkpolicy v0.1.33 h1:bVgp+XpLEsQ7ZEJt3UaUqIwhI01MMdt7F2dfIKFQg/w= -github.com/projectdiscovery/networkpolicy v0.1.33/go.mod h1:YAPddAXUc/lhoU85AFdvgOQKx8Qh8r0vzSjexRWk6Yk= +github.com/projectdiscovery/networkpolicy v0.1.34 h1:TRwNbgMwdx3NC190TKSLwtTvr0JAIZAlnWkOhW0yBME= +github.com/projectdiscovery/networkpolicy v0.1.34/go.mod h1:GJ20E7fJoA2vk8ZBSa1Cvc5WyP8RxglF5bZmYgK8jag= github.com/projectdiscovery/ratelimit v0.0.83 h1:hfb36QvznBrjA4FNfpFE8AYRVBYrfJh8qHVROLQgl54= github.com/projectdiscovery/ratelimit v0.0.83/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw= github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y= github.com/projectdiscovery/retryabledns v1.0.112 h1:4iCiuo6jMnw/pdOZRzBQrbUOUu5tOeuvGupxVV8RDLw= github.com/projectdiscovery/retryabledns v1.0.112/go.mod h1:xsJTKbo+KGqd7+88z1naEUFJybLH2yjB/zUyOweA7k0= -github.com/projectdiscovery/retryablehttp-go v1.3.4 h1:QgGah0Py9MvvjrzGxGthgzhh5jzG18uRfqkJNUXKDIo= -github.com/projectdiscovery/retryablehttp-go v1.3.4/go.mod h1:4disixzHEhNd2pEO2kpg0kqyy9Tx1WMZtgd7hI/XiuM= +github.com/projectdiscovery/retryablehttp-go v1.3.5 h1:6UXSJOEeeSE/IpI4xPrKRhSLkk3itNajfbgH91WtPPc= +github.com/projectdiscovery/retryablehttp-go v1.3.5/go.mod h1:2ma5Itx44tgfZCtHqnI7xbWEmsLXt1qXh+oOaJfmA+g= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiBLvK+SE= github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4= -github.com/projectdiscovery/useragent v0.0.106 h1:9fS08MRUUJvfBskTxcXY9TA4X1TwpH6iJ3P3YNaXNlo= -github.com/projectdiscovery/useragent v0.0.106/go.mod h1:9oVMjgd7CchIsyeweyigIPtW83gpiGf2NtR6UM5XK+o= +github.com/projectdiscovery/useragent v0.0.107 h1:45gSBda052fv2Gtxtnpx7cu2rWtUpZEQRGAoYGP6F5M= +github.com/projectdiscovery/useragent v0.0.107/go.mod h1:yv5ZZLDT/kq6P+NvBcCPq6sjEVQtZGgO+OvvHzZ+WtY= github.com/projectdiscovery/utils v0.9.0 h1:eu9vdbP0VYXI9nGSLfnOpUqBeW9/B/iSli7U8gPKZw8= github.com/projectdiscovery/utils v0.9.0/go.mod h1:zcVu1QTlMi5763qCol/L3ROnbd/UPSBP8fI5PmcnF6s= -github.com/projectdiscovery/wappalyzergo v0.2.64 h1:Y55sb5qUdFvMtR81m1hr54PdGh/hZ4XtuGPdCFAirEk= -github.com/projectdiscovery/wappalyzergo v0.2.64/go.mod h1:8FtSVcmPRZU0g1euBpdSYEBHIvB7Zz9MOb754ZqZmfU= +github.com/projectdiscovery/wappalyzergo v0.2.65 h1:5hWGkuortLiq0whmVIfxbbE9pDl7Zd5e1rVRIEimOyk= +github.com/projectdiscovery/wappalyzergo v0.2.65/go.mod h1:Oc+U2RPJObmpi6LW5lTMEDiKagcKZNkEfZfwrVMURa0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0= github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ= From 9c43bacab95303bac23beba12e10dd75839cdc1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Feb 2026 11:32:16 +0000 Subject: [PATCH 13/39] chore(deps): bump the modules group with 7 updates Bumps the modules group with 7 updates: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.2.20` | `1.2.21` | | [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) | `0.8.12` | `0.8.13` | | [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) | `0.5.3` | `0.5.4` | | [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) | `1.1.67` | `1.1.68` | | [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) | `0.0.99` | `0.0.100` | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.3.5` | `1.3.6` | | [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) | `0.2.65` | `0.2.66` | Updates `github.com/projectdiscovery/cdncheck` from 1.2.20 to 1.2.21 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.2.20...v1.2.21) Updates `github.com/projectdiscovery/dsl` from 0.8.12 to 0.8.13 - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.8.12...v0.8.13) Updates `github.com/projectdiscovery/fastdialer` from 0.5.3 to 0.5.4 - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.5.3...v0.5.4) Updates `github.com/projectdiscovery/gologger` from 1.1.67 to 1.1.68 - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.67...v1.1.68) Updates `github.com/projectdiscovery/hmap` from 0.0.99 to 0.0.100 - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.99...v0.0.100) Updates `github.com/projectdiscovery/retryablehttp-go` from 1.3.5 to 1.3.6 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.3.5...v1.3.6) Updates `github.com/projectdiscovery/wappalyzergo` from 0.2.65 to 0.2.66 - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.2.65...v0.2.66) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.2.21 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/dsl dependency-version: 0.8.13 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/fastdialer dependency-version: 0.5.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/gologger dependency-version: 1.1.68 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/hmap dependency-version: 0.0.100 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-version: 1.3.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-version: 0.2.66 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 96e41eaf..f826b775 100644 --- a/go.mod +++ b/go.mod @@ -19,24 +19,24 @@ require ( github.com/miekg/dns v1.1.68 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 - github.com/projectdiscovery/cdncheck v1.2.20 + github.com/projectdiscovery/cdncheck v1.2.21 github.com/projectdiscovery/clistats v0.1.1 - github.com/projectdiscovery/dsl v0.8.12 - github.com/projectdiscovery/fastdialer v0.5.3 + github.com/projectdiscovery/dsl v0.8.13 + github.com/projectdiscovery/fastdialer v0.5.4 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/gologger v1.1.67 - github.com/projectdiscovery/hmap v0.0.99 + github.com/projectdiscovery/gologger v1.1.68 + github.com/projectdiscovery/hmap v0.0.100 github.com/projectdiscovery/mapcidr v1.1.97 github.com/projectdiscovery/networkpolicy v0.1.34 github.com/projectdiscovery/ratelimit v0.0.83 github.com/projectdiscovery/rawhttp v0.1.90 - github.com/projectdiscovery/retryablehttp-go v1.3.5 + github.com/projectdiscovery/retryablehttp-go v1.3.6 github.com/projectdiscovery/tlsx v1.2.2 github.com/projectdiscovery/useragent v0.0.107 github.com/projectdiscovery/utils v0.9.0 - github.com/projectdiscovery/wappalyzergo v0.2.65 + github.com/projectdiscovery/wappalyzergo v0.2.66 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.11.1 @@ -135,7 +135,7 @@ require ( github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect - github.com/projectdiscovery/retryabledns v1.0.112 // indirect + github.com/projectdiscovery/retryabledns v1.0.113 // indirect github.com/refraction-networking/utls v1.7.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect diff --git a/go.sum b/go.sum index 0b54d815..25b6bb6f 100644 --- a/go.sum +++ b/go.sum @@ -324,14 +324,14 @@ github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.2.20 h1:sMzoCi5TR7qQsH4LW0NF219PmX/lYjWUeoB2Iiddwcs= -github.com/projectdiscovery/cdncheck v1.2.20/go.mod h1:gpeX5OrzaC4DmeUGDcKrC7cPUXQvRGTY/Ui0XrVfdzU= +github.com/projectdiscovery/cdncheck v1.2.21 h1:+y77BGCZoduX5bja2SGn4AdBXFwfOycaLnWWUIiZCBM= +github.com/projectdiscovery/cdncheck v1.2.21/go.mod h1:gpeX5OrzaC4DmeUGDcKrC7cPUXQvRGTY/Ui0XrVfdzU= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= -github.com/projectdiscovery/dsl v0.8.12 h1:gQL8k5zPok+5JGc7poiXzHCElNY/WnaTKoRB2wI3CYA= -github.com/projectdiscovery/dsl v0.8.12/go.mod h1:pdMfUTNHMxlt6M94CSrCpZ1QObTP44rLqWifMMWW+IA= -github.com/projectdiscovery/fastdialer v0.5.3 h1:Io57Q37ouFzrPK53ZdzK6jsELgqjIMCWcoDs+lRDGMA= -github.com/projectdiscovery/fastdialer v0.5.3/go.mod h1:euoxS1E93LDnl0OnNN0UALedAFF+EehBxyU3z+79l0g= +github.com/projectdiscovery/dsl v0.8.13 h1:HjjHta7c02saH2tUGs8CN5vDeE2MyWvCV32koT8ZCWs= +github.com/projectdiscovery/dsl v0.8.13/go.mod h1:hgFaXhz/JuO+HqIXqBqYIR3ntPnqTo38MJJAzb5tIbg= +github.com/projectdiscovery/fastdialer v0.5.4 h1:+0oesDDqZcIPE5bNDmm/Xm9Xm3yjnhl4xwP+h5D1TE4= +github.com/projectdiscovery/fastdialer v0.5.4/go.mod h1:KCzt6WnSAj9umiUBRCaC0EJSEyeshxDoowfwjxodmQw= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= @@ -340,12 +340,12 @@ github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQ github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= -github.com/projectdiscovery/gologger v1.1.67 h1:GZU3AjYiJvcwJT5TlfIv+152/TVmaz62Zyn3/wWXlig= -github.com/projectdiscovery/gologger v1.1.67/go.mod h1:35oeQP6wvj58S+o+Km6boED/t786FXQkI0exhFHJbNE= +github.com/projectdiscovery/gologger v1.1.68 h1:KfdIO/3X7BtHssWZuqhxPZ+A946epCCx2cz+3NnRAnU= +github.com/projectdiscovery/gologger v1.1.68/go.mod h1:Xae0t4SeqJVa0RQGK9iECx/+HfXhvq70nqOQp2BuW+o= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.99 h1:XPfLnD3CUrMqVCIdpK9ozD7Xmp3simx3T+2j4WWhHnU= -github.com/projectdiscovery/hmap v0.0.99/go.mod h1:koyUJi83K5G3w35ZLFXOYZIyYJsO+6hQrgDDN1RBrVE= +github.com/projectdiscovery/hmap v0.0.100 h1:DBZ3Req9lWf4P1YC9PRa4eiMvLY0Uxud43NRBcocPfs= +github.com/projectdiscovery/hmap v0.0.100/go.mod h1:2O06pR8pHOP9wSmxAoxuM45U7E+UqOqOdlSIeddM0bA= github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q= github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8QuWs2puwy+2w= @@ -356,10 +356,10 @@ github.com/projectdiscovery/ratelimit v0.0.83 h1:hfb36QvznBrjA4FNfpFE8AYRVBYrfJh github.com/projectdiscovery/ratelimit v0.0.83/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw= github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y= -github.com/projectdiscovery/retryabledns v1.0.112 h1:4iCiuo6jMnw/pdOZRzBQrbUOUu5tOeuvGupxVV8RDLw= -github.com/projectdiscovery/retryabledns v1.0.112/go.mod h1:xsJTKbo+KGqd7+88z1naEUFJybLH2yjB/zUyOweA7k0= -github.com/projectdiscovery/retryablehttp-go v1.3.5 h1:6UXSJOEeeSE/IpI4xPrKRhSLkk3itNajfbgH91WtPPc= -github.com/projectdiscovery/retryablehttp-go v1.3.5/go.mod h1:2ma5Itx44tgfZCtHqnI7xbWEmsLXt1qXh+oOaJfmA+g= +github.com/projectdiscovery/retryabledns v1.0.113 h1:s+DAzdJ8XhLxRgt5636H0HG9OqHsGRjX9wTrLSTMqlQ= +github.com/projectdiscovery/retryabledns v1.0.113/go.mod h1:+DyanDr8naxQ2dRO9c4Ezo3NHHXhz8L0tTSRYWhiwyA= +github.com/projectdiscovery/retryablehttp-go v1.3.6 h1:dLb0/YVX+oX70gpWxN5GXT8pCKpn8fdXfwOq2TsXxNY= +github.com/projectdiscovery/retryablehttp-go v1.3.6/go.mod h1:tKVxmL4ixWy1MjYk5GJvFL0Cp10fnQgSp2F6bSBEypI= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiBLvK+SE= @@ -368,8 +368,8 @@ github.com/projectdiscovery/useragent v0.0.107 h1:45gSBda052fv2Gtxtnpx7cu2rWtUpZ github.com/projectdiscovery/useragent v0.0.107/go.mod h1:yv5ZZLDT/kq6P+NvBcCPq6sjEVQtZGgO+OvvHzZ+WtY= github.com/projectdiscovery/utils v0.9.0 h1:eu9vdbP0VYXI9nGSLfnOpUqBeW9/B/iSli7U8gPKZw8= github.com/projectdiscovery/utils v0.9.0/go.mod h1:zcVu1QTlMi5763qCol/L3ROnbd/UPSBP8fI5PmcnF6s= -github.com/projectdiscovery/wappalyzergo v0.2.65 h1:5hWGkuortLiq0whmVIfxbbE9pDl7Zd5e1rVRIEimOyk= -github.com/projectdiscovery/wappalyzergo v0.2.65/go.mod h1:Oc+U2RPJObmpi6LW5lTMEDiKagcKZNkEfZfwrVMURa0= +github.com/projectdiscovery/wappalyzergo v0.2.66 h1:DEF7wthjvBo6oYKxfKL6vPNaqsKYUmiWODt7Mybcins= +github.com/projectdiscovery/wappalyzergo v0.2.66/go.mod h1:Oc+U2RPJObmpi6LW5lTMEDiKagcKZNkEfZfwrVMURa0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0= github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ= From 374fcd1770778630d2bd319d2f43e0952362b31b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 3 Feb 2026 18:34:57 +0300 Subject: [PATCH 14/39] chore: add pull request template --- .github/PULL_REQUEST_TEMPLATE.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..174478de --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## Proposed changes + + + +### Proof + + + +## Checklist + + + +- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/httpx/tree/dev) branch +- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have added necessary documentation (if appropriate) \ No newline at end of file From 8080434e71810e99810f04edd9a2b7b898dd3016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 4 Feb 2026 15:58:16 +0300 Subject: [PATCH 15/39] warn when resolver looks like file path but doesn't exist --- runner/options.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 58183df6..4a7bb069 100644 --- a/runner/options.go +++ b/runner/options.go @@ -743,11 +743,14 @@ func (options *Options) ValidateOptions() error { resolvers = append(resolvers, item) } } - } else { + } else if line != "" { resolvers = append(resolvers, line) } } } else { + if strings.ContainsAny(resolver, `/\`) { + gologger.Warning().Msgf("Resolver argument \"%s\" looks like a file path but the file does not exist", resolver) + } resolvers = append(resolvers, resolver) } } From fa950239f2ba872d361e4b11c92ec606dff226ea Mon Sep 17 00:00:00 2001 From: Adel Assakaf Date: Thu, 5 Feb 2026 03:59:29 +0100 Subject: [PATCH 16/39] fix: prevent data loss on interrupt by implementing graceful shutdown --- cmd/httpx/httpx.go | 29 ++++++++++-------- runner/runner.go | 58 ++++++++++++++++++++++++++++++++---- runner/runner_test.go | 68 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 18 deletions(-) diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 77e54f4c..78037c08 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -73,21 +73,26 @@ func main() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) go func() { - for range c { - gologger.Info().Msgf("CTRL+C pressed: Exiting\n") - httpxRunner.Close() - if options.ShouldSaveResume() { - gologger.Info().Msgf("Creating resume file: %s\n", runner.DefaultResumeFile) - err := httpxRunner.SaveResumeConfig() - if err != nil { - gologger.Error().Msgf("Couldn't create resume file: %s\n", err) - } - } - os.Exit(1) - } + // First Ctrl+C: stop dispatching, let in-flight requests finish + <-c + gologger.Info().Msgf("CTRL+C pressed: Exiting\n") + httpxRunner.Interrupt() + // Second Ctrl+C: force exit + <-c + gologger.Info().Msgf("Forcing exit\n") + os.Exit(1) }() httpxRunner.RunEnumeration() + + if httpxRunner.IsInterrupted() && options.ShouldSaveResume() { + gologger.Info().Msgf("Creating resume file: %s\n", runner.DefaultResumeFile) + err := httpxRunner.SaveResumeConfig() + if err != nil { + gologger.Error().Msgf("Couldn't create resume file: %s\n", err) + } + } + httpxRunner.Close() } diff --git a/runner/runner.go b/runner/runner.go index 14b91625..8647c23f 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -97,12 +97,32 @@ type Runner struct { simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection httpApiEndpoint *Server authProvider authprovider.AuthProvider + interruptCh chan struct{} } func (r *Runner) HTTPX() *httpx.HTTPX { return r.hp } +// Interrupt signals the runner to stop dispatching new items. +func (r *Runner) Interrupt() { + select { + case <-r.interruptCh: + default: + close(r.interruptCh) + } +} + +// IsInterrupted returns true if the runner was interrupted. +func (r *Runner) IsInterrupted() bool { + select { + case <-r.interruptCh: + return true + default: + return false + } +} + // picked based on try-fail but it seems to close to one it's used https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html#c1992 var hammingDistanceThreshold int = 22 @@ -121,7 +141,8 @@ type pHashUrl struct { // New creates a new client for running enumeration process. func New(options *Options) (*Runner, error) { runner := &Runner{ - options: options, + options: options, + interruptCh: make(chan struct{}), } var err error if options.Wappalyzer != nil { @@ -664,6 +685,16 @@ func (r *Runner) streamInput() (chan string, error) { go func() { defer close(out) + // trySend sends item to out, returning false if interrupted + trySend := func(item string) bool { + select { + case <-r.interruptCh: + return false + case out <- item: + return true + } + } + if fileutil.FileExists(r.options.InputFile) { // check if input mode is specified for special format handling if format := r.getInputFormat(); format != nil { @@ -676,9 +707,9 @@ func (r *Runner) streamInput() (chan string, error) { if err := format.Parse(finput, func(item string) bool { item = strings.TrimSpace(item) if r.options.SkipDedupe || r.testAndSet(item) { - out <- item + return trySend(item) } - return true + return !r.IsInterrupted() }); err != nil { gologger.Error().Msgf("Could not parse input file '%s': %s\n", r.options.InputFile, err) return @@ -690,7 +721,9 @@ func (r *Runner) streamInput() (chan string, error) { } for item := range fchan { if r.options.SkipDedupe || r.testAndSet(item) { - out <- item + if !trySend(item) { + return + } } } } @@ -706,7 +739,9 @@ func (r *Runner) streamInput() (chan string, error) { } for item := range fchan { if r.options.SkipDedupe || r.testAndSet(item) { - out <- item + if !trySend(item) { + return + } } } } @@ -718,7 +753,9 @@ func (r *Runner) streamInput() (chan string, error) { } for item := range fchan { if r.options.SkipDedupe || r.testAndSet(item) { - out <- item + if !trySend(item) { + return + } } } } @@ -1402,6 +1439,12 @@ func (r *Runner) RunEnumeration() { wg, _ := syncutil.New(syncutil.WithSize(r.options.Threads)) processItem := func(k string) error { + select { + case <-r.interruptCh: + return nil + default: + } + if r.options.resumeCfg != nil { r.options.resumeCfg.current = k r.options.resumeCfg.currentIndex++ @@ -1447,6 +1490,9 @@ func (r *Runner) RunEnumeration() { if r.options.Stream { for item := range streamChan { + if r.IsInterrupted() { + break + } _ = processItem(item) } } else { diff --git a/runner/runner_test.go b/runner/runner_test.go index 10b8320b..832c4e36 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -15,6 +15,74 @@ import ( "github.com/stretchr/testify/require" ) +func TestRunner_resumeAfterInterrupt(t *testing.T) { + domains := []string{"a.com", "b.com", "c.com", "d.com", "e.com", "f.com", "g.com", "h.com", "i.com", "j.com"} + interruptAfter := 4 + + // --- Full scan (reference): process all domains without interrupt --- + rFull, err := New(&Options{}) + require.Nil(t, err, "could not create httpx runner") + rFull.options.resumeCfg = &ResumeCfg{} + var fullOutput []string + for _, d := range domains { + rFull.options.resumeCfg.current = d + rFull.options.resumeCfg.currentIndex++ + fullOutput = append(fullOutput, d) + } + + // --- Interrupted scan: process items, interrupt after interruptAfter --- + rInt, err := New(&Options{}) + require.Nil(t, err, "could not create httpx runner") + rInt.options.resumeCfg = &ResumeCfg{} + var interruptedOutput []string + for _, d := range domains { + // same check as processItem: bail out if interrupted + select { + case <-rInt.interruptCh: + continue + default: + } + + rInt.options.resumeCfg.current = d + rInt.options.resumeCfg.currentIndex++ + interruptedOutput = append(interruptedOutput, d) + + if len(interruptedOutput) == interruptAfter { + rInt.Interrupt() + } + } + + // simulate SaveResumeConfig: save the index after interrupt + savedIndex := rInt.options.resumeCfg.currentIndex + + // the saved index must equal exactly the number of items that were processed + require.Equal(t, interruptAfter, savedIndex, "resume index should equal number of completed items") + // every domain before the index must be in the interrupted output + require.Equal(t, domains[:interruptAfter], interruptedOutput, "interrupted output should contain exactly the first N domains") + + // --- Resumed scan: load saved index, skip already-processed items --- + rRes, err := New(&Options{}) + require.Nil(t, err, "could not create httpx runner") + rRes.options.resumeCfg = &ResumeCfg{Index: savedIndex} + var resumedOutput []string + for _, d := range domains { + // same resume-skip logic as processItem + rRes.options.resumeCfg.current = d + rRes.options.resumeCfg.currentIndex++ + if rRes.options.resumeCfg.currentIndex <= rRes.options.resumeCfg.Index { + continue + } + resumedOutput = append(resumedOutput, d) + } + + // every domain after the index must be in the resumed output + require.Equal(t, domains[interruptAfter:], resumedOutput, "resumed output should contain exactly the remaining domains") + + // union of interrupted + resumed must equal the full scan + combined := append(interruptedOutput, resumedOutput...) + require.Equal(t, fullOutput, combined, "interrupted + resumed should equal full scan") +} + func TestRunner_domain_targets(t *testing.T) { options := &Options{} r, err := New(options) From 299cb6087113f1dd09875b7c9f6ec8f2e54b5691 Mon Sep 17 00:00:00 2001 From: "ayuxsec@proton.me" Date: Thu, 5 Feb 2026 19:45:14 +0530 Subject: [PATCH 17/39] merge: reconcile Result struct md tags with dev fields --- runner/types.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runner/types.go b/runner/types.go index 9acd0e7d..1eaa3608 100644 --- a/runner/types.go +++ b/runner/types.go @@ -102,6 +102,9 @@ type Result struct { Response *httpx.Response `json:"-" csv:"-" md:"-" mapstructure:"-"` FaviconData []byte `json:"-" csv:"-" md:"-" mapstructure:"-"` Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" md:"-" mapstructure:"trace"` + FileNameHash string `json:"-" csv:"-" md:"-" mapstructure:"-"` + CPE []CPEInfo `json:"cpe,omitempty" csv:"cpe" md:"cpe" mapstructure:"cpe"` + WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress" md:"wordpress" mapstructure:"wordpress"` } type Trace struct { From 80e0d7446e49ec4660b5bcb891e83fe4ffe3c6ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Feb 2026 11:12:11 +0000 Subject: [PATCH 18/39] chore(deps): bump golang.org/x/net from 0.49.0 to 0.50.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.49.0 to 0.50.0. - [Commits](https://github.com/golang/net/compare/v0.49.0...v0.50.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-version: 0.50.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 14 +++++++------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index f826b775..78577697 100644 --- a/go.mod +++ b/go.mod @@ -44,9 +44,9 @@ require ( go.etcd.io/bbolt v1.4.0 // indirect go.uber.org/multierr v1.11.0 golang.org/x/exp v0.0.0-20250911091902-df9299821621 - golang.org/x/net v0.49.0 - golang.org/x/sys v0.40.0 // indirect - golang.org/x/text v0.33.0 + golang.org/x/net v0.50.0 + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 ) require ( @@ -170,12 +170,12 @@ require ( github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.47.0 // indirect - golang.org/x/mod v0.31.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/mod v0.32.0 // indirect golang.org/x/oauth2 v0.28.0 // indirect golang.org/x/sync v0.19.0 // indirect - golang.org/x/term v0.39.0 // indirect + golang.org/x/term v0.40.0 // indirect golang.org/x/time v0.11.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/tools v0.41.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 25b6bb6f..f4dc82b4 100644 --- a/go.sum +++ b/go.sum @@ -530,8 +530,8 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -564,8 +564,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -596,8 +596,8 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -662,8 +662,8 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -676,8 +676,8 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -694,8 +694,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= @@ -729,8 +729,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From bbf49501177c958977cbc0e97b998731ca26371d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Feb 2026 17:57:28 +0000 Subject: [PATCH 19/39] chore(deps): bump github.com/refraction-networking/utls Bumps [github.com/refraction-networking/utls](https://github.com/refraction-networking/utls) from 1.7.1 to 1.8.2. - [Release notes](https://github.com/refraction-networking/utls/releases) - [Commits](https://github.com/refraction-networking/utls/compare/v1.7.1...v1.8.2) --- updated-dependencies: - dependency-name: github.com/refraction-networking/utls dependency-version: 1.8.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 3 +-- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f826b775..4ffed97c 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,6 @@ require ( github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.6 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect - github.com/cloudflare/circl v1.6.1 // indirect github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -136,7 +135,7 @@ require ( github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect github.com/projectdiscovery/retryabledns v1.0.113 // indirect - github.com/refraction-networking/utls v1.7.1 // indirect + github.com/refraction-networking/utls v1.8.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect diff --git a/go.sum b/go.sum index 25b6bb6f..49b22459 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= -github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a h1:Ohw57yVY2dBTt+gsC6aZdteyxwlxfbtgkFEMTEkwgSw= github.com/cnf/structhash v0.0.0-20250313080605-df4c6cc74a9a/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corona10/goimagehash v1.1.0 h1:teNMX/1e+Wn/AYSbLHX8mj+mF9r60R1kBeqE9MkoYwI= @@ -371,8 +369,8 @@ github.com/projectdiscovery/utils v0.9.0/go.mod h1:zcVu1QTlMi5763qCol/L3ROnbd/UP github.com/projectdiscovery/wappalyzergo v0.2.66 h1:DEF7wthjvBo6oYKxfKL6vPNaqsKYUmiWODt7Mybcins= github.com/projectdiscovery/wappalyzergo v0.2.66/go.mod h1:Oc+U2RPJObmpi6LW5lTMEDiKagcKZNkEfZfwrVMURa0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0= -github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ= +github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= +github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= From ea32607b65f4a6f6464dfd931a2a1be47d0d0f88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Feb 2026 14:10:01 +0000 Subject: [PATCH 20/39] chore(deps): bump the modules group with 2 updates Bumps the modules group with 2 updates: [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) and [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo). Updates `github.com/projectdiscovery/cdncheck` from 1.2.21 to 1.2.22 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.2.21...v1.2.22) Updates `github.com/projectdiscovery/wappalyzergo` from 0.2.66 to 0.2.67 - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.2.66...v0.2.67) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.2.22 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-version: 0.2.67 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 33d48226..827a2807 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/miekg/dns v1.1.68 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 - github.com/projectdiscovery/cdncheck v1.2.21 + github.com/projectdiscovery/cdncheck v1.2.23 github.com/projectdiscovery/clistats v0.1.1 github.com/projectdiscovery/dsl v0.8.13 github.com/projectdiscovery/fastdialer v0.5.4 @@ -36,7 +36,7 @@ require ( github.com/projectdiscovery/tlsx v1.2.2 github.com/projectdiscovery/useragent v0.0.107 github.com/projectdiscovery/utils v0.9.0 - github.com/projectdiscovery/wappalyzergo v0.2.66 + github.com/projectdiscovery/wappalyzergo v0.2.68 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index 78c5aa3c..6685016b 100644 --- a/go.sum +++ b/go.sum @@ -322,8 +322,8 @@ github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.2.21 h1:+y77BGCZoduX5bja2SGn4AdBXFwfOycaLnWWUIiZCBM= -github.com/projectdiscovery/cdncheck v1.2.21/go.mod h1:gpeX5OrzaC4DmeUGDcKrC7cPUXQvRGTY/Ui0XrVfdzU= +github.com/projectdiscovery/cdncheck v1.2.23 h1:PTXUEG3fJYuBSdi+7O76V/dQYd14TLn84lVABGMNhuU= +github.com/projectdiscovery/cdncheck v1.2.23/go.mod h1:Y1KQmACY+AifbuPX/W7o8lWssiWmAZ5d/KG8qkmFm9I= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= github.com/projectdiscovery/dsl v0.8.13 h1:HjjHta7c02saH2tUGs8CN5vDeE2MyWvCV32koT8ZCWs= @@ -366,8 +366,8 @@ github.com/projectdiscovery/useragent v0.0.107 h1:45gSBda052fv2Gtxtnpx7cu2rWtUpZ github.com/projectdiscovery/useragent v0.0.107/go.mod h1:yv5ZZLDT/kq6P+NvBcCPq6sjEVQtZGgO+OvvHzZ+WtY= github.com/projectdiscovery/utils v0.9.0 h1:eu9vdbP0VYXI9nGSLfnOpUqBeW9/B/iSli7U8gPKZw8= github.com/projectdiscovery/utils v0.9.0/go.mod h1:zcVu1QTlMi5763qCol/L3ROnbd/UPSBP8fI5PmcnF6s= -github.com/projectdiscovery/wappalyzergo v0.2.66 h1:DEF7wthjvBo6oYKxfKL6vPNaqsKYUmiWODt7Mybcins= -github.com/projectdiscovery/wappalyzergo v0.2.66/go.mod h1:Oc+U2RPJObmpi6LW5lTMEDiKagcKZNkEfZfwrVMURa0= +github.com/projectdiscovery/wappalyzergo v0.2.68 h1:bN/SIfzsZ0Q0H5m4AeiuB65AbJu9oOjSMqJ10YjvsHI= +github.com/projectdiscovery/wappalyzergo v0.2.68/go.mod h1:Oc+U2RPJObmpi6LW5lTMEDiKagcKZNkEfZfwrVMURa0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= From e3965cec77ab9d14d1832a0aadd33257f9d30661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 3 Mar 2026 21:01:04 +0300 Subject: [PATCH 21/39] style: compact struct and const block --- internal/db/config.go | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/internal/db/config.go b/internal/db/config.go index c057b80b..68c2c6d3 100644 --- a/internal/db/config.go +++ b/internal/db/config.go @@ -9,31 +9,21 @@ import ( ) const ( - DefaultBatchSize = 100 - + DefaultBatchSize = 100 DefaultFlushInterval = time.Minute - - DefaultTableName = "results" - - DefaultDatabaseName = "httpx" - - EnvConnectionString = "HTTPX_DB_CONNECTION_STRING" + DefaultTableName = "results" + DefaultDatabaseName = "httpx" + EnvConnectionString = "HTTPX_DB_CONNECTION_STRING" ) type Config struct { - Type DatabaseType `yaml:"type"` - - ConnectionString string `yaml:"connection-string"` - - DatabaseName string `yaml:"database-name"` - - TableName string `yaml:"table-name"` - - BatchSize int `yaml:"batch-size"` - - FlushInterval time.Duration `yaml:"flush-interval"` - - OmitRaw bool `yaml:"omit-raw"` + Type DatabaseType `yaml:"type"` + ConnectionString string `yaml:"connection-string"` + DatabaseName string `yaml:"database-name"` + TableName string `yaml:"table-name"` + BatchSize int `yaml:"batch-size"` + FlushInterval time.Duration `yaml:"flush-interval"` + OmitRaw bool `yaml:"omit-raw"` } func (c *Config) Validate() error { From 1c4561ba07a862564651ef95376e3d77563c3749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 16 Feb 2026 12:47:13 +0300 Subject: [PATCH 22/39] feat: replace page classifier with dit, add -fpt flag Closes #2403 --- .github/workflows/functional-test.yml | 2 +- .github/workflows/release-binary.yml | 4 +- .github/workflows/release-test.yml | 2 +- Dockerfile | 2 +- README.md | 5 +- common/pagetypeclassifier/clf.gob | Bin 4593 -> 0 bytes common/pagetypeclassifier/dataset.txt | 380 ------------------ .../pagetypeclassifier/pagetypeclassifier.go | 156 ------- .../pagetypeclassifier_test.go | 143 ------- go.mod | 14 +- go.sum | 28 +- runner/options.go | 8 +- runner/runner.go | 41 +- 13 files changed, 60 insertions(+), 725 deletions(-) delete mode 100644 common/pagetypeclassifier/clf.gob delete mode 100644 common/pagetypeclassifier/dataset.txt delete mode 100644 common/pagetypeclassifier/pagetypeclassifier.go delete mode 100644 common/pagetypeclassifier/pagetypeclassifier_test.go diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index cd1b9071..007e7ce0 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.21.x + go-version-file: 'go.mod' - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 884e0f4b..999ea91f 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -17,8 +17,8 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v4 - with: - go-version: 1.21.x + with: + go-version-file: 'go.mod' - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v4 diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index c9151853..bbdd4ef1 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.21.x + go-version-file: 'go.mod' - name: release test uses: goreleaser/goreleaser-action@v4 diff --git a/Dockerfile b/Dockerfile index 6cbe235c..279594ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Base -FROM golang:1.24.5-alpine AS builder +FROM golang:1.25.7-alpine AS builder RUN apk add --no-cache git build-base gcc musl-dev WORKDIR /app diff --git a/README.md b/README.md index 7833966a..5dddd737 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ # Installation Instructions -`httpx` requires **go >=1.24.0** to install successfully. Run the following command to get the repo: +`httpx` requires **go >=1.25.0** to install successfully. Run the following command to get the repo: ```sh go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest @@ -151,7 +151,8 @@ EXTRACTOR: FILTERS: -fc, -filter-code string filter response with specified status code (-fc 403,401) - -fep, -filter-error-page filter response with ML based error page detection + -fpt, -filter-page-type string[] filter response with specified page type (e.g. -fpt login,captcha,parked) + -fep, -filter-error-page [DEPRECATED: use -fpt] filter response with ML based error page detection -fd, -filter-duplicates filter out near-duplicate responses (only first response is retained) -fl, -filter-length string filter response with specified content length (-fl 23,33) -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) diff --git a/common/pagetypeclassifier/clf.gob b/common/pagetypeclassifier/clf.gob deleted file mode 100644 index 7624ee775612207404113e5c48afcf31f7cc09b1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4593 zcmY+I&yFic6^F;=>T-{d*$|7PlB*TUcQo<6UEF!T$NR+$FcHQZ&>Qr~z z*Bc0bRy+Z)JHzjZqE#Nac8DoU4PE`o$vh04}bXeazQ|piJ z6MJ_3r;Y7y%fh7;)5AYjRZPiMgD!S)47DzI&edC6-78(k zFl-EyxfC|#80)D4Ttn(SXIzk6kp>tbTcp7~s;bTP!vSGuf{t9j&gu^(OE>r&SA zaqRV~wR=8^jG5!B47(yH`s%x^ZBuo4XLWab*D+3;N+wRK=sVXm`g#}B*4Ip}S1iX? zhLU5tH=V@eT7Hr*_F;w;$8BR&Jm}@!?Z>)Eet%%2gKr%t(wbRYWo#FlxG(iNj%lM0 zv32D##;(+j-^uC?kxE)O@yK5i7+;mjhPB@LP`^}uD2)y0Hm1_-#%AXm&gc8Cv6UuG zZ6|>zNJsBZYSoXOL`#u?_?iu?l@CYT_*%%OPGsTRbzUgdq23c z*d;Q|<~fB{`&h0zJII06k^|hx#PwJWMh^$yztF_Vmn&p&w7%I8hr16~az;X{4JY@N zSL?x5hrn+(D1D0Q-W24j?-ko_obA=}2e)^6mE2CmtklUFW2=f0uU7M`dc_0Dg(xzO zt-Nyka=G(w%o?$vs{!rsp?6KKuWjA>;Hd#UDf*ZCvUVZH`GmU4nHVpxJUiag+ zuafUr$!%l2SL@pEcD`b5ul2!-e2$h=AGD$%4RWshP7|n)jX`1}f63mqaqZ2j&JsH7 zq9R52dO7xvNnFz@UD@8L($-SIV4&lQO79?pCV$aNp#0IIT`i1w_~W%Gr;++r%P}>23559X;66kU-`A^ciYIDU z#2`sk9QO;TG~5yXkFw%=q?Y3^x~7z4)5H_yA45$TK$ajiNFXYB0xUFpt2bqcWM82C z&x?F=n)H)k$_q90K!nwD?*^s!qpx#p)&^uXLXWu`FKE!9;fv8r{U>LugDF}Ys9FMw zlBkhB2CW0&2M}p*!&4X7aJ>CgUYt@CJ;hoZWK|JXgIHV0rGa*>uha^0coMBvfF_AT ziq6HZQP1Q0I&RkxyWtfpIL@^qxeo%eT)~tKWxm!@qw4}m_#;)EtKm{qay$vKRv+E5 z@!d|~nn^4KKE9i|^=bvY^88tNParL5rfDGR?$iTTB-i_WkhE5gwWRQo9TiLgyWb=sfLy(Vs4}8jg2%^4suG5kYi5MY;TI(!rA4a7;O-mJH2M3fUAuKXCCYb?M z5Q5|ZQaOWQ3(}Kwrm$POO(uQ@vuQcK-EWY3)8ynFeLvcU`7GlT$>B(2XLK*j&|A8b zdInK7HwZxlY3*t4(!M0f5Q6%?h*p&qB4%}xB|XfmGHc$)V6JQ1AGXn^n$N;sn!(96 zob9$?HV<_wVqeX>QQ>sV{S@eFk#&)~m9HL{iM=(tFGbt20WE zpCI-`iJ2hJ8I6vC&;?BF1#dyc5qlU?WL~T_)J+=NJjsjORF<0Msqy#cPY{+`wa z;QZbXs*Jh4k3p1|hksV=W#A3Foebp6Z&V<5YvR{u6t8x7H>-~tAahS=&d8$m$#NIZh+Tw|a!co&z^<0Pnl zF@24wcte*w4{-vZ%C_hyn)Z!#+}STU*xZ0uTNm6;<`DcR55ASMb*XT8n9NgIU^@K% z(L_wf00iY`yPnzuDY9p2NyapgzsI+TSk}v$DxNkEMZi)FJD=JOViK^4+XOBMS(FW~ zX4goS&FvUaP)!P5+GaQit@yxPIu)-0^k(yDg_dG-Ch0%?O(IlJP8uE0Hul>q5ks); zt4Baj1$r%pj%fL)iq_Sh{?>>>mbUY1CH9*Qe&?bG#t6kZ()xU4kzs!=&6?;FlAjJ;C6USIfVE(d=s(;w2h(moFBnfWG*_$8Zmtsh$=M>Ca> z>Hdt8pjFp3T!z&3d;y4tZv^ELaTY6k&n343hqPD^0$@6Esfp z17U;f)YJ$V8hN2S;ehh|MRtLPqt{+8Giph(s};7liDxKAoSHMauSszeyZ^tTT&&te ztq;ry3b_7U3Lp34gLEkus0%S3bObfiX07QWonEmF;(+yZJXS6`8yqD>DLEIYa8QOQ zCV^daDua9ctvSX9yGw%WT5yD-x2C1Q@@*pE zdS?^1&V`llX|pHEj>1gM@Tq|;Y5w`>4q?qTXttJa&n;(Xv&j`A&y1UV{ssfzdEl6V zPsor)_83_u#BR0%>ghfp&n9DcX>YS{{}p!$#PiryG}Ud*#KqZ0aMXr}I7R5d6(2gR zEIcO6j|S9-ZX7oA9c6A~kBvody&#l#@Z7Wp`WDyOJV7Q97fO*wb{*o=zjHx*=t`a$lW4;JE4G*0L*tYZ{NdRRn6F! zPxj+x@%HAwk6{mD-MdkW9n#bNPQeRUb}w)ofMkhl7;^B8kneD&u)$-n>n=<0t? CYrh}> diff --git a/common/pagetypeclassifier/dataset.txt b/common/pagetypeclassifier/dataset.txt deleted file mode 100644 index c55ab2ef..00000000 --- a/common/pagetypeclassifier/dataset.txt +++ /dev/null @@ -1,380 +0,0 @@ -The Forum page seems to have a glitch. Our technicians are on it.||error -There was a problem with the Product Details page. Try reloading.||error -Error 500: The E-books page is experiencing a problem.||error -Unfortunately, the Video Tutorials page is down for maintenance.||error -Our Archive page is currently unavailable. We apologize for the inconvenience.||error -We're having trouble loading the Membership Details page.||error -An error occurred while trying to access the Profile Settings page.||error -Error 404: The Team page could not be found.||error -Our Project Highlights page seems to be having some technical issues.||error -We're sorry, but we can't seem to find the Donations page.||error -You've landed on our Forum page. Engage in interesting discussions.||nonerror -Welcome to the Product Details page. Learn more about our products here.||nonerror -You are now on our E-books page. Enjoy a wealth of knowledge.||nonerror -This is the Video Tutorials page. Learn with our easy-to-follow videos.||nonerror -Welcome to our Archive. Dive into our rich history.||nonerror -You're now on the Membership Details page. See the benefits of joining us.||nonerror -This is your Profile Settings page. Update your personal details as needed.||nonerror -You're on the Team page. Meet the people behind our organization.||nonerror -Welcome to our Project Highlights page. See what we've been up to.||nonerror -You've landed on the Donations page. Every contribution helps us do more.||nonerror -500 - Server Error This is highly unusual! Our tech team have been notified and are working on it.||error -Sorry this page is currently under maintenance.||error -Access Denied - You don't have permission to access this page.||error -This page seems to be missing 404 Error!||error -Sorry something went wrong. Please try again later.||error -We're sorry this page could not be found 404.||error -The page you requested could not be found on our site.||error -500 - Internal server error. There is a problem with the resource you are looking for and it cannot be displayed.||error -Error 401 Unauthorized: Access is denied due to invalid credentials.||error -Bad request 400. Your browser sent a request that this server could not understand.||error -This is a 404 error page||error -Sorry this page does not exist||error -Error 500: Internal Server Error||error -Oops! That page can’t be found.Try searching from the field above or go to the home page.||error -An error has occurred while processing your request. It happens to the best of us! Don't worry! There are no bugs without a fix! Let's try again! What were you looking for? If you are an adventurer search this site! If difficulties persist please contact the website administrator and report the error below. 404 Page not found||error -Whoops our bad... The page you requested was not found and we have a fine guess why. If you typed the URL directly please make sure the spelling is correct. If you clicked on a link to get here the link is outdated. What can you do? Have no fear help is near! There are many ways you can get back on track with Magento Store. Go back to the previous page. Use the search bar at the top of the page to search for your products. Follow these links to get you back on track! Store Home My Account||error -404 - Page not found Unfortunately the requested page could not be found.||error -PAGE NOT FOUND The page you're looking for doesn't seem to exist anymore… Return to the homepage||error -Who moved my... lemon? Oh no - looks like we can't find the page you are looking for. But you know the saying; when life gives you lemons... okay we can't find a clever way to end that sentence but we do have 2 suggestions to help you find what you were looking for: Go to the front page Or Search for a specific topic If something you need really is missing we would love it if you would let us know ❤️️||error -404—page not found||error -Apologies but there's a 503 Service Unavailable error. The server cannot handle the request.||error -Sorry you don't have access rights to this page. Error 403: Forbidden.||error -404 - Oops! The page you are looking for has been misplaced.||error -Sorry the server encountered an unexpected condition that prevented it from fulfilling the request. Error 500: Internal Server Error.||error -Whoa! The page you're looking for seems to have vanished. Error 404.||error -Sorry this page has moved or doesn't exist anymore. Error 404.||error -Sorry but your request timed out. Please try again. Error 504: Gateway Timeout.||error -We're sorry but an unknown error occurred while processing your request.||error -Error 502: Bad Gateway. The server encountered a temporary error and could not complete your request.||error -The requested resource could not be found on this server. Please verify your request and try again. Error 404.||error -This Help Center page is temporarily unavailable.||error -Privacy Policy page not found. Please try again later.||error -There seems to be an error on our Services page. We're working to fix it.||error -An error occurred while loading the Search Results page.||error -Category page not found. It might have been removed or relocated.||error -There was a problem loading the Cart page. Please try again.||error -Our Terms of Service page is currently down for maintenance.||error -We're sorry, but the Sitemap is not available at the moment.||error -We're having trouble loading the Reviews page.||error -An error occurred while trying to access the Partners page.||error -Settings page is currently unavailable. We apologize for the inconvenience.||error -Error 404: Resources page not found.||error -Our Press Releases page seems to be having some technical issues.||error -We're sorry, but we can't seem to find the Case Studies page.||error -There was a problem loading the Community page. Please refresh the page.||error -Error 503: The Subscriptions page is temporarily unavailable.||error -There's a problem with our Customer Support page. We're on it.||error -We're having trouble finding the Notifications page. It may have been moved.||error -There was a problem with the Feedback page. Try again later.||error -Our Transactions page is currently experiencing some issues. We appreciate your patience.||error -Your request has been successfully submitted.||nonerror -You have successfully logged out.||nonerror -Congratulations on successfully completing the course!||nonerror -The payment has been processed successfully.||nonerror -Thank you for your feedback!||nonerror -Your download will start shortly.||nonerror -Profile updated successfully.||nonerror -Thanks for contacting us! We'll get back to you as soon as possible.||nonerror -Sign-up successful. Welcome to our community!||nonerror -Your booking has been confirmed. Check your email for details.||nonerror -Welcome! Your registration was successful.||nonerror -Congratulations! You've successfully updated your profile.||nonerror -Great! Your order was placed successfully. We'll send you an email confirmation soon.||nonerror -Welcome back! Your login was successful.||nonerror -Success! You've added the item to your cart.||nonerror -Your request was sent successfully. We'll get back to you as soon as possible.||nonerror -Great job! Your settings have been saved.||nonerror -Your message has been submitted successfully. We appreciate your feedback.||nonerror -Thank you for subscribing to our newsletter!||nonerror -Great news! Your transaction was successful.||nonerror -Welcome to our homepage. Feel free to browse around||nonerror -Thanks for signing up! You're now a registered user.||nonerror -Your order has been placed successfully! You'll receive a confirmation email shortly||nonerror -Congratulations your account has been successfully created||nonerror -Thank you for your inquiry. We will respond to your message within 24 hours||nonerror -You've successfully added the item to your cart!||nonerror -Success! Your password has been updated||nonerror -Welcome back! You have successfully logged in||nonerror -Great job! Your profile has been updated||nonerror -Your message was sent successfully. We'll get back to you shortly||nonerror -Welcome to our website. Explore and enjoy our services.||nonerror -Thank you for visiting our About Us page. Learn more about our journey and team.||nonerror -You are now browsing our Products page. Check out our latest offerings.||nonerror -This is our Contact Us page. Feel free to reach out with any queries or feedback.||nonerror -You have reached the end of the page. Scroll up to continue browsing.||nonerror -Welcome to the News section. Stay updated with our latest announcements.||nonerror -Now viewing: Image Gallery. Enjoy a visual tour of our activities.||nonerror -You're on our FAQ page. Get answers to common questions.||nonerror -Welcome to the Blog section. Engage with our thoughts and insights.||nonerror -This is the Discussion Forum. Join in, ask questions, or help others.||nonerror -You're on the Login page. Enter your credentials to access your account.||login -Welcome to the Sign-Up page. Join our community today.||nonerror -This is your User Dashboard. Manage your account and settings here.||nonerror -You've reached the Checkout page. Review your order and proceed to payment.||nonerror -Welcome to the Download section . Access our digital resources here.||nonerror -This is the Careers page. Explore job opportunities with us.||nonerror -You're viewing the Events Calendar. Keep track of upcoming activities.||nonerror -This is the User Profile page. Update your information as needed.||nonerror -Welcome to our Testimonials page. Read reviews and stories from our users.||nonerror -You are now on the Home page. Start exploring from here.||nonerror -Welcome to home page||nonerror -You're now on our Help Center page. Find answers to common questions here.||nonerror -Welcome to our Privacy Policy page. Learn how we protect your personal information.||nonerror -You've landed on the Services page. Explore what we have to offer.||nonerror -This is the Search Results page. Did you find what you were looking for?||nonerror -Now browsing the Category page. View all items in this category.||nonerror -You're now on the Cart page. Review your selections before proceeding to checkout.||nonerror -Welcome to our Terms of Service page. Understand our conditions for providing services.||nonerror -You are currently on our Sitemap. Navigate our website with ease.||nonerror -You are on the Reviews page. Check out what others have to say about us.||nonerror -Now viewing the Partners page. Meet the organizations we collaborate with.||nonerror -You're on the Settings page. Customize your user experience.||nonerror -This is our Resources page. Access useful documents and guides.||nonerror -You've landed on the Press Releases page. Stay updated with our latest news.||nonerror -Welcome to our Case Studies page. Discover our past projects and achievements.||nonerror -You're now on the Community page. Connect and interact with other members.||nonerror -You are currently on the Subscriptions page. Manage your preferences here.||nonerror -Now viewing the Customer Support page. We're here to help.||nonerror -This is the Notifications page. Keep track of your updates and alerts.||nonerror -You've landed on the Feedback page. Share your thoughts with us.||nonerror -Welcome to the Transactions page. Monitor your past and current transactions.||nonerror -500 - Server Error This is highly unusual! Our tech team have been notified and are working on it.||error -Sorry this page is currently under maintenance.||error -Access Denied - You don't have permission to access this page.||error -This page seems to be missing 404 Error!||error -Sorry something went wrong. Please try again later.||error -We're sorry this page could not be found 404.||error -The page you requested could not be found on our site.||error -500 - Internal server error. There is a problem with the resource you are looking for and it cannot be displayed.||error -Error 401 Unauthorized: Access is denied due to invalid credentials.||error -Bad request 400. Your browser sent a request that this server could not understand.||error -Your request has been successfully submitted.||nonerror -You have successfully logged out.||nonerror -Congratulations on successfully completing the course!||nonerror -The payment has been processed successfully.||nonerror -Thank you for your feedback!||nonerror -Your download will start shortly.||nonerror -Profile updated successfully.||nonerror -Thanks for contacting us! We'll get back to you as soon as possible.||nonerror -Sign-up successful. Welcome to our community!||nonerror -Your booking has been confirmed. Check your email for details.||nonerror -This is a 404 error page||error -Sorry this page does not exist||error -Error 500: Internal Server Error||error -Oops! That page can’t be found.Try searching from the field above or go to the home page.||error -An error has occurred while processing your request. It happens to the best of us! Don't worry! There are no bugs without a fix! Let's try again! What were you looking for? If you are an adventurer search this site! If difficulties persist please contact the website administrator and report the error below. 404 Page not found||error -Whoops our bad... The page you requested was not found and we have a fine guess why. If you typed the URL directly please make sure the spelling is correct. If you clicked on a link to get here the link is outdated. What can you do? Have no fear help is near! There are many ways you can get back on track with Magento Store. Go back to the previous page. Use the search bar at the top of the page to search for your products. Follow these links to get you back on track! Store Home | My Account||error -404 - Page not found Unfortunately the requested page could not be found.||error -PAGE NOT FOUND The page you're looking for doesn't seem to exist anymore… Return to the homepage||error -Who moved my... lemon? Oh no - looks like we can't find the page you are looking for. But you know the saying; when life gives you lemons... okay we can't find a clever way to end that sentence but we do have 2 suggestions to help you find what you were looking for: Go to the front page Or Search for a specific topic If something you need really is missing we would love it if you would let us know ❤️️||error -404—page not found||error -Apologies but there's a 503 Service Unavailable error. The server cannot handle the request.||error -Sorry you don't have access rights to this page. Error 403: Forbidden.||error -404 - Oops! The page you are looking for has been misplaced.||error -Sorry the server encountered an unexpected condition that prevented it from fulfilling the request. Error 500: Internal Server Error.||error -Whoa! The page you're looking for seems to have vanished. Error 404.||error -Sorry this page has moved or doesn't exist anymore. Error 404.||error -Sorry but your request timed out. Please try again. Error 504: Gateway Timeout.||error -We're sorry but an unknown error occurred while processing your request.||error -Error 502: Bad Gateway. The server encountered a temporary error and could not complete your request.||error -The requested resource could not be found on this server. Please verify your request and try again. Error 404.||error -Welcome! Your registration was successful.||nonerror -Congratulations! You've successfully updated your profile.||nonerror -Great! Your order was placed successfully. We'll send you an email confirmation soon.||nonerror -Welcome back! Your login was successful.||nonerror -Success! You've added the item to your cart.||nonerror -Your request was sent successfully. We'll get back to you as soon as possible.||nonerror -Great job! Your settings have been saved.||nonerror -Your message has been submitted successfully. We appreciate your feedback.||nonerror -Thank you for subscribing to our newsletter!||nonerror -Great news! Your transaction was successful.||nonerror -Welcome to our homepage. Feel free to browse around||nonerror -Thanks for signing up! You're now a registered user.||nonerror -Your order has been placed successfully! You'll receive a confirmation email shortly||nonerror -Congratulations your account has been successfully created||nonerror -Thank you for your inquiry. We will respond to your message within 24 hours||nonerror -You've successfully added the item to your cart!||nonerror -Success! Your password has been updated||nonerror -Welcome back! You have successfully logged in||nonerror -Great job! Your profile has been updated||nonerror -Your message was sent successfully. We'll get back to you shortly||nonerror -Welcome to the Login page. Please sign in to continue.||login -Please enter your username and password on the login page.||login -You have reached the login page. Access your account by logging in.||login -Login required. Please authenticate to access this page.||login -Welcome back! Please log in to your account.||login -Sign in to your account on this login page.||login -Secure Login: Enter your credentials to proceed.||login -This is the login page. Please enter your email and password.||login -Access denied. Please log in to continue.||login -You're on the login page. Forgot your password? Click here to reset.||login -User Login: Please provide your username and password.||login -Login to your account to access exclusive features.||login -Authentication required. Please log in.||login -Welcome back! Sign in to access your dashboard.||login -Please log in to proceed to the checkout page.||login -Member login: Enter your credentials below.||login -Staff login portal. Please enter your login details.||login -Customer login: Sign in to view your orders.||login -Partner login: Please authenticate to access partner resources.||login -Administrator login page. Enter your admin credentials.||login -Please log in to access your profile settings.||login -Login successful. Redirecting to your account dashboard.||login -Incorrect password. Please try again.||login -Session expired. Please log in again.||login -Welcome to the secure login page. Your privacy is important to us.||login -Access restricted. Please log in to view this content.||login -Please log in to access the members-only area.||login -Sign in with your social media account on the login page.||login -New user? Register here or log in if you already have an account.||login -Log in to participate in the forum discussions.||login -Access your account by logging in here.||login -Please log in to access your personalized dashboard.||login -Enter your login details to continue.||login -Login Page: Securely enter your credentials.||login -Welcome to the user login portal.||login -Sign in to manage your account settings.||login -This is the login screen. Please authenticate.||login -Returning user? Please log in.||login -Please log in to view your messages.||login -Log in to access premium content.||login -Authentication page: Enter your username and password.||login -Please enter your login information to proceed.||login -User authentication required. Please log in.||login -Log in now to unlock exclusive features.||login -Sign in to check your account balance.||login -Welcome back! Please enter your login credentials.||login -Member login area: Access restricted content by logging in.||login -Please sign in to continue to your profile.||login -Staff members, please log in to access internal resources.||login -Enter your email and password to log in.||login -Login required to view this page. Please sign in.||login -Access your profile by logging into your account.||login -Please provide your login credentials to access the system.||login -Log in to track your order status.||login -Welcome to the employee login page.||login -Secure area: Please log in to continue.||login -Please log in to update your preferences.||login -Sign in to access your learning materials.||login -Please authenticate to proceed to the next step.||login -Login Page: Your session has expired, please log in again.||login -Welcome back! Enter your credentials to sign in.||login -Client login: Access your project details here.||login -Agent login portal: Please sign in with your ID.||login -Enter your user ID and password to log in.||login -Log in to view your subscription details.||login -Login Page: Forgot your password? Click here to reset it.||login -Access restricted to authorized users only. Please log in.||login -Vendor login: Manage your listings by logging in.||login -Please log in to access your saved items.||login -Log in to participate in our online courses.||login -Sign in to view your appointment schedule.||login -Welcome to the admin login page.||login -Please enter your credentials to log in securely.||login -Log in to view your recent activities.||login -Authentication needed. Please sign in to proceed.||login -Member login: Keep me signed in checkbox available.||login -Log in with your email or username.||login -Access your account dashboard by logging in.||login -Sign in to post comments on articles.||login -Please log in to access your billing information.||login -Log in to access your personalized recommendations.||login -Please sign in to view your shopping cart.||login -Enter your credentials to log in and start shopping.||login -Welcome to the customer login page. Sign in to continue.||login -Authentication required. Please log in with your secure ID.||login -Log in to access exclusive member discounts.||login -Please log in to view and manage your wishlist.||login -Sign in to access your event tickets and details.||login -Faculty login: Please enter your staff ID and password.||login -Log in to access your investment portfolio.||login -Access your medical records by logging in securely.||login -Please sign in to continue to the payment gateway.||login -Login required to access your order history.||login -Welcome back! Log in to resume your session.||login -Please log in to submit your application.||login -Enter your username and password to log in to the portal.||login -Student login: Access your course materials by signing in.||login -Log in to customize your news feed preferences.||login -Please authenticate to access your secure messages.||login -Sign in to sync your data across devices.||login -Log in to join the live webinar.||login -Please log in to access your reservation details.||login -Welcome to the supplier login page.||login -Log in to access your support tickets.||login -Enter your credentials to log in and view analytics.||login -Please sign in to access developer resources.||login -Login required to view confidential documents.||login -Log in to participate in the survey.||login -Please authenticate to access the admin dashboard.||login -Sign in to view your loyalty points balance.||login -Log in to manage your email subscriptions.||login -Please log in to proceed with the enrollment process.||login -Access your download history by logging in.||login -Welcome back! Please log in to renew your membership.||login -Enter your employee ID to log in to the time tracking system.||login -Log in to update your security settings.||login -Please sign in to access your saved searches.||login -Authentication required for accessing project files.||login -Log in to collaborate with your team members.||login -Please enter your PIN and password to log in.||login -Sign in to access your fitness progress dashboard.||login -Log in to check your test results.||login -Please log in to schedule your appointments.||login -Welcome to the volunteer login page.||login -Log in to view your donation history.||login -Please authenticate to access the control panel.||login -Sign in to review and accept your job offer.||login -Log in to access premium tutorials and guides.||login -Please log in to manage your API keys.||login -Please log in with your email address and password.||login -Enter your username and password to access your account.||login -Sign in to your account using your email and password.||login -Welcome back! Please enter your login credentials.||login -Email address: [input field] Password: [input field]||login -Login to your account. Don't have one? Sign up now.||login -Username: [input field] Password: [input field] Remember me?||login -Forgot your password? Click here to reset it.||login -Please enter your email and password to continue.||login -Secure login portal. Enter credentials below.||login -Access your account by logging in below.||login -Remember me on this device.||login -Login required. Please sign in to proceed.||login -Forgot password? Reset it here.||login -Sign in with your email address and password.||login -Welcome! Please log in to your account.||login -User login: Enter your email and password.||login -Email: [input field] Password: [input field]||login -Please authenticate by entering your login details.||login -Sign in to your account or register for a new one.||login -Login page: Access restricted to authorized users only.||login -Need help logging in? Click here.||login -Enter your credentials to log in.||login -Keep me signed in.||login -Please sign in to access exclusive content.||login -Welcome back! Sign in to your dashboard.||login -Forgot your username or password? Retrieve them here.||login -Log in using your email or username.||login -Authentication required. Please log in.||login -Password recovery: Reset your password now.||login -Log in to manage your account settings.||login -Sign in to continue to checkout.||login -Enter your login information below.||login -Sign in to access your personalized dashboard.||login -Welcome to the member login page.||login -Already have an account? Log in here.||login -Enter email and password to sign in.||login -Sign in to your profile.||login -Member login: Access your account here.||login -Please log in to continue.||login -Enter your password to log in.||login -Sign in to view your messages.||login -Login to your profile to see updates.||login -Log in to your account to access features.||login -Please provide your username and password.||login -Log in to manage your subscriptions.||login -Sign in using your credentials.||login -Access denied. Please log in first.||login -Authentication portal. Enter login details.||login -Need an account? Sign up or log in if you already have one.||login diff --git a/common/pagetypeclassifier/pagetypeclassifier.go b/common/pagetypeclassifier/pagetypeclassifier.go deleted file mode 100644 index 5e333b1c..00000000 --- a/common/pagetypeclassifier/pagetypeclassifier.go +++ /dev/null @@ -1,156 +0,0 @@ -package pagetypeclassifier - -import ( - _ "embed" - "fmt" - "strings" - "sync" - - htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2" - "github.com/microcosm-cc/bluemonday" - "github.com/projectdiscovery/utils/ml/naive_bayes" -) - -//go:embed clf.gob -var classifierData []byte - -type PageTypeClassifier struct { - classifier *naive_bayes.NaiveBayesClassifier -} - -func New() (*PageTypeClassifier, error) { - classifier, err := naive_bayes.NewClassifierFromFileData(classifierData) - if err != nil { - return nil, err - } - return &PageTypeClassifier{classifier: classifier}, nil -} - -func (n *PageTypeClassifier) Classify(html string) string { - text, err := htmlToText(html) - if err != nil || text == "" { - return "other" - } - return n.classifier.Classify(text) -} - -var ( - // sanitizerPolicy is an aggressive bluemonday policy that strips most HTML - // to reduce nesting depth and prevent parser stack overflow - sanitizerPolicy *bluemonday.Policy - sanitizerPolicyOnce sync.Once -) - -// getSanitizerPolicy returns an ultra-aggressive HTML sanitizer policy that strips -// almost all elements to minimize nesting depth and prevent parser stack overflow. -func getSanitizerPolicy() *bluemonday.Policy { - sanitizerPolicyOnce.Do(func() { - p := bluemonday.NewPolicy() - // Ultra-aggressive policy: Allow only the most basic text elements - // to minimize nesting and reduce parser stack depth - p.AllowElements("p", "br", "h1", "h2", "h3", "h4", "h5", "h6") - p.AllowElements("strong", "em", "b", "i") - // Remove div, span, ul, ol, li as they can create deep nesting - // No attributes allowed to prevent style-based nesting issues - sanitizerPolicy = p - }) - return sanitizerPolicy -} - -// htmlToText safely converts HTML to text with multiple fallback strategies. -// The 512 node limit in golang.org/x/net/html is hardcoded and cannot be increased. -// Strategy: -// 1. Length limit the input HTML to prevent massive documents -// 2. Sanitize HTML aggressively with bluemonday to reduce nesting -// 3. Convert sanitized HTML to markdown with panic recovery -// 4. If conversion fails, fallback to plain text extraction -func htmlToText(html string) (text string, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("html parser panic: %v", r) - text = "" - } - }() - - // Limit input size to prevent processing extremely large HTML documents - const maxHTMLSize = 1024 * 1024 // 1MB limit - if len(html) > maxHTMLSize { - html = html[:maxHTMLSize] - } - - // First, sanitize HTML with ultra-aggressive bluemonday policy - sanitizedHTML := getSanitizerPolicy().Sanitize(html) - - // If sanitization failed or produced empty result, try plain text fallback - if sanitizedHTML == "" { - return extractPlainText(html), nil - } - - // Convert sanitized HTML to markdown - text, err = htmltomarkdown.ConvertString(sanitizedHTML) - if err != nil { - // If markdown conversion fails, fallback to plain text extraction - return extractPlainText(sanitizedHTML), nil - } - - if text == "" { - // If result is empty, try plain text fallback - return extractPlainText(sanitizedHTML), nil - } - - return text, nil -} - -// extractPlainText is a simple fallback that extracts text content without HTML parsing -// This is used when the HTML parser fails due to complexity or nesting depth -func extractPlainText(html string) string { - // Simple regex-based text extraction as fallback - // Remove script and style tags first - text := html - - // Remove script tags and content - for { - start := strings.Index(text, "") - if end == -1 { - text = text[:start] - break - } - text = text[:start] + text[start+end+9:] - } - - // Remove style tags and content - for { - start := strings.Index(text, "") - if end == -1 { - text = text[:start] - break - } - text = text[:start] + text[start+end+8:] - } - - // Simple HTML tag removal (not perfect but safe) - result := "" - inTag := false - for _, char := range text { - if char == '<' { - inTag = true - } else if char == '>' { - inTag = false - result += " " // Replace tags with spaces - } else if !inTag { - result += string(char) - } - } - - // Clean up multiple spaces - words := strings.Fields(result) - return strings.Join(words, " ") -} diff --git a/common/pagetypeclassifier/pagetypeclassifier_test.go b/common/pagetypeclassifier/pagetypeclassifier_test.go deleted file mode 100644 index baf2116b..00000000 --- a/common/pagetypeclassifier/pagetypeclassifier_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package pagetypeclassifier - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestPageTypeClassifier(t *testing.T) { - t.Run("test creation of new PageTypeClassifier", func(t *testing.T) { - epc, err := New() - require.NoError(t, err) - require.NotNil(t, epc) - }) - - t.Run("test classification non error page text", func(t *testing.T) { - epc, err := New() - require.NoError(t, err) - require.NotNil(t, epc) - require.Equal(t, "nonerror", epc.Classify(` - - - - Terms of Service - - -

Welcome to our Terms of Service page.

-

Understand our conditions for providing services.

- - - `)) - }) - - t.Run("test classification on error page text", func(t *testing.T) { - epc, err := New() - require.NoError(t, err) - require.NotNil(t, epc) - require.Equal(t, "error", epc.Classify(` - - - Error 403: Forbidden - - - -
-

Error 403: Forbidden

-

Sorry you don't have access rights to this page.

-
- - - `)) - }) - - t.Run("test resilience with deeply nested HTML", func(t *testing.T) { - epc, err := New() - require.NoError(t, err) - require.NotNil(t, epc) - - // Generate deeply nested HTML that would have exceeded the 512 node stack limit - // With our enhanced sanitization and fallback mechanisms, this should now work - deeplyNestedHTML := "
" - for i := 0; i < 600; i++ { - deeplyNestedHTML += "
" - } - deeplyNestedHTML += "Some text content" - for i := 0; i < 600; i++ { - deeplyNestedHTML += "
" - } - deeplyNestedHTML += "
" - - // Should not panic and should successfully classify the content - result := epc.Classify(deeplyNestedHTML) - require.NotEmpty(t, result) - // Should be able to extract and classify the text content - require.NotEqual(t, "", result) - }) - - t.Run("test htmlToText with deeply nested HTML", func(t *testing.T) { - // Generate deeply nested HTML that would have exceeded the 512 node stack limit - deeplyNestedHTML := "
" - for i := 0; i < 600; i++ { - deeplyNestedHTML += "
" - } - deeplyNestedHTML += "Some text content" - for i := 0; i < 600; i++ { - deeplyNestedHTML += "
" - } - deeplyNestedHTML += "
" - - // Should not panic and should successfully extract text with enhanced sanitization - result, err := htmlToText(deeplyNestedHTML) - require.NoError(t, err) - require.NotEmpty(t, result) - require.Contains(t, result, "Some text content") - }) - - t.Run("test htmlToText with normal HTML", func(t *testing.T) { - normalHTML := `

Title

Some content here

` - result, err := htmlToText(normalHTML) - require.NoError(t, err) - require.NotEmpty(t, result) - }) - - t.Run("test htmlToText with extremely large HTML", func(t *testing.T) { - // Create a very large HTML document (over 1MB) - largeContent := strings.Repeat("

This is a test paragraph with some content. ", 50000) - largeHTML := "" + largeContent + "" - - // Should handle large documents without panic - result, err := htmlToText(largeHTML) - require.NoError(t, err) - require.NotEmpty(t, result) - }) - - t.Run("test extractPlainText fallback", func(t *testing.T) { - htmlWithScriptAndStyle := ` - - - - - -

Title

-

Some important content here

-
Nested content
- - ` - - result := extractPlainText(htmlWithScriptAndStyle) - require.NotEmpty(t, result) - require.Contains(t, result, "Title") - require.Contains(t, result, "important") - require.Contains(t, result, "content") - // Should not contain script or style content - require.NotContains(t, result, "alert") - require.NotContains(t, result, "color: red") - }) -} diff --git a/go.mod b/go.mod index 16d765e5..2edf3e32 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/httpx -go 1.24.1 +go 1.25.7 require ( github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 @@ -50,13 +50,13 @@ require ( ) require ( - github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 github.com/dustin/go-humanize v1.0.1 github.com/go-sql-driver/mysql v1.9.3 github.com/lib/pq v1.10.9 go.mongodb.org/mongo-driver v1.17.6 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 + github.com/happyhackingspace/dit v0.0.9 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 github.com/seh-msft/burpxml v1.0.1 github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 @@ -66,9 +66,8 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect - github.com/JohannesKaufmann/dom v0.2.0 // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect - github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/STARRY-S/zip v0.2.3 // indirect github.com/VividCortex/ewma v1.2.0 // indirect @@ -107,14 +106,13 @@ require ( github.com/gorilla/css v1.0.1 // indirect github.com/gosimple/slug v1.15.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect - github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/go-version v1.8.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/iangcarroll/cookiemonster v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kataras/jwt v0.1.10 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/klauspost/pgzip v1.2.6 // indirect - github.com/kljensen/snowball v0.8.0 // indirect github.com/logrusorgru/aurora/v4 v4.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect @@ -179,10 +177,10 @@ require ( go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/mod v0.32.0 // indirect - golang.org/x/oauth2 v0.28.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/term v0.40.0 // indirect - golang.org/x/time v0.11.0 // indirect + golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.41.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 004cf18c..97966ff8 100644 --- a/go.sum +++ b/go.sum @@ -22,14 +22,10 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= -github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo= -github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 h1:mklaPbT4f/EiDr1Q+zPrEt9lgKAkVrIBtWf33d9GpVA= -github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0/go.mod h1:D56Cl9r8M5i3UwAchE+LlLc5hPN3kJtdZNVJn06lSHU= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= @@ -210,8 +206,10 @@ github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo= github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/happyhackingspace/dit v0.0.9 h1:8Owu16jksGExJQdMODAHuFN6k0FTl0ZAlPTjv7X8dtY= +github.com/happyhackingspace/dit v0.0.9/go.mod h1:lGjPkdt8Xdmge1QeiWAoQpUhDTLNmKLniKRSCFGgjuY= +github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= +github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -243,8 +241,6 @@ github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxh github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/kljensen/snowball v0.8.0 h1:WU4cExxK6sNW33AiGdbn4e8RvloHrhkAssu2mVJ11kg= -github.com/kljensen/snowball v0.8.0/go.mod h1:OGo5gFWjaeXqCu4iIrMl5OYip9XUJHGOU5eSkPjVg2A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -393,12 +389,8 @@ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uRPSpfNZkY= github.com/sashabaranov/go-openai v1.37.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= -github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc= -github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/seh-msft/burpxml v1.0.1 h1:5G3QPSzvfA1WcX7LkxmKBmK2RnNyGviGWnJPumE0nwg= github.com/seh-msft/burpxml v1.0.1/go.mod h1:lTViCHPtGGS0scK0B4krm6Ld1kVZLWzQccwUomRc58I= -github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= -github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -618,8 +610,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= -golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -713,8 +705,8 @@ golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/runner/options.go b/runner/options.go index 6361f5c1..af720514 100644 --- a/runner/options.go +++ b/runner/options.go @@ -202,6 +202,7 @@ type Options struct { OutputMatchContentLength string OutputFilterStatusCode string OutputFilterErrorPage bool + OutputFilterPageType goflags.StringSlice FilterOutDuplicates bool OutputFilterContentLength string InputRawRequest string @@ -451,7 +452,8 @@ func ParseOptions() *Options { flagSet.CreateGroup("filters", "Filters", flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"), - flagSet.BoolVarP(&options.OutputFilterErrorPage, "filter-error-page", "fep", false, "filter response with ML based error page detection"), + flagSet.StringSliceVarP(&options.OutputFilterPageType, "filter-page-type", "fpt", nil, "filter response with specified page type (e.g. -fpt login,captcha,parked)", goflags.CommaSeparatedStringSliceOptions), + flagSet.BoolVarP(&options.OutputFilterErrorPage, "filter-error-page", "fep", false, "[DEPRECATED: use -fpt] filter response with ML based error page detection"), flagSet.BoolVarP(&options.FilterOutDuplicates, "filter-duplicates", "fd", false, "filter out near-duplicate responses (only first response is retained)"), flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"), flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"), @@ -872,6 +874,10 @@ func (options *Options) configureOutput() { if options.CSVOutputEncoding != "" { options.CSVOutput = true } + if options.OutputFilterErrorPage && len(options.OutputFilterPageType) == 0 { + gologger.Info().Msg("-fep is deprecated, use -fpt error instead") + options.OutputFilterPageType = goflags.StringSlice{"error"} + } } func (options *Options) configureResume() error { diff --git a/runner/runner.go b/runner/runner.go index 8647c23f..69556a91 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -36,7 +36,7 @@ import ( "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" "github.com/projectdiscovery/httpx/common/inputformats" - "github.com/projectdiscovery/httpx/common/pagetypeclassifier" + "github.com/happyhackingspace/dit" "github.com/projectdiscovery/httpx/common/authprovider" "github.com/projectdiscovery/httpx/static" "github.com/projectdiscovery/mapcidr/asn" @@ -92,7 +92,7 @@ type Runner struct { ratelimiter ratelimit.Limiter HostErrorsCache gcache.Cache[string, int] browser *Browser - pageTypeClassifier *pagetypeclassifier.PageTypeClassifier // Include this for general page classification + ditClassifier *dit.Classifier pHashClusters []pHashCluster simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection httpApiEndpoint *Server @@ -430,11 +430,11 @@ func New(options *Options) (*Runner, error) { } runner.simHashes = gcache.New[uint64, struct{}](1000).ARC().Build() - pageTypeClassifier, err := pagetypeclassifier.New() + ditClassifier, err := dit.New() if err != nil { - return nil, err + gologger.Warning().Msgf("Could not initialize page classifier: %s", err) } - runner.pageTypeClassifier = pageTypeClassifier + runner.ditClassifier = ditClassifier if options.SecretFile != "" { authProviderOpts := &authprovider.AuthProviderOptions{ @@ -652,6 +652,22 @@ func (r *Runner) duplicate(result *Result) bool { return false } +func (r *Runner) classifyPage(body string, pHash uint64) map[string]interface{} { + kb := map[string]any{"pHash": pHash} + if r.ditClassifier == nil { + return kb + } + result, err := r.ditClassifier.ExtractPageType(body) + if err != nil { + return kb + } + kb["PageType"] = result.Type + if len(result.Forms) > 0 { + kb["Forms"] = result.Forms + } + return kb +} + func (r *Runner) testAndSet(k string) bool { // skip empty lines k = strings.TrimSpace(k) @@ -1105,9 +1121,13 @@ func (r *Runner) RunEnumeration() { } } - if r.options.OutputFilterErrorPage && resp.KnowledgeBase["PageType"] == "error" { - logFilteredErrorPage(r.options.OutputFilterErrorPagePath, resp.URL) - continue + if len(r.options.OutputFilterPageType) > 0 { + if pageType, ok := resp.KnowledgeBase["PageType"].(string); ok { + if stringsutil.EqualFoldAny(pageType, r.options.OutputFilterPageType...) { + logFilteredErrorPage(r.options.OutputFilterErrorPagePath, resp.URL) + continue + } + } } if r.options.FilterOutDuplicates && r.duplicate(&resp) { @@ -2616,10 +2636,7 @@ retry: ExtractRegex: extractRegex, ScreenshotBytes: screenshotBytes, HeadlessBody: headlessBody, - KnowledgeBase: map[string]interface{}{ - "PageType": r.pageTypeClassifier.Classify(respData), - "pHash": pHash, - }, + KnowledgeBase: r.classifyPage(respData, pHash), TechnologyDetails: technologyDetails, Resolvers: resolvers, RequestRaw: requestDump, From 0affea0721ce057a203fa0debfe112d0ffae8648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 16 Feb 2026 12:49:42 +0300 Subject: [PATCH 23/39] fix: bump golangci-lint-action to v7 for Go 1.25 support --- .github/workflows/build-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index f58ebd80..6dfc158c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -16,9 +16,9 @@ jobs: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v5 + uses: golangci/golangci-lint-action@v7 with: - version: latest + version: v2 args: --timeout 5m working-directory: . From 8d2e876623edba1502398ee293ff662bae427b29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 16 Feb 2026 12:50:27 +0300 Subject: [PATCH 24/39] fix: bump actions/setup-go to v5 for Go 1.25 support --- .github/workflows/functional-test.yml | 2 +- .github/workflows/release-binary.yml | 2 +- .github/workflows/release-test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 007e7ce0..24f2c686 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -16,7 +16,7 @@ jobs: os: [ubuntu-latest, windows-latest, macOS-latest] steps: - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version-file: 'go.mod' diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 999ea91f..1745bfac 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -16,7 +16,7 @@ jobs: fetch-depth: 0 - name: "Set up Go" - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version-file: 'go.mod' diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index bbdd4ef1..07e6ab74 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version-file: 'go.mod' From 38c67c6994fcaee5e3c89d528f4280f570ba0be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 16 Feb 2026 12:53:08 +0300 Subject: [PATCH 25/39] fix: use projectdiscovery/actions for Go setup and linting --- .github/workflows/build-test.yml | 7 +------ .github/workflows/functional-test.yml | 11 ++++------- .github/workflows/release-binary.yml | 19 ++++++++----------- .github/workflows/release-test.yml | 11 ++++------- 4 files changed, 17 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6dfc158c..11bda129 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -15,12 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v7 - with: - version: v2 - args: --timeout 5m - working-directory: . + - uses: projectdiscovery/actions/golangci-lint/v2@v1 build: name: Test Builds diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 24f2c686..f712ddda 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -7,7 +7,7 @@ on: - '**.mod' workflow_dispatch: -jobs: +jobs: functional: name: Functional Test runs-on: ${{ matrix.os }} @@ -15,13 +15,10 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] steps: - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - uses: projectdiscovery/actions/setup/go@v1 - name: Functional Tests run: | diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 1745bfac..b906129d 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -11,23 +11,20 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: "Set up Go" - uses: actions/setup-go@v5 + uses: actions/checkout@v4 with: - go-version-file: 'go.mod' - + fetch-depth: 0 + + - uses: projectdiscovery/actions/setup/go@v1 + - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v4 - with: + with: args: "release --clean" version: latest workdir: . - env: + env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" - DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" \ No newline at end of file + DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 07e6ab74..c163e9c7 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -12,15 +12,12 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" - uses: actions/checkout@v3 - with: + uses: actions/checkout@v4 + with: fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - + - uses: projectdiscovery/actions/setup/go@v1 + - name: release test uses: goreleaser/goreleaser-action@v4 with: From 8d99c3034fed66e9cf9cf83418880694203e994a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 16 Feb 2026 13:11:03 +0300 Subject: [PATCH 26/39] fix: suppress errcheck lint for deferred file close --- runner/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 69556a91..da379832 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -719,7 +719,7 @@ func (r *Runner) streamInput() (chan string, error) { gologger.Error().Msgf("Could not open input file '%s': %s\n", r.options.InputFile, err) return } - defer finput.Close() + defer finput.Close() //nolint:errcheck if err := format.Parse(finput, func(item string) bool { item = strings.TrimSpace(item) if r.options.SkipDedupe || r.testAndSet(item) { @@ -805,7 +805,7 @@ func (r *Runner) loadFromFormat(filePath string, format inputformats.Format) (nu if err != nil { return 0, err } - defer finput.Close() + defer finput.Close() //nolint:errcheck err = format.Parse(finput, func(target string) bool { target = strings.TrimSpace(target) From f4272cbc0f272cdc9921ed2159c18eda341348c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 16 Feb 2026 13:33:42 +0300 Subject: [PATCH 27/39] refactor: lazy init dit classifier, prefer headless body for classification --- runner/runner.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index da379832..7df5c1c8 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -430,11 +430,13 @@ func New(options *Options) (*Runner, error) { } runner.simHashes = gcache.New[uint64, struct{}](1000).ARC().Build() - ditClassifier, err := dit.New() - if err != nil { - gologger.Warning().Msgf("Could not initialize page classifier: %s", err) + if options.JSONOutput || options.CSVOutput || len(options.OutputFilterPageType) > 0 { + ditClassifier, err := dit.New() + if err != nil { + gologger.Warning().Msgf("Could not initialize page classifier: %s", err) + } + runner.ditClassifier = ditClassifier } - runner.ditClassifier = ditClassifier if options.SecretFile != "" { authProviderOpts := &authprovider.AuthProviderOptions{ @@ -652,12 +654,16 @@ func (r *Runner) duplicate(result *Result) bool { return false } -func (r *Runner) classifyPage(body string, pHash uint64) map[string]interface{} { +func (r *Runner) classifyPage(headlessBody, body string, pHash uint64) map[string]any { kb := map[string]any{"pHash": pHash} if r.ditClassifier == nil { return kb } - result, err := r.ditClassifier.ExtractPageType(body) + html := body + if headlessBody != "" { + html = headlessBody + } + result, err := r.ditClassifier.ExtractPageType(html) if err != nil { return kb } @@ -2636,7 +2642,7 @@ retry: ExtractRegex: extractRegex, ScreenshotBytes: screenshotBytes, HeadlessBody: headlessBody, - KnowledgeBase: r.classifyPage(respData, pHash), + KnowledgeBase: r.classifyPage(headlessBody, respData, pHash), TechnologyDetails: technologyDetails, Resolvers: resolvers, RequestRaw: requestDump, From b97e22b6ae788604faddfcd88ca8136e6ef1c6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 3 Mar 2026 20:47:50 +0300 Subject: [PATCH 28/39] fix: address PR review feedback - Add Deprecated comment to OutputFilterErrorPage field - Coerce PageType to string for safe type assertion --- runner/options.go | 3 ++- runner/runner.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/runner/options.go b/runner/options.go index af720514..96339510 100644 --- a/runner/options.go +++ b/runner/options.go @@ -201,7 +201,8 @@ type Options struct { OutputMatchStatusCode string OutputMatchContentLength string OutputFilterStatusCode string - OutputFilterErrorPage bool + // Deprecated: use OutputFilterPageType with "error" instead. + OutputFilterErrorPage bool OutputFilterPageType goflags.StringSlice FilterOutDuplicates bool OutputFilterContentLength string diff --git a/runner/runner.go b/runner/runner.go index 7df5c1c8..ad530ece 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -667,7 +667,7 @@ func (r *Runner) classifyPage(headlessBody, body string, pHash uint64) map[strin if err != nil { return kb } - kb["PageType"] = result.Type + kb["PageType"] = fmt.Sprint(result.Type) if len(result.Forms) > 0 { kb["Forms"] = result.Forms } From 0e3ceee5d053102cf0f9ddc263bf92fc3543277c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 3 Mar 2026 20:57:07 +0300 Subject: [PATCH 29/39] fix: use fmt.Fprintf instead of WriteString(fmt.Sprintf) in healthcheck --- runner/healthcheck.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/runner/healthcheck.go b/runner/healthcheck.go index ef79db7f..42b63c04 100644 --- a/runner/healthcheck.go +++ b/runner/healthcheck.go @@ -14,11 +14,11 @@ func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string { // RW permissions on config file cfgFilePath, _ := flagSet.GetConfigFilePath() var test strings.Builder - test.WriteString(fmt.Sprintf("Version: %s\n", Version)) - test.WriteString(fmt.Sprintf("Operative System: %s\n", runtime.GOOS)) - test.WriteString(fmt.Sprintf("Architecture: %s\n", runtime.GOARCH)) - test.WriteString(fmt.Sprintf("Go Version: %s\n", runtime.Version())) - test.WriteString(fmt.Sprintf("Compiler: %s\n", runtime.Compiler)) + fmt.Fprintf(&test, "Version: %s\n", Version) + fmt.Fprintf(&test, "Operative System: %s\n", runtime.GOOS) + fmt.Fprintf(&test, "Architecture: %s\n", runtime.GOARCH) + fmt.Fprintf(&test, "Go Version: %s\n", runtime.Version()) + fmt.Fprintf(&test, "Compiler: %s\n", runtime.Compiler) var testResult string ok, err := fileutil.IsReadable(cfgFilePath) @@ -30,7 +30,7 @@ func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string { if err != nil { testResult += fmt.Sprintf(" (%s)", err) } - test.WriteString(fmt.Sprintf("Config file \"%s\" Read => %s\n", cfgFilePath, testResult)) + fmt.Fprintf(&test, "Config file \"%s\" Read => %s\n", cfgFilePath, testResult) ok, err = fileutil.IsWriteable(cfgFilePath) if ok { testResult = "Ok" @@ -40,7 +40,7 @@ func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string { if err != nil { testResult += fmt.Sprintf(" (%s)", err) } - test.WriteString(fmt.Sprintf("Config file \"%s\" Write => %s\n", cfgFilePath, testResult)) + fmt.Fprintf(&test, "Config file \"%s\" Write => %s\n", cfgFilePath, testResult) c4, err := net.Dial("tcp4", "scanme.sh:80") if err == nil && c4 != nil { _ = c4.Close() @@ -49,7 +49,7 @@ func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string { if err != nil { testResult = fmt.Sprintf("Ko (%s)", err) } - test.WriteString(fmt.Sprintf("IPv4 connectivity to scanme.sh:80 => %s\n", testResult)) + fmt.Fprintf(&test, "IPv4 connectivity to scanme.sh:80 => %s\n", testResult) c6, err := net.Dial("tcp6", "scanme.sh:80") if err == nil && c6 != nil { _ = c6.Close() @@ -58,7 +58,7 @@ func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string { if err != nil { testResult = fmt.Sprintf("Ko (%s)", err) } - test.WriteString(fmt.Sprintf("IPv6 connectivity to scanme.sh:80 => %s\n", testResult)) + fmt.Fprintf(&test, "IPv6 connectivity to scanme.sh:80 => %s\n", testResult) return test.String() } From 30032e69bd4ddbd75c4c3244285bdce130f4a161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 3 Mar 2026 21:07:09 +0300 Subject: [PATCH 30/39] fix: suppress errcheck for deferred stmt.Close --- internal/db/mysql.go | 2 +- internal/db/postgres.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/db/mysql.go b/internal/db/mysql.go index 6ebc0011..d9d44b87 100644 --- a/internal/db/mysql.go +++ b/internal/db/mysql.go @@ -212,7 +212,7 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } - defer stmt.Close() + defer stmt.Close() //nolint:errcheck for _, r := range results { aJSON, _ := json.Marshal(r.A) diff --git a/internal/db/postgres.go b/internal/db/postgres.go index b53d5aaf..c451583d 100644 --- a/internal/db/postgres.go +++ b/internal/db/postgres.go @@ -220,7 +220,7 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } - defer stmt.Close() + defer stmt.Close() //nolint:errcheck for _, r := range results { hashJSON, _ := json.Marshal(r.Hashes) From 579fa0547e851f0c6c30ec5dba184fae97f85ddb Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 4 Mar 2026 12:35:05 +0100 Subject: [PATCH 31/39] using explicit syntax --- internal/db/mysql.go | 4 +++- internal/db/postgres.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/db/mysql.go b/internal/db/mysql.go index d9d44b87..96c2f391 100644 --- a/internal/db/mysql.go +++ b/internal/db/mysql.go @@ -212,7 +212,9 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } - defer stmt.Close() //nolint:errcheck + defer func() { + _ = stmt.Close() + }() for _, r := range results { aJSON, _ := json.Marshal(r.A) diff --git a/internal/db/postgres.go b/internal/db/postgres.go index c451583d..0eca9831 100644 --- a/internal/db/postgres.go +++ b/internal/db/postgres.go @@ -220,7 +220,9 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } - defer stmt.Close() //nolint:errcheck + defer func() { + _ = stmt.Close() + }() for _, r := range results { hashJSON, _ := json.Marshal(r.Hashes) From 39ab027953e4053a958172e324bc53a1a39437ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Mar 2026 12:07:50 +0000 Subject: [PATCH 32/39] chore(deps): bump filippo.io/edwards25519 from 1.1.0 to 1.1.1 Bumps [filippo.io/edwards25519](https://github.com/FiloSottile/edwards25519) from 1.1.0 to 1.1.1. - [Commits](https://github.com/FiloSottile/edwards25519/compare/v1.1.0...v1.1.1) --- updated-dependencies: - dependency-name: filippo.io/edwards25519 dependency-version: 1.1.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2edf3e32..515f45c5 100644 --- a/go.mod +++ b/go.mod @@ -52,20 +52,20 @@ require ( require ( github.com/dustin/go-humanize v1.0.1 github.com/go-sql-driver/mysql v1.9.3 - github.com/lib/pq v1.10.9 - go.mongodb.org/mongo-driver v1.17.6 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/happyhackingspace/dit v0.0.9 + github.com/lib/pq v1.10.9 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 github.com/seh-msft/burpxml v1.0.1 github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 + go.mongodb.org/mongo-driver v1.17.6 gopkg.in/yaml.v3 v3.0.1 ) require ( aead.dev/minisign v0.2.0 // indirect - filippo.io/edwards25519 v1.1.0 // indirect + filippo.io/edwards25519 v1.1.1 // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect diff --git a/go.sum b/go.sum index 97966ff8..622acbaf 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= +filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= From c4ee6f1cd154db6c5e918f835b1a8b4d08958665 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Mar 2026 12:07:50 +0000 Subject: [PATCH 33/39] chore(deps): bump golang.org/x/net from 0.50.0 to 0.51.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.50.0 to 0.51.0. - [Commits](https://github.com/golang/net/compare/v0.50.0...v0.51.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-version: 0.51.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2edf3e32..9de59c13 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( go.etcd.io/bbolt v1.4.0 // indirect go.uber.org/multierr v1.11.0 golang.org/x/exp v0.0.0-20250911091902-df9299821621 - golang.org/x/net v0.50.0 + golang.org/x/net v0.51.0 golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 ) @@ -52,14 +52,14 @@ require ( require ( github.com/dustin/go-humanize v1.0.1 github.com/go-sql-driver/mysql v1.9.3 - github.com/lib/pq v1.10.9 - go.mongodb.org/mongo-driver v1.17.6 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/happyhackingspace/dit v0.0.9 + github.com/lib/pq v1.10.9 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 github.com/seh-msft/burpxml v1.0.1 github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 + go.mongodb.org/mongo-driver v1.17.6 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 97966ff8..bcacdc75 100644 --- a/go.sum +++ b/go.sum @@ -602,8 +602,8 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From 91c489e75dfbc4e41950eada184f3dba5a25fcad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 5 Mar 2026 14:12:46 +0300 Subject: [PATCH 34/39] bump version --- runner/banner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/banner.go b/runner/banner.go index f3018ab4..c87eb42d 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -16,7 +16,7 @@ const banner = ` ` // Version is the current Version of httpx -const Version = `v1.8.1` +const Version = `v1.9.0` // showBanner is used to show the banner to the user func showBanner() { From 6243439c9401b2d6f5f449d2dc2a4ddd8420d3ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 10:55:30 +0000 Subject: [PATCH 35/39] chore(deps): bump the modules group with 2 updates Bumps the modules group with 2 updates: [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) and [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo). Updates `github.com/projectdiscovery/cdncheck` from 1.2.23 to 1.2.26 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.2.23...v1.2.26) Updates `github.com/projectdiscovery/wappalyzergo` from 0.2.68 to 0.2.71 - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.2.68...v0.2.71) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.2.26 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-version: 0.2.71 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 51e9f363..3d1fb9fa 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/miekg/dns v1.1.68 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 - github.com/projectdiscovery/cdncheck v1.2.23 + github.com/projectdiscovery/cdncheck v1.2.26 github.com/projectdiscovery/clistats v0.1.1 github.com/projectdiscovery/dsl v0.8.13 github.com/projectdiscovery/fastdialer v0.5.4 @@ -36,7 +36,7 @@ require ( github.com/projectdiscovery/tlsx v1.2.2 github.com/projectdiscovery/useragent v0.0.107 github.com/projectdiscovery/utils v0.9.0 - github.com/projectdiscovery/wappalyzergo v0.2.68 + github.com/projectdiscovery/wappalyzergo v0.2.71 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index 29a484be..cfa2adf4 100644 --- a/go.sum +++ b/go.sum @@ -326,8 +326,8 @@ github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.2.23 h1:PTXUEG3fJYuBSdi+7O76V/dQYd14TLn84lVABGMNhuU= -github.com/projectdiscovery/cdncheck v1.2.23/go.mod h1:Y1KQmACY+AifbuPX/W7o8lWssiWmAZ5d/KG8qkmFm9I= +github.com/projectdiscovery/cdncheck v1.2.26 h1:0iLVppSfXDHWu/jPlDJGTsyX+qziQgO34qjctkXfGyc= +github.com/projectdiscovery/cdncheck v1.2.26/go.mod h1:Y1KQmACY+AifbuPX/W7o8lWssiWmAZ5d/KG8qkmFm9I= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= github.com/projectdiscovery/dsl v0.8.13 h1:HjjHta7c02saH2tUGs8CN5vDeE2MyWvCV32koT8ZCWs= @@ -370,8 +370,8 @@ github.com/projectdiscovery/useragent v0.0.107 h1:45gSBda052fv2Gtxtnpx7cu2rWtUpZ github.com/projectdiscovery/useragent v0.0.107/go.mod h1:yv5ZZLDT/kq6P+NvBcCPq6sjEVQtZGgO+OvvHzZ+WtY= github.com/projectdiscovery/utils v0.9.0 h1:eu9vdbP0VYXI9nGSLfnOpUqBeW9/B/iSli7U8gPKZw8= github.com/projectdiscovery/utils v0.9.0/go.mod h1:zcVu1QTlMi5763qCol/L3ROnbd/UPSBP8fI5PmcnF6s= -github.com/projectdiscovery/wappalyzergo v0.2.68 h1:bN/SIfzsZ0Q0H5m4AeiuB65AbJu9oOjSMqJ10YjvsHI= -github.com/projectdiscovery/wappalyzergo v0.2.68/go.mod h1:Oc+U2RPJObmpi6LW5lTMEDiKagcKZNkEfZfwrVMURa0= +github.com/projectdiscovery/wappalyzergo v0.2.71 h1:MdENrw/8a1qrxjqIJGbFktDiqVLeaMq7AEIJPMO0JGY= +github.com/projectdiscovery/wappalyzergo v0.2.71/go.mod h1:Oc+U2RPJObmpi6LW5lTMEDiKagcKZNkEfZfwrVMURa0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= From d5737fbebdb513870f047c4b2e1677faf420d6a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 10:55:37 +0000 Subject: [PATCH 36/39] chore(deps): bump github.com/lib/pq from 1.10.9 to 1.11.2 Bumps [github.com/lib/pq](https://github.com/lib/pq) from 1.10.9 to 1.11.2. - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.10.9...v1.11.2) --- updated-dependencies: - dependency-name: github.com/lib/pq dependency-version: 1.11.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 51e9f363..c744e7bc 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/happyhackingspace/dit v0.0.9 - github.com/lib/pq v1.10.9 + github.com/lib/pq v1.11.2 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 github.com/seh-msft/burpxml v1.0.1 github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 diff --git a/go.sum b/go.sum index 29a484be..8d559a68 100644 --- a/go.sum +++ b/go.sum @@ -250,8 +250,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= +github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= From e4aff87efe2b0e62a4f12bb2074a9e6197336e21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 10:55:43 +0000 Subject: [PATCH 37/39] chore(deps): bump go.mongodb.org/mongo-driver from 1.17.6 to 1.17.9 Bumps [go.mongodb.org/mongo-driver](https://github.com/mongodb/mongo-go-driver) from 1.17.6 to 1.17.9. - [Release notes](https://github.com/mongodb/mongo-go-driver/releases) - [Commits](https://github.com/mongodb/mongo-go-driver/compare/v1.17.6...v1.17.9) --- updated-dependencies: - dependency-name: go.mongodb.org/mongo-driver dependency-version: 1.17.9 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 51e9f363..acd56094 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 github.com/seh-msft/burpxml v1.0.1 github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 - go.mongodb.org/mongo-driver v1.17.6 + go.mongodb.org/mongo-driver v1.17.9 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 29a484be..86a24897 100644 --- a/go.sum +++ b/go.sum @@ -508,8 +508,8 @@ github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77/go.mod h1:aSvf+uTU222 github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= -go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU= +go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= From 9be814d5af7d67d38702fb8b699d7490dfa231ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 21:02:20 +0000 Subject: [PATCH 38/39] chore(deps): bump github.com/weppos/publicsuffix-go Bumps [github.com/weppos/publicsuffix-go](https://github.com/weppos/publicsuffix-go) from 0.50.3-0.20260104170930-90713dec78f2 to 0.50.3. - [Changelog](https://github.com/weppos/publicsuffix-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/weppos/publicsuffix-go/commits/v0.50.3) --- updated-dependencies: - dependency-name: github.com/weppos/publicsuffix-go dependency-version: 0.50.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 083779e4..38f22291 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( github.com/lib/pq v1.11.2 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 github.com/seh-msft/burpxml v1.0.1 - github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 + github.com/weppos/publicsuffix-go v0.50.3 go.mongodb.org/mongo-driver v1.17.9 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 9f62e7a9..fcf24857 100644 --- a/go.sum +++ b/go.sum @@ -458,8 +458,8 @@ github.com/vulncheck-oss/go-exploit v1.51.0 h1:HTmJ4Q94tbEDPb35mQZn6qMg4rT+Sw9n+ github.com/vulncheck-oss/go-exploit v1.51.0/go.mod h1:J28w0dLnA6DnCrnBm9Sbt6smX8lvztnnN2wCXy7No6c= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.40.2/go.mod h1:XsLZnULC3EJ1Gvk9GVjuCTZ8QUu9ufE4TZpOizDShko= -github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 h1:LiQSn5u8Nc6V/GixI+SWxt+YkNIyfKIlkVRULSw2Zt0= -github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2/go.mod h1:CbQCKDtXF8UcT7hrxeMa0MDjwhpOI9iYOU7cfq+yo8k= +github.com/weppos/publicsuffix-go v0.50.3 h1:eT5dcjHQcVDNc0igpFEsGHKIip30feuB2zuuI9eJxiE= +github.com/weppos/publicsuffix-go v0.50.3/go.mod h1:/rOa781xBykZhHK/I3QeHo92qdDKVmKZKF7s8qAEM/4= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= From 358457f59741660d9cb81c44e78bbdcbf7aaa231 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 21:02:24 +0000 Subject: [PATCH 39/39] chore(deps): bump github.com/happyhackingspace/dit from 0.0.9 to 0.0.14 Bumps [github.com/happyhackingspace/dit](https://github.com/happyhackingspace/dit) from 0.0.9 to 0.0.14. - [Release notes](https://github.com/happyhackingspace/dit/releases) - [Commits](https://github.com/happyhackingspace/dit/compare/v0.0.9...v0.0.14) --- updated-dependencies: - dependency-name: github.com/happyhackingspace/dit dependency-version: 0.0.14 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 083779e4..5f3e63e2 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/go-sql-driver/mysql v1.9.3 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 - github.com/happyhackingspace/dit v0.0.9 + github.com/happyhackingspace/dit v0.0.14 github.com/lib/pq v1.11.2 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index 9f62e7a9..4275d2b2 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo= github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= -github.com/happyhackingspace/dit v0.0.9 h1:8Owu16jksGExJQdMODAHuFN6k0FTl0ZAlPTjv7X8dtY= -github.com/happyhackingspace/dit v0.0.9/go.mod h1:lGjPkdt8Xdmge1QeiWAoQpUhDTLNmKLniKRSCFGgjuY= +github.com/happyhackingspace/dit v0.0.14 h1:rkIu0HuFqvqr8F2PJgG0F+lx6DbX/tQE1hXKwIF2NQQ= +github.com/happyhackingspace/dit v0.0.14/go.mod h1:+WeAxrX7QYeiDmXLVaDgrqpyfD4O/sHlOL4wtbiIpUQ= github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=