diff --git a/.gitignore b/.gitignore index 10269185b7..67beffa25b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ pkg/protocols/headless/engine/.cache /bindgen /jsdocgen /scrapefuncs +/integration_tests/.cache/ +/integration_tests/.nuclei-config/ +/*.yaml \ No newline at end of file diff --git a/cmd/integration-test/flow.go b/cmd/integration-test/flow.go index 097a1c5140..334f7756fe 100644 --- a/cmd/integration-test/flow.go +++ b/cmd/integration-test/flow.go @@ -67,7 +67,7 @@ func (t *iterateValuesFlow) Execute(filePath string) error { if err != nil { return err } - return expectResultsCount(results, 1) + return expectResultsCount(results, 2) } type dnsNsProbe struct{} @@ -77,7 +77,7 @@ func (t *dnsNsProbe) Execute(filePath string) error { if err != nil { return err } - return expectResultsCount(results, 1) + return expectResultsCount(results, 2) } func getBase64(input string) string { diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index de3f9c15bf..6b81d9a3b1 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "encoding/json" "errors" "fmt" @@ -21,7 +20,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/testutils" "github.com/projectdiscovery/retryablehttp-go" errorutil "github.com/projectdiscovery/utils/errors" - fileutil "github.com/projectdiscovery/utils/file" logutil "github.com/projectdiscovery/utils/log" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" @@ -77,7 +75,6 @@ var httpTestcases = []TestCaseInfo{ {Path: "protocols/http/get-without-scheme.yaml", TestCase: &httpGetWithoutScheme{}}, {Path: "protocols/http/cl-body-without-header.yaml", TestCase: &httpCLBodyWithoutHeader{}}, {Path: "protocols/http/cl-body-with-header.yaml", TestCase: &httpCLBodyWithHeader{}}, - {Path: "protocols/http/save-extractor-values-to-file.yaml", TestCase: &httpSaveExtractorValuesToFile{}}, {Path: "protocols/http/cli-with-constants.yaml", TestCase: &ConstantWithCliVar{}}, {Path: "protocols/http/matcher-status.yaml", TestCase: &matcherStatusTest{}}, {Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}}, @@ -1382,34 +1379,6 @@ func (h *httpCLBodyWithHeader) Execute(filePath string) error { return expectResultsCount(got, 1) } -type httpSaveExtractorValuesToFile struct{} - -func (h *httpSaveExtractorValuesToFile) Execute(filePath string) error { - router := httprouter.New() - router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - var buff bytes.Buffer - for i := 0; i < 10; i++ { - buff.WriteString(fmt.Sprintf(`"value": %v`+"\n", i)) - } - _, _ = w.Write(buff.Bytes()) - }) - ts := httptest.NewServer(router) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) - if err != nil { - return err - } - - // remove output.txt file if exists - if !fileutil.FileExists("output.txt") { - return fmt.Errorf("extractor output file output.txt file does not exist") - } else { - _ = os.Remove("output.txt") - } - return expectResultsCount(results, 1) -} - // constant shouldn't be overwritten by cli var with same name type ConstantWithCliVar struct{} diff --git a/go.mod b/go.mod index 12eeb5f1b9..ccb6723c85 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.49 - github.com/projectdiscovery/hmap v0.0.29 + github.com/projectdiscovery/fastdialer v0.0.52 + github.com/projectdiscovery/hmap v0.0.33 github.com/projectdiscovery/interactsh v1.1.8 - github.com/projectdiscovery/rawhttp v0.1.28 - github.com/projectdiscovery/retryabledns v1.0.47 - github.com/projectdiscovery/retryablehttp-go v1.0.38 + github.com/projectdiscovery/rawhttp v0.1.33 + github.com/projectdiscovery/retryabledns v1.0.50 + github.com/projectdiscovery/retryablehttp-go v1.0.42 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -39,7 +39,7 @@ require ( github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.18.0 + golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.11.0 golang.org/x/text v0.14.0 gopkg.in/yaml.v2 v2.4.0 @@ -65,7 +65,7 @@ require ( github.com/docker/go-units v0.5.0 github.com/dop251/goja v0.0.0-20230828202809-3dbe69dd2b8e github.com/fatih/structs v1.1.0 - github.com/go-git/go-git/v5 v5.7.0 + github.com/go-git/go-git/v5 v5.11.0 github.com/go-ldap/ldap/v3 v3.4.5 github.com/go-pg/pg v8.0.7+incompatible github.com/go-sql-driver/mysql v1.6.0 @@ -76,21 +76,21 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/ory/dockertest/v3 v3.10.0 github.com/praetorian-inc/fingerprintx v1.1.9 - github.com/projectdiscovery/dsl v0.0.35 + github.com/projectdiscovery/dsl v0.0.38 github.com/projectdiscovery/fasttemplate v0.0.2 - github.com/projectdiscovery/goflags v0.1.29 - github.com/projectdiscovery/gologger v1.1.11 + github.com/projectdiscovery/goflags v0.1.34 + github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.1 github.com/projectdiscovery/httpx v1.3.7 github.com/projectdiscovery/mapcidr v1.1.16 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 - github.com/projectdiscovery/ratelimit v0.0.20 + github.com/projectdiscovery/ratelimit v0.0.23 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6-0.20231116215000-e842dc367a74 github.com/projectdiscovery/uncover v1.0.7 - github.com/projectdiscovery/utils v0.0.68 + github.com/projectdiscovery/utils v0.0.72 github.com/projectdiscovery/wappalyzergo v0.0.109 github.com/redis/go-redis/v9 v9.1.0 github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02 @@ -104,6 +104,7 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect + dario.cat/mergo v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect @@ -128,6 +129,7 @@ require ( github.com/cloudflare/cfssl v1.6.4 // indirect github.com/cloudflare/circl v1.3.6 // indirect github.com/containerd/continuity v0.4.2 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.10.0 // indirect @@ -163,9 +165,11 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kataras/jwt v0.1.10 // indirect github.com/klauspost/compress v1.17.3 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mackerelio/go-osstat v0.2.4 // indirect + github.com/mholt/archiver/v3 v3.5.1 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -176,6 +180,7 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.9 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/projectdiscovery/asnmap v1.0.6 // indirect @@ -185,7 +190,7 @@ require ( github.com/refraction-networking/utls v1.5.4 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/skeema/knownhosts v1.1.1 // indirect + github.com/skeema/knownhosts v1.2.1 // indirect github.com/smartystreets/assertions v1.0.0 // indirect github.com/tidwall/btree v1.7.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect @@ -226,7 +231,7 @@ require ( github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/dsnet/compress v0.0.1 // indirect + github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -259,7 +264,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/networkpolicy v0.0.6 + github.com/projectdiscovery/networkpolicy v0.0.7 github.com/rivo/uniseg v0.4.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect @@ -292,8 +297,7 @@ require ( require ( github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect - github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect github.com/alecthomas/chroma v0.10.0 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect @@ -307,7 +311,7 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562 github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.4.1 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect diff --git a/go.sum b/go.sum index a33cbad317..ffd9251729 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= @@ -75,8 +77,8 @@ github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd/go.mod h1:A github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= -github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek= -github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= +github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= +github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= @@ -85,8 +87,6 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= @@ -107,6 +107,7 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= @@ -196,6 +197,7 @@ github.com/bsm/ginkgo/v2 v2.9.5/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbA github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= @@ -228,6 +230,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.6 h1:/xbKIqSHbZXHwkhbrhrt2YOHIwYJlXH94E3tI/gDlUg= github.com/cloudflare/circl v1.3.6/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -247,6 +250,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -284,8 +289,8 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562 h1:ObbB2tzHWWAxzsG5futqeq2Ual2zYlo/+eMkSc5sn8w= github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562/go.mod h1:X2TOTJ+Uamd454RFp7ig2tmP3hQg0Z2Qk8gbVQmU0mk= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -293,8 +298,8 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= -github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= @@ -336,12 +341,12 @@ github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= -github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= -github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE= -github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= +github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -440,6 +445,7 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -458,8 +464,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= @@ -605,6 +612,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -612,6 +620,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -650,8 +660,6 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= -github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -673,6 +681,8 @@ github.com/mholt/acmez v1.2.0 h1:1hhLxSgY5FvH5HCnGUuwbKY2VQVo8IU7rxXKSnZ7F30= github.com/mholt/acmez v1.2.0/go.mod h1:VT9YwH1xgNX1kmYY89gY8xPJC84BFAisjo8Egigt4kE= github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= +github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= +github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= @@ -721,7 +731,7 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= @@ -738,8 +748,8 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -768,6 +778,8 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= +github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= @@ -793,24 +805,24 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4= github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.0.35 h1:kj+yVotGDweY+OGX2UUpBVIyfYpd7ADWatWP9pe7rxE= -github.com/projectdiscovery/dsl v0.0.35/go.mod h1:IT6OlBEW+7yJl8F77GXfwDM9zZpb4sOj5IEAzdmJBPE= -github.com/projectdiscovery/fastdialer v0.0.49 h1:YJ2EDSklvcq6putHko49+0RNKZKAIGwTKY5zGhQC/tE= -github.com/projectdiscovery/fastdialer v0.0.49/go.mod h1:GwdxQhD65npOhDuKLhHxvZ6I/HqqnMOrC450Q/wUuYo= +github.com/projectdiscovery/dsl v0.0.38 h1:wzObLZ4m4fMtPD0RiirnAp5naF43dJymjCmEeZzJLjM= +github.com/projectdiscovery/dsl v0.0.38/go.mod h1:jYaosMHbna8jGxhClWovGFBNJGM19Go0wbk6FfrV/PA= +github.com/projectdiscovery/fastdialer v0.0.52 h1:K7EjNm/u79B2pAK+UAEjPf6nd6KSsN78S7Il8XcxpK8= +github.com/projectdiscovery/fastdialer v0.0.52/go.mod h1:aLhrsv+PyfuB5/Jm09cuplIXawNtLSXBJM0bFIkhsz4= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= -github.com/projectdiscovery/goflags v0.1.29 h1:RIfBxJrm9ApJLKxizNm9vZq3/8uH7WBHyInXC4GjNSY= -github.com/projectdiscovery/goflags v0.1.29/go.mod h1:JuNseKacVYPqxwhN7NLw/UPrzyZOuJ58ufLrPuZsXDY= -github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= -github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= +github.com/projectdiscovery/goflags v0.1.34 h1:s64GffNF7lsdMz4V4ZS2vfo+x+FxaMB0/TpqZC2q6FI= +github.com/projectdiscovery/goflags v0.1.34/go.mod h1:Br+lowrSpQuQOwDon5w1ZLq9BPJvxZhBUpUTc98UXZs= +github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= +github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.1 h1:f08ZnYlbDZV/TNGDvIXV9s/oB/sAI+HWaSbW4em4aKM= github.com/projectdiscovery/gozero v0.0.1/go.mod h1:/dHwbly+1lhOX9UreVure4lEe7K4hIHeu/c/wZGNTDo= -github.com/projectdiscovery/hmap v0.0.29 h1:YPnwrvyeF0jxWloRq45rG3GZrUTu5gZYizEIn2EVp50= -github.com/projectdiscovery/hmap v0.0.29/go.mod h1:08/3+VcqLg9W/RViplkbIiYXDNltCDlF+P7oCQblprA= +github.com/projectdiscovery/hmap v0.0.33 h1:kDkw4xVE8uyko6Cv3Cd9MZsHByn9BtXK3y7PeLKVBs4= +github.com/projectdiscovery/hmap v0.0.33/go.mod h1:IlKSbnFKtn68STLiNwc5Kbu4GaR6aIsGaHbpFOYNFGY= github.com/projectdiscovery/httpx v1.3.7 h1:g/ZQIBdWWPQLF+niv39b7jRhAkyrcroJJfqbTQDKhyQ= github.com/projectdiscovery/httpx v1.3.7/go.mod h1:FqEmL2zWZArgD1vSQ+tqHvmUItPqxYhOgKyfN8GyWMQ= github.com/projectdiscovery/interactsh v1.1.8 h1:mDD+f/oo2tV4Z1WyUync0tgYeJyuiS89Un64Gm6Pvgk= @@ -819,18 +831,18 @@ github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclg github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q= github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8= github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc= -github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= -github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= -github.com/projectdiscovery/ratelimit v0.0.20 h1:xKvCZyXZn7YzPtmw+Cvv7JiNxH9mp8CW2fsI39cZBhY= -github.com/projectdiscovery/ratelimit v0.0.20/go.mod h1:Ddd6DaiibjUDAzUnYeRGA/xafYJNLlv/dmuSdc5Lvrw= -github.com/projectdiscovery/rawhttp v0.1.28 h1:6cR6JpjzEMjtyXHOWKwfFUNdmo0CXtUbOn6w6RsBYf4= -github.com/projectdiscovery/rawhttp v0.1.28/go.mod h1:VfGWfefvtSzixCdsst+gMRYVMMnOvrLieW1l9xDdO0U= +github.com/projectdiscovery/networkpolicy v0.0.7 h1:AwHqBRXBqDQgnWzBMuoJtHBNEYBw+NFp/4qIK688x7o= +github.com/projectdiscovery/networkpolicy v0.0.7/go.mod h1:CK0CnFoLF1Nou6mY7P4WODSAxhPN8g8g7XpapgEP8tI= +github.com/projectdiscovery/ratelimit v0.0.23 h1:Fz2A57UW6GK0L0huOGVXd97EhASrJV41SC1NrGImShU= +github.com/projectdiscovery/ratelimit v0.0.23/go.mod h1:042iuvdggjUnsgAIzyxM3iLFveMaXnGTRwlCpfd03I0= +github.com/projectdiscovery/rawhttp v0.1.33 h1:H+QM7zVq/pprGrQrO0JyCSRwrCYdIHbQ3hueh6P5u/4= +github.com/projectdiscovery/rawhttp v0.1.33/go.mod h1:3W9ZZr3feIVbK0VQHZv6c5OKqb+tnTdh3cbNOOO8AY8= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.47 h1:9h/9H2YWg68mampF2re5CCQ2lAxBLzq/N0daN6kdUTQ= -github.com/projectdiscovery/retryabledns v1.0.47/go.mod h1:TRErBaFkW2Gx73EepHeYSLgtD2ZCbNj1dTS9SuLd5GA= -github.com/projectdiscovery/retryablehttp-go v1.0.38 h1:MvXTiqL58+HKNL0fxvGEXUWuTrYENtrNklxxOIGFh0o= -github.com/projectdiscovery/retryablehttp-go v1.0.38/go.mod h1:vTDxZuLKrAots/HUBM6g+E5tc64XaFCTOpxPXp+F12E= +github.com/projectdiscovery/retryabledns v1.0.50 h1:0nM3x29G5LAZ7urfl0jSs501RQ5q57SkPwkdY19ECn8= +github.com/projectdiscovery/retryabledns v1.0.50/go.mod h1:CbQhVC9JjtqU/89gz25gs6UgpQKYwFN2RoWoW5a/M9Q= +github.com/projectdiscovery/retryablehttp-go v1.0.42 h1:NW76U/r0pWNi6iudBqggG69sN8aguuXLLbGRkLvniyo= +github.com/projectdiscovery/retryablehttp-go v1.0.42/go.mod h1:NWR4amTNHwM+ALk1QL1HiyzhFejRTMCHapM+oSoNSv8= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -839,8 +851,8 @@ github.com/projectdiscovery/tlsx v1.1.6-0.20231116215000-e842dc367a74 h1:G0gw+3z github.com/projectdiscovery/tlsx v1.1.6-0.20231116215000-e842dc367a74/go.mod h1:YH8el7/6pyZbNed1IibjzbGpeigiCVyvE28g5+LsPAw= github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A= github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= -github.com/projectdiscovery/utils v0.0.68 h1:rWvuG61oWeNzboYtugc3sG2uw5k8uptfHoth4CypVQI= -github.com/projectdiscovery/utils v0.0.68/go.mod h1:c5XnwkcffXqma9Hf781Osekfuqehb981gdlQiBZ5QvU= +github.com/projectdiscovery/utils v0.0.72 h1:sJ1lBcaWO6dJ65F+fVhSJbguhgWjixgy9mjj7jKBUW8= +github.com/projectdiscovery/utils v0.0.72/go.mod h1:VPnijH51D8wB1VJiEujUp7UZ+TUTKN8PpoW82nivUVY= github.com/projectdiscovery/wappalyzergo v0.0.109 h1:BERfwTRn1dvB1tbhyc5m67R8VkC9zbVuPsEq4VEm07k= github.com/projectdiscovery/wappalyzergo v0.0.109/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -893,8 +905,8 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02 h1:Nk74A6E84pynxLN74hIrQ7Q3cS0/0L5I7coOLNSFAMs= github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02/go.mod h1:OGEfzIZJs5m/VgAb1BvWR8fH17RTQWx84HTB1koGf9s= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -927,8 +939,8 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE= -github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= +github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= +github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -1004,7 +1016,8 @@ github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= @@ -1135,6 +1148,7 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= @@ -1224,6 +1238,7 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -1231,8 +1246,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1339,6 +1354,7 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= @@ -1356,6 +1372,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= diff --git a/integration_tests/flow/conditional-flow-negative.yaml b/integration_tests/flow/conditional-flow-negative.yaml index d1e2cbf9d2..dd76c89b54 100644 --- a/integration_tests/flow/conditional-flow-negative.yaml +++ b/integration_tests/flow/conditional-flow-negative.yaml @@ -15,6 +15,7 @@ dns: - type: word words: - "ghost.io" + internal: true http: - method: GET diff --git a/integration_tests/flow/conditional-flow.yaml b/integration_tests/flow/conditional-flow.yaml index d1e2cbf9d2..dd76c89b54 100644 --- a/integration_tests/flow/conditional-flow.yaml +++ b/integration_tests/flow/conditional-flow.yaml @@ -15,6 +15,7 @@ dns: - type: word words: - "ghost.io" + internal: true http: - method: GET diff --git a/integration_tests/flow/dns-ns-probe.yaml b/integration_tests/flow/dns-ns-probe.yaml index 569a9e766c..ef88e6dd92 100644 --- a/integration_tests/flow/dns-ns-probe.yaml +++ b/integration_tests/flow/dns-ns-probe.yaml @@ -22,6 +22,7 @@ dns: - type: word words: - "IN\tNS" + internal: true extractors: - type: regex internal: true diff --git a/integration_tests/flow/flow-hide-matcher.yaml b/integration_tests/flow/flow-hide-matcher.yaml index f8ffc27184..98bbbdf339 100644 --- a/integration_tests/flow/flow-hide-matcher.yaml +++ b/integration_tests/flow/flow-hide-matcher.yaml @@ -1,10 +1,10 @@ id: flow-hide-matcher info: - name: Test HTTP Template + name: Test Flow Hide Matcher author: pdteam severity: info - description: In flow matcher output of previous step is hidden and only last event matcher output is shown + description: In Template any matcher can be marked as internal which hides it from the output. flow: http(1) && http(2) @@ -17,6 +17,7 @@ http: - type: word words: - ok + internal: true - method: GET path: diff --git a/integration_tests/flow/iterate-values-flow.yaml b/integration_tests/flow/iterate-values-flow.yaml index b92dee4a45..f8fd911757 100644 --- a/integration_tests/flow/iterate-values-flow.yaml +++ b/integration_tests/flow/iterate-values-flow.yaml @@ -21,9 +21,9 @@ http: extractors: - type: regex name: emails - internal: true regex: - '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' + internal: true - method: GET path: @@ -32,4 +32,10 @@ http: matchers: - type: word words: - - "Welcome" \ No newline at end of file + - "Welcome" + + extractors: + - type: dsl + name: email + dsl: + - email \ No newline at end of file diff --git a/integration_tests/protocols/http/save-extractor-values-to-file.yaml b/integration_tests/protocols/http/save-extractor-values-to-file.yaml deleted file mode 100644 index 64bf63f40d..0000000000 --- a/integration_tests/protocols/http/save-extractor-values-to-file.yaml +++ /dev/null @@ -1,18 +0,0 @@ -id: save-extractor-values-to-file - -info: - name: save extractor values to file - author: pdteam - severity: info - -requests: - - method: GET - path: - - "{{BaseURL}}" - - extractors: - - type: regex - part: body - regex: - - '[0-9]+' - to: output.txt \ No newline at end of file diff --git a/lib/config.go b/lib/config.go index f24e272fc2..3f3e21ae22 100644 --- a/lib/config.go +++ b/lib/config.go @@ -227,7 +227,7 @@ type NetworkConfig struct { TrackError []string // Adds given errors to max host error watchlist DisableMaxHostErr bool // Disable max host error optimization (Hosts are not skipped even if they are not responding) Interface string // Interface to use for network scan - + SourceIP string // SourceIP sets custom source IP address for network requests } // WithNetworkConfig allows setting network config options @@ -241,6 +241,7 @@ func WithNetworkConfig(opts NetworkConfig) NucleiSDKOptions { e.opts.LeaveDefaultPorts = opts.LeaveDefaultPorts e.hostErrCache = hosterrorscache.New(opts.MaxHostError, hosterrorscache.DefaultMaxHostsCount, opts.TrackError) e.opts.Interface = opts.Interface + e.opts.SourceIP = opts.SourceIP return nil } } diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index 9a26137f25..fe7bd2b9f7 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.1.3` + Version = `v3.1.4` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/pkg/core/inputs/hybrid/hmap.go b/pkg/core/inputs/hybrid/hmap.go index c5a6341aed..bbc675dc6b 100644 --- a/pkg/core/inputs/hybrid/hmap.go +++ b/pkg/core/inputs/hybrid/hmap.go @@ -18,12 +18,12 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/filekv" "github.com/projectdiscovery/hmap/store/hybrid" - "github.com/projectdiscovery/mapcidr" "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/utils/expand" uncoverlib "github.com/projectdiscovery/uncover" fileutil "github.com/projectdiscovery/utils/file" iputil "github.com/projectdiscovery/utils/ip" @@ -121,10 +121,10 @@ func (i *Input) initializeInputSources(opts *Options) error { for _, target := range options.Targets { switch { case iputil.IsCIDR(target): - ips := i.expandCIDRInputValue(target) + ips := expand.CIDR(target) i.addTargets(ips) case asn.IsASN(target): - ips := i.expandASNInputValue(target) + ips := expand.ASN(target) i.addTargets(ips) default: i.Set(target) @@ -174,10 +174,10 @@ func (i *Input) initializeInputSources(opts *Options) error { for _, target := range options.ExcludeTargets { switch { case iputil.IsCIDR(target): - ips := i.expandCIDRInputValue(target) + ips := expand.CIDR(target) i.removeTargets(ips) case asn.IsASN(target): - ips := i.expandASNInputValue(target) + ips := expand.ASN(target) i.removeTargets(ips) default: i.Del(target) @@ -195,10 +195,10 @@ func (i *Input) scanInputFromReader(reader io.Reader) { item := scanner.Text() switch { case iputil.IsCIDR(item): - ips := i.expandCIDRInputValue(item) + ips := expand.CIDR(item) i.addTargets(ips) case asn.IsASN(item): - ips := i.expandASNInputValue(item) + ips := expand.ASN(item) i.addTargets(ips) default: i.Set(item) @@ -489,26 +489,6 @@ func (i *Input) Scan(callback func(value *contextargs.MetaInput) bool) { } } -// expandCIDRInputValue expands CIDR and stores expanded IPs -func (i *Input) expandCIDRInputValue(value string) []string { - var ips []string - ipsCh, _ := mapcidr.IPAddressesAsStream(value) - for ip := range ipsCh { - ips = append(ips, ip) - } - return ips -} - -// expandASNInputValue expands CIDRs for given ASN and stores expanded IPs -func (i *Input) expandASNInputValue(value string) []string { - var ips []string - cidrs, _ := asn.GetCIDRsForASNNum(value) - for _, cidr := range cidrs { - ips = append(ips, i.expandCIDRInputValue(cidr.String())...) - } - return ips -} - func (i *Input) addTargets(targets []string) { for _, target := range targets { i.Set(target) diff --git a/pkg/core/inputs/hybrid/hmap_test.go b/pkg/core/inputs/hybrid/hmap_test.go index 40d385841f..c11c6efdf6 100644 --- a/pkg/core/inputs/hybrid/hmap_test.go +++ b/pkg/core/inputs/hybrid/hmap_test.go @@ -12,10 +12,11 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/utils/expand" "github.com/stretchr/testify/require" ) -func Test_expandCIDRInputValue(t *testing.T) { +func Test_expandCIDR(t *testing.T) { tests := []struct { cidr string expected []string @@ -33,7 +34,7 @@ func Test_expandCIDRInputValue(t *testing.T) { require.Nil(t, err, "could not create temporary input file") input := &Input{hostMap: hm} - ips := input.expandCIDRInputValue(tt.cidr) + ips := expand.CIDR(tt.cidr) input.addTargets(ips) // scan got := []string{} @@ -170,7 +171,7 @@ func Test_expandASNInputValue(t *testing.T) { require.Nil(t, err, "could not create temporary input file") input := &Input{hostMap: hm} // get the IP addresses for ASN number - ips := input.expandASNInputValue(tt.asn) + ips := expand.ASN(tt.asn) input.addTargets(ips) // scan the hmap got := []string{} diff --git a/pkg/operators/extractors/compile.go b/pkg/operators/extractors/compile.go index 2b55d374ab..a70975f774 100644 --- a/pkg/operators/extractors/compile.go +++ b/pkg/operators/extractors/compile.go @@ -10,6 +10,10 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl" ) +const ( + ExtractedResultsDir = "extracted" +) + // CompileExtractors performs the initial setup operation on an extractor func (e *Extractor) CompileExtractors() error { // Set up the extractor type diff --git a/pkg/operators/extractors/extract.go b/pkg/operators/extractors/extract.go index f17fba5258..61b2cc5a08 100644 --- a/pkg/operators/extractors/extract.go +++ b/pkg/operators/extractors/extract.go @@ -30,7 +30,6 @@ func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} { } } } - e.SaveToFile(results) return results } @@ -58,7 +57,6 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} results[itemString] = struct{}{} } } - e.SaveToFile(results) return results } @@ -96,7 +94,6 @@ func (e *Extractor) ExtractHTML(corpus string) map[string]struct{} { } } } - e.SaveToFile(results) return results } @@ -127,7 +124,6 @@ func (e *Extractor) ExtractXML(corpus string) map[string]struct{} { } } } - e.SaveToFile(results) return results } @@ -164,7 +160,6 @@ func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { } } } - e.SaveToFile(results) return results } @@ -187,6 +182,5 @@ func (e *Extractor) ExtractDSL(data map[string]interface{}) map[string]struct{} } } } - e.SaveToFile(results) return results } diff --git a/pkg/operators/extractors/extractors.go b/pkg/operators/extractors/extractors.go index f531c01319..7071a0bda5 100644 --- a/pkg/operators/extractors/extractors.go +++ b/pkg/operators/extractors/extractors.go @@ -1,14 +1,10 @@ package extractors import ( - "os" - "path/filepath" "regexp" "github.com/Knetic/govaluate" "github.com/itchyny/gojq" - "github.com/projectdiscovery/gologger" - fileutil "github.com/projectdiscovery/utils/file" ) // Extractor is used to extract part of response using a regex. @@ -117,36 +113,4 @@ type Extractor struct { // - false // - true CaseInsensitive bool `yaml:"case-insensitive,omitempty" json:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"` - // description: | - // ToFile (to) saves extracted requests to file and if file is present values are appended to file. - ToFile string `yaml:"to,omitempty" json:"to,omitempty" jsonschema:"title=save extracted values to file,description=save extracted values to file"` -} - -// SaveToFile saves extracted values to file if `to` is present and valid -func (e *Extractor) SaveToFile(data map[string]struct{}) { - if e.ToFile == "" { - return - } - - if !fileutil.FileExists(e.ToFile) { - baseDir := filepath.Dir(e.ToFile) - if baseDir != "." && !fileutil.FolderExists(baseDir) { - if err := fileutil.CreateFolder(baseDir); err != nil { - gologger.Error().Msgf("extractor: could not create folder %s: %s\n", baseDir, err) - return - } - } - } - file, err := os.OpenFile(e.ToFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) - if err != nil { - gologger.Error().Msgf("extractor: could not open file %s: %s\n", e.ToFile, err) - return - } - defer file.Close() - for k := range data { - if _, err = file.WriteString(k + "\n"); err != nil { - gologger.Error().Msgf("extractor: could not write to file %s: %s\n", e.ToFile, err) - return - } - } } diff --git a/pkg/operators/matchers/matchers.go b/pkg/operators/matchers/matchers.go index 670113d42c..29dd37b84a 100644 --- a/pkg/operators/matchers/matchers.go +++ b/pkg/operators/matchers/matchers.go @@ -120,6 +120,14 @@ type Matcher struct { // - false // - true MatchAll bool `yaml:"match-all,omitempty" json:"match-all,omitempty" jsonschema:"title=match all values,description=match all matcher values ignoring condition"` + // description: | + // Internal when true hides the matcher from output. Default is false. + // It is meant to be used in multiprotocol / flow templates to create internal matcher condition without printing it in output. + // or other similar use cases. + // values: + // - false + // - true + Internal bool `yaml:"internal,omitempty" json:"internal,omitempty" jsonschema:"title=hide matcher from output,description=hide matcher from output"` // cached data for the compiled matcher condition ConditionType // todo: this field should be the one used for overridden marshal ops diff --git a/pkg/operators/matchers/validate.go b/pkg/operators/matchers/validate.go index 9e0a7aba81..0f6a5b916f 100644 --- a/pkg/operators/matchers/validate.go +++ b/pkg/operators/matchers/validate.go @@ -11,7 +11,7 @@ import ( "gopkg.in/yaml.v3" ) -var commonExpectedFields = []string{"Type", "Condition", "Name", "MatchAll", "Negative"} +var commonExpectedFields = []string{"Type", "Condition", "Name", "MatchAll", "Negative", "Internal"} // Validate perform initial validation on the matcher structure func (matcher *Matcher) Validate() error { diff --git a/pkg/operators/operators.go b/pkg/operators/operators.go index 2a504d12af..a3b4fc561b 100644 --- a/pkg/operators/operators.go +++ b/pkg/operators/operators.go @@ -90,6 +90,8 @@ type Result struct { // Optional lineCounts for file protocol LineCount string + // Operators is reference to operators that generated this result (Read-Only) + Operators *Operators } func (result *Result) HasMatch(name string) bool { @@ -194,7 +196,11 @@ func (r *Result) Merge(result *Result) { } } for k, v := range result.DynamicValues { - r.DynamicValues[k] = v + if _, ok := r.DynamicValues[k]; !ok { + r.DynamicValues[k] = v + } else { + r.DynamicValues[k] = sliceutil.Dedupe(append(r.DynamicValues[k], v...)) + } } for k, v := range result.PayloadValues { r.PayloadValues[k] = v @@ -217,10 +223,17 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc Extracts: make(map[string][]string), DynamicValues: make(map[string][]string), outputUnique: make(map[string]struct{}), + Operators: operators, } + // state variable to check if all extractors are internal + var allInternalExtractors bool = true + // Start with the extractors first and evaluate them. for _, extractor := range operators.Extractors { + if !extractor.Internal && allInternalExtractors { + allInternalExtractors = false + } var extractorResults []string for match := range extract(data, extractor) { extractorResults = append(extractorResults, match) @@ -241,6 +254,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc if len(extractorResults) > 0 && !extractor.Internal && extractor.Name != "" { result.Extracts[extractor.Name] = extractorResults } + // update data with whatever was extracted doesn't matter if it is internal or not (skip unless it empty) + if len(extractorResults) > 0 { + data[extractor.Name] = getExtractedValue(extractorResults) + } } // expose dynamic values to same request matchers @@ -288,7 +305,9 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc result.Matched = matches result.Extracted = len(result.OutputExtracts) > 0 - if len(result.DynamicValues) > 0 { + if len(result.DynamicValues) > 0 && allInternalExtractors { + // only return early if all extractors are internal + // if some are internal and some are not then followthrough return result, true } @@ -339,3 +358,13 @@ func (operators *Operators) IsEmpty() bool { func (operators *Operators) Len() int { return len(operators.Matchers) + len(operators.Extractors) } + +// getExtractedValue takes array of extracted values if it only has one value +// then it is flattened and returned as a string else original type is returned +func getExtractedValue(values []string) any { + if len(values) == 1 { + return values[0] + } else { + return values + } +} diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 0e96864af7..58526a0fb5 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -139,8 +139,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa metaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v}) } gOutput, err := request.gozero.Eval(context.Background(), request.src, metaSrc) - if err != nil { - return err + if err != nil && gOutput == nil { + return errorutil.NewWithErr(err).Msgf("[%s] Could not execute code on local machine %v", request.options.TemplateID, input.MetaInput.Input) } gologger.Verbose().Msgf("[%s] Executed code on local machine %v", request.options.TemplateID, input.MetaInput.Input) diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go index a58e0a1c94..3e269eda68 100644 --- a/pkg/protocols/common/automaticscan/automaticscan.go +++ b/pkg/protocols/common/automaticscan/automaticscan.go @@ -202,6 +202,8 @@ func (s *Service) processWappalyzerInputPair(input *contextargs.MetaInput) { if len(items) == 0 { return } + // Add tags as addition to -as for comprehensive scans. Ref: nuclei/issues/3348 + items = append(items, s.opts.Options.Tags...) uniqueTags := sliceutil.Dedupe(items) templatesList := s.store.LoadTemplatesWithTags(s.allTemplates, uniqueTags) diff --git a/pkg/protocols/common/helpers/eventcreator/eventcreator.go b/pkg/protocols/common/helpers/eventcreator/eventcreator.go index c87264536a..078963ceed 100644 --- a/pkg/protocols/common/helpers/eventcreator/eventcreator.go +++ b/pkg/protocols/common/helpers/eventcreator/eventcreator.go @@ -30,6 +30,14 @@ func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent out if compiledOperator != nil { result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract, isResponseDebug) if ok && result != nil { + // if result has both extracted values and dynamic values, put dynamic values in data + // and remove dynamic values to avoid skipping legitimate event + if (len(result.Extracts) > 0 || len(result.OutputExtracts) > 0) && len(result.DynamicValues) > 0 { + for k, v := range result.DynamicValues { + event.InternalEvent[k] = v + } + result.DynamicValues = nil + } event.OperatorsResult = result if addAdditionalOptions != nil { addAdditionalOptions(event) diff --git a/pkg/protocols/common/protocolstate/file.go b/pkg/protocols/common/protocolstate/file.go index 7d6618c860..199aa44f2a 100644 --- a/pkg/protocols/common/protocolstate/file.go +++ b/pkg/protocols/common/protocolstate/file.go @@ -32,3 +32,8 @@ func NormalizePath(filePath string) (string, error) { } return "", errorutil.New("path %v is outside nuclei-template directory and -lfa is not enabled", filePath) } + +// IsLFAAllowed returns true if local file access is allowed +func IsLFAAllowed() bool { + return lfaAllowed +} diff --git a/pkg/protocols/common/protocolstate/headless.go b/pkg/protocols/common/protocolstate/headless.go index 77e5d52cdd..0b58d5e326 100644 --- a/pkg/protocols/common/protocolstate/headless.go +++ b/pkg/protocols/common/protocolstate/headless.go @@ -17,7 +17,7 @@ import ( var ( ErrURLDenied = errorutil.NewWithFmt("headless: url %v dropped by rule: %v") ErrHostDenied = errorutil.NewWithFmt("host %v dropped by network policy") - networkPolicy *networkpolicy.NetworkPolicy + NetworkPolicy *networkpolicy.NetworkPolicy allowLocalFileAccess bool ) @@ -51,14 +51,11 @@ func FailWithReason(page *rod.Page, e *proto.FetchRequestPaused) error { } // InitHeadless initializes headless protocol state -func InitHeadless(RestrictLocalNetworkAccess bool, localFileAccess bool) { +func InitHeadless(localFileAccess bool, np *networkpolicy.NetworkPolicy) { allowLocalFileAccess = localFileAccess - if !RestrictLocalNetworkAccess { - return + if np != nil { + NetworkPolicy = np } - networkPolicy, _ = networkpolicy.New(networkpolicy.Options{ - DenyList: append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...), - }) } // isValidHost checks if the host is valid (only limited to http/https protocols) @@ -66,7 +63,7 @@ func isValidHost(targetUrl string) bool { if !stringsutil.HasPrefixAny(targetUrl, "http:", "https:") { return true } - if networkPolicy == nil { + if NetworkPolicy == nil { return true } urlx, err := urlutil.Parse(targetUrl) @@ -75,15 +72,15 @@ func isValidHost(targetUrl string) bool { return false } targetUrl = urlx.Hostname() - _, ok := networkPolicy.ValidateHost(targetUrl) + _, ok := NetworkPolicy.ValidateHost(targetUrl) return ok } // IsHostAllowed checks if the host is allowed by network policy func IsHostAllowed(targetUrl string) bool { - if networkPolicy == nil { + if NetworkPolicy == nil { return true } - _, ok := networkPolicy.ValidateHost(targetUrl) + _, ok := NetworkPolicy.ValidateHost(targetUrl) return ok } diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index 7a9f286bb4..6d4bd7672a 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -9,8 +9,10 @@ import ( "golang.org/x/net/proxy" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/networkpolicy" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/utils/expand" ) // Dialer is a shared fastdialer instance for host DNS resolution @@ -29,7 +31,27 @@ func Init(options *types.Options) error { if options.DialerKeepAlive > 0 { opts.DialerKeepAlive = options.DialerKeepAlive } - InitHeadless(options.RestrictLocalNetworkAccess, options.AllowLocalFileAccess) + + var expandedDenyList []string + for _, excludeTarget := range options.ExcludeTargets { + switch { + case asn.IsASN(excludeTarget): + expandedDenyList = append(expandedDenyList, expand.ASN(excludeTarget)...) + default: + expandedDenyList = append(expandedDenyList, excludeTarget) + } + } + + if options.RestrictLocalNetworkAccess { + expandedDenyList = append(expandedDenyList, networkpolicy.DefaultIPv4DenylistRanges...) + expandedDenyList = append(expandedDenyList, networkpolicy.DefaultIPv6DenylistRanges...) + } + npOptions := &networkpolicy.Options{ + DenyList: expandedDenyList, + } + opts.WithNetworkPolicyOptions = npOptions + NetworkPolicy, _ = networkpolicy.New(*npOptions) + InitHeadless(options.AllowLocalFileAccess, NetworkPolicy) switch { case options.SourceIP != "" && options.Interface != "": @@ -99,9 +121,9 @@ func Init(options *types.Options) error { if options.ResolversFile != "" { opts.BaseResolvers = options.InternalResolversList } - if options.RestrictLocalNetworkAccess { - opts.Deny = append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...) - } + + opts.Deny = append(opts.Deny, expandedDenyList...) + opts.WithDialerHistory = true opts.SNIName = options.SNI diff --git a/pkg/protocols/file/operators.go b/pkg/protocols/file/operators.go index aee2abdfda..ff18af0979 100644 --- a/pkg/protocols/file/operators.go +++ b/pkg/protocols/file/operators.go @@ -48,6 +48,10 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto return extractor.ExtractRegex(itemStr) case extractors.KValExtractor: return extractor.ExtractKval(data) + case extractors.JSONExtractor: + return extractor.ExtractJSON(itemStr) + case extractors.XPathExtractor: + return extractor.ExtractXPath(itemStr) case extractors.DSLExtractor: return extractor.ExtractDSL(data) } diff --git a/pkg/protocols/headless/engine/.nuclei-config/nuclei/.templates-config.json b/pkg/protocols/headless/engine/.nuclei-config/nuclei/.templates-config.json deleted file mode 100644 index 54f806915d..0000000000 --- a/pkg/protocols/headless/engine/.nuclei-config/nuclei/.templates-config.json +++ /dev/null @@ -1 +0,0 @@ -{"nuclei-templates-directory":"/Users/tarun/nuclei-templates","custom-s3-templates-directory":"/Users/tarun/nuclei-templates/s3","custom-github-templates-directory":"/Users/tarun/nuclei-templates/github","custom-gitlab-templates-directory":"/Users/tarun/nuclei-templates/gitlab","custom-azure-templates-directory":"/Users/tarun/nuclei-templates/azure","nuclei-latest-version":"","nuclei-templates-latest-version":""} \ No newline at end of file diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 05b1c5c553..7338db44ab 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http" @@ -395,6 +396,24 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error { if err != nil { return errors.Wrap(err, "could not take screenshot") } + targetPath := p.getActionArgWithDefaultValues(act, "to") + targetPath, err = fileutil.CleanPath(targetPath) + if err != nil { + return errorutil.New("could not clean output screenshot path %s", targetPath) + } + // allow if targetPath is child of current working directory + if !protocolstate.IsLFAAllowed() { + cwd, err := os.Getwd() + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not get current working directory") + } + if !strings.HasPrefix(targetPath, cwd) { + // writing outside of cwd requires -lfa flag + return ErrLFAccessDenied + } + } + + // edgecase create directory if mkdir=true and path contains directory if p.getActionArgWithDefaultValues(act, "mkdir") == "true" && stringsutil.ContainsAny(to, folderutil.UnixPathSeparator, folderutil.WindowsPathSeparator) { // creates new directory if needed based on path `to` // TODO: replace all permission bits with fileutil constants (https://github.com/projectdiscovery/utils/issues/113) @@ -402,8 +421,10 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error { return errorutil.NewWithErr(err).Msgf("failed to create directory while writing screenshot") } } - filePath := to - if !strings.HasSuffix(to, ".png") { + + // actual file path to write + filePath := targetPath + if !strings.HasSuffix(filePath, ".png") { filePath += ".png" } diff --git a/pkg/protocols/headless/engine/page_actions_test.go b/pkg/protocols/headless/engine/page_actions_test.go index 80e0e505c6..a4b69eeff0 100644 --- a/pkg/protocols/headless/engine/page_actions_test.go +++ b/pkg/protocols/headless/engine/page_actions_test.go @@ -577,7 +577,12 @@ func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData map[string]string)) { t.Helper() - _ = protocolstate.Init(&types.Options{}) + + lfa := getBoolFromEnv("LOCAL_FILE_ACCESS", true) + rna := getBoolFromEnv("RESTRICED_LOCAL_NETWORK_ACCESS", false) + opts := &types.Options{AllowLocalFileAccess: lfa, RestrictLocalNetworkAccess: rna} + + _ = protocolstate.Init(opts) browser, err := New(&types.Options{ShowBrowser: false, UseInstalledChrome: testheadless.HeadlessLocal}) require.Nil(t, err, "could not create browser") @@ -594,10 +599,7 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle input.CookieJar, err = cookiejar.New(nil) require.Nil(t, err) - lfa := getBoolFromEnv("LOCAL_FILE_ACCESS", true) - rna := getBoolFromEnv("RESTRICED_LOCAL_NETWORK_ACCESS", false) - - extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout, Options: &types.Options{AllowLocalFileAccess: lfa, RestrictLocalNetworkAccess: rna}}) // allow file access in test + extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout, Options: opts}) // allow file access in test assert(page, err, extractedData) if page != nil { diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index 7a19491fff..e1db829697 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -17,6 +17,7 @@ import ( ) // Match matches a generic data response again a given matcher +// TODO: Try to consolidate this in protocols.MakeDefaultMatchFunc to avoid any inconsistencies func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := request.getMatchPart(matcher.Part, data) if !ok && matcher.Type.MatcherType != matchers.DSLMatcher { diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 2328c8f30c..d1dd7accf7 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -303,6 +303,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous if err != nil { continue } + input.MetaInput = &contextargs.MetaInput{Input: generated.URL()} for _, rule := range request.Fuzzing { err = rule.Execute(&fuzz.ExecuteRuleInput{ Input: input, diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index f1f73804b8..8cc154b710 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -2,7 +2,9 @@ package scan import ( "context" + "fmt" "strings" + "sync" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" @@ -10,46 +12,52 @@ import ( type ScanContext struct { context.Context - Input *contextargs.Context - errors []error - events []*output.InternalWrappedEvent + // exported / configurable fields + Input *contextargs.Context + // callbacks or hooks OnError func(error) OnResult func(e *output.InternalWrappedEvent) + + // unexported state fields + errors []error + warnings []string + events []*output.InternalWrappedEvent + + // might not be required but better to sync + m sync.Mutex } +// NewScanContext creates a new scan context using input func NewScanContext(input *contextargs.Context) *ScanContext { return &ScanContext{Input: input} } +// GenerateResult returns final results slice from all events func (s *ScanContext) GenerateResult() []*output.ResultEvent { + s.m.Lock() + defer s.m.Unlock() return aggregateResults(s.events) } -func aggregateResults(events []*output.InternalWrappedEvent) []*output.ResultEvent { - var results []*output.ResultEvent - for _, e := range events { - results = append(results, e.Results...) - } - return results -} - -func joinErrors(errors []error) string { - var errorMessages []string - for _, e := range errors { - errorMessages = append(errorMessages, e.Error()) - } - return strings.Join(errorMessages, "; ") -} - +// LogEvent logs events to all events and triggeres any callbacks func (s *ScanContext) LogEvent(e *output.InternalWrappedEvent) { + s.m.Lock() + defer s.m.Unlock() + if e == nil { + // do not log nil events + return + } if s.OnResult != nil { s.OnResult(e) } s.events = append(s.events, e) } +// LogError logs error to all events and triggeres any callbacks func (s *ScanContext) LogError(err error) { + s.m.Lock() + defer s.m.Unlock() if err == nil { return } @@ -68,3 +76,37 @@ func (s *ScanContext) LogError(err error) { e.InternalEvent["error"] = errorMessage } } + +// LogWarning logs warning to all events +func (s *ScanContext) LogWarning(format string, args ...any) { + s.m.Lock() + defer s.m.Unlock() + val := fmt.Sprintf(format, args...) + s.warnings = append(s.warnings, val) + + for _, e := range s.events { + if e.InternalEvent != nil { + e.InternalEvent["warning"] = strings.Join(s.warnings, "; ") + } + } +} + +// aggregateResults aggregates results from multiple events +func aggregateResults(events []*output.InternalWrappedEvent) []*output.ResultEvent { + var results []*output.ResultEvent + for _, e := range events { + results = append(results, e.Results...) + } + return results +} + +// joinErrors joins multiple errors and returns a single error string +func joinErrors(errors []error) string { + var errorMessages []string + for _, e := range errors { + if e != nil { + errorMessages = append(errorMessages, e.Error()) + } + } + return strings.Join(errorMessages, "; ") +} diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index 45f5058779..04235d5016 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -47,7 +47,7 @@ func NewTemplateExecuter(requests []protocols.Request, options *protocols.Execut // we use a dummy input here because goal of flow executor at this point is to just check // syntax and other things are correct before proceeding to actual execution // during execution new instance of flow will be created as it is tightly coupled with lot of executor options - e.engine = flow.NewFlowExecutor(requests, contextargs.NewWithInput("dummy"), options, e.results) + e.engine = flow.NewFlowExecutor(requests, scan.NewScanContext(contextargs.NewWithInput("dummy")), options, e.results) } else { // Review: // multiproto engine is only used if there is more than one protocol in template @@ -117,6 +117,22 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { // something went wrong return } + // check for internal true matcher event + if event.HasOperatorResult() && event.OperatorsResult.Matched && event.OperatorsResult.Operators != nil { + // note all matchers should have internal:true if it is a combination then print it + allInternalMatchers := true + for _, matcher := range event.OperatorsResult.Operators.Matchers { + if allInternalMatchers && !matcher.Internal { + allInternalMatchers = false + break + } + } + if allInternalMatchers { + // this is a internal event and no meant to be printed + return + } + } + // If no results were found, and also interactsh is not being used // in that case we can skip it, otherwise we've to show failure in // case of matcher-status flag. @@ -139,8 +155,9 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { // so in compile step earlier we compile it to validate javascript syntax and other things // and while executing we create new instance of flow executor everytime if e.options.Flow != "" { - flowexec := flow.NewFlowExecutor(e.requests, ctx.Input, e.options, results) + flowexec := flow.NewFlowExecutor(e.requests, ctx, e.options, results) if err := flowexec.Compile(); err != nil { + ctx.LogError(err) return false, err } err = flowexec.ExecuteWithResults(ctx) diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 690a4be991..4457d8e529 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -9,9 +9,7 @@ import ( "github.com/dop251/goja" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/scan" @@ -38,13 +36,12 @@ type ProtoOptions struct { // FlowExecutor is a flow executor for executing a flow type FlowExecutor struct { - input *contextargs.Context + ctx *scan.ScanContext // scan context (includes target etc) options *protocols.ExecutorOptions // javascript runtime reference and compiled program - jsVM *goja.Runtime - program *goja.Program // compiled js program - lastEvent *output.InternalWrappedEvent // contains last event that was emitted + jsVM *goja.Runtime + program *goja.Program // compiled js program // protocol requests and their callback functions allProtocols map[string][]protocols.Request @@ -56,7 +53,9 @@ type FlowExecutor struct { } // NewFlowExecutor creates a new flow executor from a list of requests -func NewFlowExecutor(requests []protocols.Request, input *contextargs.Context, options *protocols.ExecutorOptions, results *atomic.Bool) *FlowExecutor { +// Note: Unlike other engine for every target x template flow needs to be compiled and executed everytime +// unlike other engines where we compile once and execute multiple times +func NewFlowExecutor(requests []protocols.Request, ctx *scan.ScanContext, options *protocols.ExecutorOptions, results *atomic.Bool) *FlowExecutor { allprotos := make(map[string][]protocols.Request) for _, req := range requests { switch req.Type() { @@ -81,7 +80,8 @@ func NewFlowExecutor(requests []protocols.Request, input *contextargs.Context, o case templateTypes.JavascriptProtocol: allprotos[templateTypes.JavascriptProtocol.String()] = append(allprotos[templateTypes.JavascriptProtocol.String()], req) default: - gologger.Error().Msgf("invalid request type %s", req.Type().String()) + ctx.LogError(fmt.Errorf("invalid request type %s", req.Type().String())) + return nil } } f := &FlowExecutor{ @@ -94,7 +94,7 @@ func NewFlowExecutor(requests []protocols.Request, input *contextargs.Context, o protoFunctions: map[string]func(call goja.FunctionCall) goja.Value{}, results: results, jsVM: protocolstate.NewJSRuntime(), - input: input, + ctx: ctx, } return f } @@ -105,7 +105,7 @@ func (f *FlowExecutor) Compile() error { f.results = new(atomic.Bool) } // load all variables and evaluate with existing data - variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()) + variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()) // cli options optionVars := generators.BuildPayloadFromOptions(f.options.Options) // constants @@ -118,11 +118,11 @@ func (f *FlowExecutor) Compile() error { if value, err := f.ReadDataFromFile(str); err == nil { allVars[k] = value } else { - gologger.Warning().Msgf("could not load file '%s' for variable '%s': %s", str, k, err) + f.ctx.LogWarning("could not load file '%s' for variable '%s': %s", str, k, err) } } } - f.options.GetTemplateCtx(f.input.MetaInput).Merge(allVars) // merge all variables into template context + f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Merge(allVars) // merge all variables into template context // ---- define callback functions/objects---- f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{} @@ -165,24 +165,24 @@ func (f *FlowExecutor) Compile() error { func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { defer func() { if e := recover(); e != nil { + f.ctx.LogError(fmt.Errorf("panic occurred while executing target %v with flow: %v", ctx.Input.MetaInput.Input, e)) gologger.Error().Label(f.options.TemplateID).Msgf("panic occurred while executing target %v with flow: %v", ctx.Input.MetaInput.Input, e) - panic(e) } }() - f.input = ctx.Input + f.ctx.Input = ctx.Input // -----Load all types of variables----- // add all input args to template context - if f.input != nil && f.input.HasArgs() { - f.input.ForEach(func(key string, value interface{}) { - f.options.GetTemplateCtx(f.input.MetaInput).Set(key, value) + if f.ctx.Input != nil && f.ctx.Input.HasArgs() { + f.ctx.Input.ForEach(func(key string, value interface{}) { + f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(key, value) }) } if ctx.OnResult == nil { return fmt.Errorf("output callback cannot be nil") } // pass flow and execute the js vm and handle errors - value, err := f.jsVM.RunProgram(f.program) + _, err := f.jsVM.RunProgram(f.program) if err != nil { ctx.LogError(err) return errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) @@ -192,13 +192,7 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { ctx.LogError(runtimeErr) return errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow") } - // this is where final result is generated/created - ctx.LogEvent(f.lastEvent) - if value.Export() != nil { - f.results.Store(value.ToBoolean()) - } else { - f.results.Store(true) - } + return nil } diff --git a/pkg/tmplexec/flow/flow_internal.go b/pkg/tmplexec/flow/flow_internal.go index b435e92728..5f2f858c2e 100644 --- a/pkg/tmplexec/flow/flow_internal.go +++ b/pkg/tmplexec/flow/flow_internal.go @@ -1,6 +1,7 @@ package flow import ( + "fmt" "reflect" "sync/atomic" @@ -21,11 +22,11 @@ import ( func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool { defer func() { // evaluate all variables after execution of each protocol - variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()) - f.options.GetTemplateCtx(f.input.MetaInput).Merge(variableMap) // merge all variables into template context + variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()) + f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Merge(variableMap) // merge all variables into template context // to avoid polling update template variables everytime we execute a protocol - var m map[string]interface{} = f.options.GetTemplateCtx(f.input.MetaInput).GetAll() + var m map[string]interface{} = f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll() _ = f.jsVM.Set("template", m) }() matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool @@ -34,7 +35,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req // execution logic for http()/dns() etc for index := range f.allProtocols[opts.protoName] { req := f.allProtocols[opts.protoName][index] - err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, f.getProtoRequestCallback(req, matcherStatus, opts)) + err := req.ExecuteWithResults(f.ctx.Input, output.InternalEvent(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()), nil, f.protocolResultCallback(req, matcherStatus, opts)) if err != nil { // save all errors in a map with id as key // its less likely that there will be race condition but just in case @@ -44,7 +45,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req } err = f.allErrs.Set(opts.protoName+":"+id, err) if err != nil { - gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + f.ctx.LogError(fmt.Errorf("failed to store flow runtime errors got %v", err)) } return matcherStatus.Load() } @@ -56,36 +57,38 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req for _, id := range opts.reqIDS { req, ok := reqMap[id] if !ok { - gologger.Error().Msgf("[%v] invalid request id '%s' provided", f.options.TemplateID, id) + f.ctx.LogError(fmt.Errorf("[%v] invalid request id '%s' provided", f.options.TemplateID, id)) // compile error if err := f.allErrs.Set(opts.protoName+":"+id, ErrInvalidRequestID.Msgf(f.options.TemplateID, id)); err != nil { - gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + f.ctx.LogError(fmt.Errorf("failed to store flow runtime errors got %v", err)) } return matcherStatus.Load() } - err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, f.getProtoRequestCallback(req, matcherStatus, opts)) + err := req.ExecuteWithResults(f.ctx.Input, output.InternalEvent(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()), nil, f.protocolResultCallback(req, matcherStatus, opts)) if err != nil { index := id err = f.allErrs.Set(opts.protoName+":"+index, err) if err != nil { - gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + f.ctx.LogError(fmt.Errorf("failed to store flow runtime errors got %v", err)) } } } return matcherStatus.Load() } -// getProtoRequestCallback returns a callback that is executed +// protocolResultCallback returns a callback that is executed // after execution of each protocol request -func (f *FlowExecutor) getProtoRequestCallback(req protocols.Request, matcherStatus *atomic.Bool, opts *ProtoOptions) func(result *output.InternalWrappedEvent) { +func (f *FlowExecutor) protocolResultCallback(req protocols.Request, matcherStatus *atomic.Bool, opts *ProtoOptions) func(result *output.InternalWrappedEvent) { return func(result *output.InternalWrappedEvent) { if result != nil { - f.results.CompareAndSwap(false, true) - f.lastEvent = result + // Note: flow specific implicit behaviours should be handled here + // before logging the event + f.ctx.LogEvent(result) // export dynamic values from operators (i.e internal:true) // add add it to template context // this is a conflicting behaviour with iterate-all if result.HasOperatorResult() { + f.results.CompareAndSwap(false, true) // this is to handle case where there is any operator result (matcher or extractor) matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) if !result.OperatorsResult.Matched && !hasMatchers(req.GetCompiledOperators()) { @@ -95,7 +98,7 @@ func (f *FlowExecutor) getProtoRequestCallback(req protocols.Request, matcherSta } if len(result.OperatorsResult.DynamicValues) > 0 { for k, v := range result.OperatorsResult.DynamicValues { - f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v) + f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(k, v) } } } else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) { @@ -130,7 +133,7 @@ func (f *FlowExecutor) registerBuiltInFunctions() error { default: gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) } - return goja.Null() + return call.Argument(0) // return the same value }); err != nil { return err } @@ -138,7 +141,7 @@ func (f *FlowExecutor) registerBuiltInFunctions() error { if err := f.jsVM.Set("set", func(call goja.FunctionCall) goja.Value { varName := call.Argument(0).Export() varValue := call.Argument(1).Export() - f.options.GetTemplateCtx(f.input.MetaInput).Set(types.ToString(varName), varValue) + f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(types.ToString(varName), varValue) return goja.Null() }); err != nil { return err @@ -179,7 +182,7 @@ func (f *FlowExecutor) registerBuiltInFunctions() error { return err } - var m = f.options.GetTemplateCtx(f.input.MetaInput).GetAll() + var m = f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll() if m == nil { m = map[string]interface{}{} } diff --git a/pkg/utils/expand/expand.go b/pkg/utils/expand/expand.go new file mode 100644 index 0000000000..4243c718ce --- /dev/null +++ b/pkg/utils/expand/expand.go @@ -0,0 +1,26 @@ +package expand + +import ( + "github.com/projectdiscovery/mapcidr" + "github.com/projectdiscovery/mapcidr/asn" +) + +// Expands CIDR to IPs +func CIDR(value string) []string { + var ips []string + ipsCh, _ := mapcidr.IPAddressesAsStream(value) + for ip := range ipsCh { + ips = append(ips, ip) + } + return ips +} + +// Expand ASN to IPs +func ASN(value string) []string { + var ips []string + cidrs, _ := asn.GetCIDRsForASNNum(value) + for _, cidr := range cidrs { + ips = append(ips, CIDR(cidr.String())...) + } + return ips +}