Skip to content

Commit

Permalink
Merge pull request #3193 from projectdiscovery/dev
Browse files Browse the repository at this point in the history
nuclei v2.8.7
  • Loading branch information
ehsandeep committed Jan 15, 2023
2 parents 885e37e + 08e89cc commit fa3c4fa
Show file tree
Hide file tree
Showing 31 changed files with 737 additions and 271 deletions.
6 changes: 5 additions & 1 deletion .github/workflows/build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,8 @@ jobs:
- name: Race Condition Tests
if: ${{ matrix.os != 'windows-latest' }} # known issue: https://github.com/golang/go/issues/46099
run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version
working-directory: v2/cmd/nuclei/
working-directory: v2/cmd/nuclei/

- name: Example Code Tests
run: go build .
working-directory: v2/examples/
107 changes: 1 addition & 106 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -362,112 +362,7 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu

### Using Nuclei From Go Code

An example of using Nuclei From Go Code to run templates on targets is provided below.

```go
package main

import (
"context"
"fmt"
"log"
"os"
"path"
"time"

"github.com/logrusorgru/aurora"

"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v2/pkg/core"
"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/ratelimit"
)

func main() {
cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount)
defer cache.Close()

mockProgress := &testutils.MockProgressClient{}
reportingClient, _ := reporting.New(&reporting.Options{}, "")
defer reportingClient.Close()

outputWriter := testutils.NewMockOutputWriter()
outputWriter.WriteCallback = func(event *output.ResultEvent) {
fmt.Printf("Got Result: %v\n", event)
}

defaultOpts := types.DefaultOptions()
protocolstate.Init(defaultOpts)
protocolinit.Init(defaultOpts)

defaultOpts.Templates = goflags.StringSlice{"dns/cname-service.yaml"}
defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags

interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress)
interactClient, err := interactsh.New(interactOpts)
if err != nil {
log.Fatalf("Could not create interact client: %s\n", err)
}
defer interactClient.Close()

home, _ := os.UserHomeDir()
catalog := disk.NewCatalog(path.Join(home, "nuclei-templates"))
executerOpts := protocols.ExecuterOptions{
Output: outputWriter,
Options: defaultOpts,
Progress: mockProgress,
Catalog: catalog,
IssuesClient: reportingClient,
RateLimiter: ratelimit.New(context.Background(), 150, time.Second),
Interactsh: interactClient,
HostErrorsCache: cache,
Colorizer: aurora.NewAurora(true),
ResumeCfg: types.NewResumeCfg(),
}
engine := core.New(defaultOpts)
engine.SetExecuterOptions(executerOpts)

workflowLoader, err := parsers.NewLoader(&executerOpts)
if err != nil {
log.Fatalf("Could not create workflow loader: %s\n", err)
}
executerOpts.WorkflowLoader = workflowLoader

configObject, err := config.ReadConfiguration()
if err != nil {
log.Fatalf("Could not read config: %s\n", err)
}
store, err := loader.New(loader.NewConfig(defaultOpts, configObject, catalog, executerOpts))
if err != nil {
log.Fatalf("Could not create loader client: %s\n", err)
}
store.Load()

inputArgs := []*contextargs.MetaInput{
&contextargs.MetaInput{
Input: "docs.hackerone.com",
},
}

input := &inputs.SimpleInputProvider{Inputs: inputArgs}
_ = engine.Execute(store.Templates(), input)
engine.WorkPool().Wait() // Wait for the scan to finish
}
```
Examples of using Nuclei From Go Code to run templates on targets are provided in the [examples](v2/examples/) folder.


### Resources
Expand Down
15 changes: 15 additions & 0 deletions integration_tests/http/cl-body-with-header.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
id: cl-body-with-header

info:
name: CL Get Request - Body with header
author: pdteam
severity: info

requests:
- method: GET
path:
- "{{BaseURL}}"
matchers:
- type: dsl
dsl:
- "content_length==50000 && len(body)==14"
15 changes: 15 additions & 0 deletions integration_tests/http/cl-body-without-header.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
id: cl-body-without-header

info:
name: CL Get Request - Body without header
author: pdteam
severity: info

requests:
- method: GET
path:
- "{{BaseURL}}"
matchers:
- type: dsl
dsl:
- "content_length==14"
6 changes: 6 additions & 0 deletions v2/.goreleaser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@ before:
hooks:
- go mod tidy

# release options (https://goreleaser.com/customization/release/)

release:
draft: true
prerelease: auto

builds:

- main: cmd/nuclei/main.go
Expand Down
46 changes: 46 additions & 0 deletions v2/cmd/integration-test/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/julienschmidt/httprouter"

"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
logutil "github.com/projectdiscovery/utils/log"
stringsutil "github.com/projectdiscovery/utils/strings"
)

Expand Down Expand Up @@ -59,6 +60,8 @@ var httpTestcases = map[string]testutils.TestCase{
"http/custom-attack-type.yaml": &customAttackType{},
"http/get-all-ips.yaml": &scanAllIPS{},
"http/get-without-scheme.yaml": &httpGetWithoutScheme{},
"http/cl-body-without-header.yaml": &httpCLBodyWithoutHeader{},
"http/cl-body-with-header.yaml": &httpCLBodyWithHeader{},
}

type httpInteractshRequest struct{}
Expand Down Expand Up @@ -1037,3 +1040,46 @@ func (h *httpGetWithoutScheme) Execute(filePath string) error {
}
return expectResultsCount(got, 1)
}

// content-length in case the response has no header but has a body
type httpCLBodyWithoutHeader struct{}

// Execute executes a test case and returns an error if occurred
func (h *httpCLBodyWithoutHeader) Execute(filePath string) error {
logutil.DisableDefaultLogger()
defer logutil.EnableDefaultLogger()

router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.Header()["Content-Length"] = []string{"-1"}
fmt.Fprintf(w, "this is a test")
})
ts := httptest.NewTLSServer(router)
defer ts.Close()

got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
return expectResultsCount(got, 1)
}

// content-length in case the response has content-length header and a body
type httpCLBodyWithHeader struct{}

// Execute executes a test case and returns an error if occurred
func (h *httpCLBodyWithHeader) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.Header()["Content-Length"] = []string{"50000"}
fmt.Fprintf(w, "this is a test")
})
ts := httptest.NewTLSServer(router)
defer ts.Close()

got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
return expectResultsCount(got, 1)
}
8 changes: 7 additions & 1 deletion v2/cmd/nuclei/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/interactsh/pkg/client"
"github.com/projectdiscovery/nuclei/v2/internal/runner"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
Expand Down Expand Up @@ -305,23 +306,28 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVarP(&options.AddTarget, "add-target", "atr", "", "add target(s) to cloud"),
flagSet.StringVarP(&options.AddTemplate, "add-template", "atm", "", "add template(s) to cloud"),
flagSet.BoolVarP(&options.ScanList, "list-scan", "lsn", false, "list previous cloud scans"),
flagSet.StringVarP(&options.ScanOutput, "list-output", "lso", "", "list scan output by scan id"),
flagSet.BoolVarP(&options.ListTargets, "list-target", "ltr", false, "list cloud target by id"),
flagSet.BoolVarP(&options.ListTemplates, "list-template", "ltm", false, "list cloud template by id"),
flagSet.BoolVarP(&options.ListDatasources, "list-datasource", "lds", false, "list cloud datasource by id"),
flagSet.BoolVarP(&options.ListReportingSources, "list-reportsource", "lrs", false, "list reporting sources"),
flagSet.StringVarP(&options.DeleteScan, "delete-scan", "dsn", "", "delete cloud scan by id"),
flagSet.StringVarP(&options.RemoveTarget, "delete-target", "dtr", "", "delete target(s) from cloud"),
flagSet.StringVarP(&options.RemoveTemplate, "delete-template", "dtm", "", "delete template(s) from cloud"),
flagSet.StringVarP(&options.RemoveDatasource, "delete-datasource", "dds", "", "delete specified data source"),
flagSet.StringVarP(&options.DisableReportingSource, "disable-reportsource", "drs", "", "disable specified reporting source"),
flagSet.StringVarP(&options.EnableReportingSource, "enable-reportsource", "ers", "", "enable specified reporting source"),
flagSet.StringVarP(&options.GetTarget, "get-target", "gtr", "", "get target content by id"),
flagSet.StringVarP(&options.GetTemplate, "get-template", "gtm", "", "get template content by id"),
flagSet.BoolVarP(&options.NoStore, "no-store", "nos", false, "disable scan/output storage on cloud"),
flagSet.StringVarP(&options.ScanOutput, "scan-output", "sno", "", "display scan output by scan id"),
flagSet.BoolVar(&options.NoTables, "no-tables", false, "do not display pretty-printed tables"),
flagSet.IntVar(&options.OutputLimit, "limit", 100, "limit the number of output to display"),
)

_ = flagSet.Parse()

gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug)

if options.LeaveDefaultPorts {
http.LeaveDefaultPorts = true
}
Expand Down
98 changes: 98 additions & 0 deletions v2/examples/simple.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package main

import (
"context"
"fmt"
"log"
"os"
"path"
"time"

"github.com/logrusorgru/aurora"

"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v2/pkg/core"
"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/ratelimit"
)

func main() {
cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount)
defer cache.Close()

mockProgress := &testutils.MockProgressClient{}
reportingClient, _ := reporting.New(&reporting.Options{}, "")
defer reportingClient.Close()

outputWriter := testutils.NewMockOutputWriter()
outputWriter.WriteCallback = func(event *output.ResultEvent) {
fmt.Printf("Got Result: %v\n", event)
}

defaultOpts := types.DefaultOptions()
protocolstate.Init(defaultOpts)
protocolinit.Init(defaultOpts)

defaultOpts.IncludeIds = goflags.StringSlice{"cname-service"}
defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags

interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress)
interactClient, err := interactsh.New(interactOpts)
if err != nil {
log.Fatalf("Could not create interact client: %s\n", err)
}
defer interactClient.Close()

home, _ := os.UserHomeDir()
catalog := disk.NewCatalog(path.Join(home, "nuclei-templates"))
executerOpts := protocols.ExecuterOptions{
Output: outputWriter,
Options: defaultOpts,
Progress: mockProgress,
Catalog: catalog,
IssuesClient: reportingClient,
RateLimiter: ratelimit.New(context.Background(), 150, time.Second),
Interactsh: interactClient,
HostErrorsCache: cache,
Colorizer: aurora.NewAurora(true),
ResumeCfg: types.NewResumeCfg(),
}
engine := core.New(defaultOpts)
engine.SetExecuterOptions(executerOpts)

workflowLoader, err := parsers.NewLoader(&executerOpts)
if err != nil {
log.Fatalf("Could not create workflow loader: %s\n", err)
}
executerOpts.WorkflowLoader = workflowLoader

configObject, err := config.ReadConfiguration()
if err != nil {
log.Fatalf("Could not read config: %s\n", err)
}
store, err := loader.New(loader.NewConfig(defaultOpts, configObject, catalog, executerOpts))
if err != nil {
log.Fatalf("Could not create loader client: %s\n", err)
}
store.Load()

inputArgs := []*contextargs.MetaInput{{Input: "docs.hackerone.com"}}

input := &inputs.SimpleInputProvider{Inputs: inputArgs}
_ = engine.Execute(store.Templates(), input)
engine.WorkPool().Wait() // Wait for the scan to finish
}
Loading

0 comments on commit fa3c4fa

Please sign in to comment.