Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
23 lines (18 sloc) 360 Bytes
{
"From": [
"https://www.example.com/"
],
"Include": [
"^(https?://)?www\\.example\\.com/.*"
],
"Exclude": [],
"MaxDepth": 3,
"WaitTime": "100ms",
"Connections": 20,
"UserAgent": "Crawler/1.0",
"RobotsUserAgent": "Crawler",
"RespectNofollow": true,
"Header": [
{"K": "X-ample", "V":"alue"}
]
}
You can’t perform that action at this time.