Skip to content

Commit

Permalink
up 2023-01-06
Browse files Browse the repository at this point in the history
  • Loading branch information
@hktalent
hktalent committed Jan 6, 2023
1 parent b9681ee commit 0520752
Show file tree
Hide file tree
Showing 47 changed files with 1,055 additions and 781 deletions.
Empty file added 360.net.json
View file
Empty file.
3 changes: 2 additions & 1 deletion README.md
View file
Expand Up @@ -193,6 +193,7 @@ more see: <a href=https://github.com/hktalent/ProScan4all/discussions>discussion
# Communication group (WeChat, QQ，Tg)
| Wechat | Or | QQchat | Or | Tg |
| --- |--- |--- |--- |--- |

|<img width=166 src=https://github.com/hktalent/ProScan4all/blob/main/static/wcq.JPG>||<img width=166 src=https://github.com/hktalent/ProScan4all/blob/main/static/qqc.jpg>||<img width=166 src=https://github.com/hktalent/ProScan4all/blob/main/static/tg.jpg>|


Expand All @@ -202,5 +203,5 @@ more see: <a href=https://github.com/hktalent/ProScan4all/discussions>discussion
# Donation
| Wechat Pay | AliPay | Paypal | BTC Pay |BCH Pay |
| --- | --- | --- | --- | --- |
|<img src=https://github.com/hktalent/myhktools/blob/master/md/wc.png>|<img width=166 src=https://github.com/hktalent/myhktools/blob/master/md/zfb.png>|[paypal](https://www.paypal.me/pwned2019) **miracletalent@gmail.com**|<img width=166 src=https://github.com/hktalent/myhktools/blob/master/md/BTC.png>|<img width=166 src=https://github.com/hktalent/myhktools/blob/master/md/BCH.jpg>|
|<img src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/wc.png>|<img width=166 src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/zfb.png>|[paypal](https://www.paypal.me/pwned2019) **miracletalent@gmail.com**|<img width=166 src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/BTC.png>|<img width=166 src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/BCH.jpg>|

2 changes: 1 addition & 1 deletion README_CN.md
View file
Expand Up @@ -239,7 +239,7 @@ more see: <a href=https://github.com/hktalent/ProScan4all/discussions>discussion
# 交流群(微信、QQ、Tg)
| Wechat | Or | QQchat | Or | Tg |
| --- |--- |--- |--- |--- |
|<img width=166 src=https://github.com/hktalent/ProScan4all/blob/main/static/wcq.JPG>||<img width=166 src=https://github.com/hktalent/ProScan4all/blob/main/static/qqc.jpg>||<img width=166 src=https://github.com/hktalent/ProScan4all/blob/main/static/tg.jpg>|
|<img src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/wc.png>|<img width=166 src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/zfb.png>|[paypal](https://www.paypal.me/pwned2019) **miracletalent@gmail.com**|<img width=166 src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/BTC.png>|<img width=166 src=https://raw.githubusercontent.com/hktalent/myhktools/main/md/BCH.jpg>|


## 💖Star
Expand Down
234 changes: 121 additions & 113 deletions brute/filefuzz.go
View file
Expand Up @@ -22,14 +22,17 @@ import (
)

// 备份、敏感文件后缀
//
//go:embed dicts/bakSuffix.txt
var bakSuffix string

// 备份、敏感文件 http头类型 ContentType 检测
//
//go:embed dicts/fuzzContentType1.txt
var fuzzct string

// 敏感文件前缀
//
//go:embed dicts/prefix.txt
var szPrefix string

Expand Down Expand Up @@ -97,7 +100,8 @@ func reqPage(u string) (*util.Page, *util.Response, error) {
}

// 敏感文件头信息检测:
// 检测头信息是否有敏感文件、本份文件、流文件等敏感信息
//
// 检测头信息是否有敏感文件、本份文件、流文件等敏感信息
func CheckBakPage(req *util.Response) bool {
if x0, ok := (*req.Header)["Content-Type"]; ok && 0 < len(x0) {
x0B := []byte(x0[0])
Expand Down Expand Up @@ -180,8 +184,9 @@ type FuzzData struct {
var r001 = regexp.MustCompile(`\.(aac)|(abw)|(arc)|(avif)|(avi)|(azw)|(bin)|(bmp)|(bz)|(bz2)|(cda)|(csh)|(css)|(csv)|(doc)|(docx)|(eot)|(epub)|(gz)|(gif)|(ico)|(ics)|(jar)|(jpeg)|(jpg)|(js)|(json)|(jsonld)|(mid)|(midi)|(mjs)|(mp3)|(mp4)|(mpeg)|(mpkg)|(odp)|(ods)|(odt)|(oga)|(ogv)|(ogx)|(opus)|(otf)|(png)|(pdf)|(php)|(ppt)|(pptx)|(rar)|(rtf)|(sh)|(svg)|(tar)|(tif)|(tiff)|(ts)|(ttf)|(txt)|(vsd)|(wav)|(weba)|(webm)|(webp)|(woff)|(woff2)|(xhtml)|(xls)|(xlsx)|(xml)|(xul)|(zip)|(3gp)|(3g2)|(7z)$`)

// 重写了fuzz：优化流程、优化算法、修复线程安全bug、增加智能功能
// 两次 ioutil.ReadAll(resp.Body)，第二次就会 Read返回EOF error
// 去除指纹请求的路径，避免重复
//
// 两次 ioutil.ReadAll(resp.Body)，第二次就会 Read返回EOF error
// 去除指纹请求的路径，避免重复
func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody string) ([]string, []string) {
DoInitMap()
u01, err := url.Parse(strings.TrimSpace(u))
Expand Down Expand Up @@ -250,7 +255,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
var lst200 *util.Response
t001 := time.NewTicker(3 * time.Second)
var nCnt int32 = 0
go func() {
util.DefaultPool.Submit(func() {
for {
select {
case <-ctx2.Done():
Expand Down Expand Up @@ -284,7 +289,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
// <-time.After(time.Duration(100) * time.Millisecond)
}
}
}()
})
log.Printf("wait for file fuzz(dicts:%d) %s \r", len(filedic), u)

BreakAll:
Expand All @@ -307,124 +312,126 @@ BreakAll:
endP := u[len(u)-1:] == "/"
ch <- struct{}{}
wg.Add(1)
go func(payload string) {
payload = strings.TrimSpace(payload)
defer func() {
wg.Done() // 控制所有线程结束
<-ch // 并发控制
}()
atomic.AddInt32(&nCnt, 1)
for {
select {
case <-ctx.Done(): // 00-捕获所有线程关闭信号，并退出，close for all
atomic.AddInt32(&errorTimes, MaxErrorTimes)
return
default:
//if _, ok := noRpt.Load(szKey001Over); ok {
// stop()
// return
//}
// 01-异常>20关闭所有fuzz
if atomic.LoadInt32(&errorTimes) >= MaxErrorTimes {
stop() //发停止指令
func(payload string) {
util.DefaultPool.Submit(func() {
payload = strings.TrimSpace(payload)
defer func() {
wg.Done() // 控制所有线程结束
<-ch // 并发控制
}()
atomic.AddInt32(&nCnt, 1)
for {
select {
case <-ctx.Done(): // 00-捕获所有线程关闭信号，并退出，close for all
atomic.AddInt32(&errorTimes, MaxErrorTimes)
return
}
// 修复url，默认 认为 payload 不包含/
szUrl := u + payload
if strings.HasPrefix(payload, "/") && endP {
szUrl = u + payload[1:]
}
//log.Printf("start fuzz: [%s]", szUrl)
if fuzzPage, req, err := reqPage(szUrl); err == nil && nil != req && 0 < len(req.Body) {
if 200 == req.StatusCode {
if nil == lst200 {
lst200 = req
} else if lst200.Body == req.Body { // 无意义的 200
continue
}
if oU1, err := url.Parse(szUrl); nil == err {
a50 := r001.FindStringSubmatch(oU1.Path)
if 0 < len(a50) {
s2 := mime.TypeByExtension(filepath.Ext(a50[0]))
ct := (*req).Header.Get("Content-Type")
if "" != ct && "" != s2 && strings.Contains(ct, s2) {
continue
}
}
}
//log.Printf("%d : %s \n", req.StatusCode, szUrl)
if IsLoginPage(szUrl, req.Body, req.StatusCode) {
technologies = append(technologies, "loginpage")
}
}
go util.CheckHeader(req.Header, u)
// 02-状态码和req1相同，且与req1相似度>9.5，关闭所有fuzz
fXsd := strsim.Compare(url404req.Body, req.Body)
bBig95 := 9.5 < fXsd
//if "/bea_wls_internal/classes/mejb@/org/omg/stub/javax/management/j2ee/_ManagementHome_Stub.class" == payload {
// log.Println("start debug")
default:
//if _, ok := noRpt.Load(szKey001Over); ok {
// stop()
// return
//}
if url404.StatusCode == fuzzPage.StatusCode && bBig95 {
// 01-异常>20关闭所有fuzz
if atomic.LoadInt32(&errorTimes) >= MaxErrorTimes {
stop() //发停止指令
atomic.AddInt32(&errorTimes, MaxErrorTimes)
return
}
var path1, technologies1 = []string{}, []string{}
// 03-异常页面（>400），或相似度与404匹配
if fuzzPage.StatusCode >= 400 || bBig95 || fuzzPage.StatusCode != 200 {
// 03.01-异常页面指纹匹配
technologies = Addfingerprints404(technologies, req, fuzzPage) //基于404页面文件扫描指纹添加
// 03.02-与绝对404相似度低于0.8，添加body 404 body list
// 03.03-添加404titlelist
if 0.8 > fXsd && fuzzPage.StatusCode != 200 && fuzzPage.StatusCode != url404.StatusCode {
StudyErrPageAI(req, fuzzPage, "") // 异常页面学习
// 修复url，默认 认为 payload 不包含/
szUrl := u + payload
if strings.HasPrefix(payload, "/") && endP {
szUrl = u + payload[1:]
}
//log.Printf("start fuzz: [%s]", szUrl)
if fuzzPage, req, err := reqPage(szUrl); err == nil && nil != req && 0 < len(req.Body) {
if 200 == req.StatusCode {
if nil == lst200 {
lst200 = req
} else if lst200.Body == req.Body { // 无意义的 200
continue
}
if oU1, err := url.Parse(szUrl); nil == err {
a50 := r001.FindStringSubmatch(oU1.Path)
if 0 < len(a50) {
s2 := mime.TypeByExtension(filepath.Ext(a50[0]))
ct := (*req).Header.Get("Content-Type")
if "" != ct && "" != s2 && strings.Contains(ct, s2) {
continue
}
}
}
//log.Printf("%d : %s \n", req.StatusCode, szUrl)
if IsLoginPage(szUrl, req.Body, req.StatusCode) {
technologies = append(technologies, "loginpage")
}
}
go util.CheckHeader(req.Header, u)
// 02-状态码和req1相同，且与req1相似度>9.5，关闭所有fuzz
fXsd := strsim.Compare(url404req.Body, req.Body)
bBig95 := 9.5 < fXsd
//if "/bea_wls_internal/classes/mejb@/org/omg/stub/javax/management/j2ee/_ManagementHome_Stub.class" == payload {
// log.Println("start debug")
//}
if url404.StatusCode == fuzzPage.StatusCode && bBig95 {
stop() //发停止指令
atomic.AddInt32(&errorTimes, MaxErrorTimes)
return
}
// 04-403： 403 by pass
if fuzzPage.Is403 && !url404.Is403 {
a11 := ByPass403(&u, &payload, &wg)
// 表示 ByPass403 成功了, 结果、控制台输出点什么？
if 0 < len(a11) {
async_data <- &FuzzData{Path: &a11, Req: fuzzPage}
var path1, technologies1 = []string{}, []string{}
// 03-异常页面（>400），或相似度与404匹配
if fuzzPage.StatusCode >= 400 || bBig95 || fuzzPage.StatusCode != 200 {
// 03.01-异常页面指纹匹配
technologies = Addfingerprints404(technologies, req, fuzzPage) //基于404页面文件扫描指纹添加
// 03.02-与绝对404相似度低于0.8，添加body 404 body list
// 03.03-添加404titlelist
if 0.8 > fXsd && fuzzPage.StatusCode != 200 && fuzzPage.StatusCode != url404.StatusCode {
StudyErrPageAI(req, fuzzPage, "") // 异常页面学习
}
// 04-403： 403 by pass
if fuzzPage.Is403 && !url404.Is403 {
a11 := ByPass403(&u, &payload, &wg)
// 表示 ByPass403 成功了, 结果、控制台输出点什么？
if 0 < len(a11) {
async_data <- &FuzzData{Path: &a11, Req: fuzzPage}
}
}
return
}
// 当前和绝对404不等于404，后续的比较也没有意义了，都等于[200,301,302]都没有意义了，都说明没有fuzz成功
if url404.StatusCode != 404 && url404.StatusCode == fuzzPage.StatusCode {
return
}
return
}
// 当前和绝对404不等于404，后续的比较也没有意义了，都等于[200,301,302]都没有意义了，都说明没有fuzz成功
if url404.StatusCode != 404 && url404.StatusCode == fuzzPage.StatusCode {
return
}

// 05-跳转检测,即便是跳转，如果和绝对404不一样，说明检测成功
//if CheckDirckt(fuzzPage, req) && url404.StatusCode != fuzzPage.StatusCode {
// return
//}
// 1、状态码和绝对404一样 2、智能识别算出来
is404Page := url404.StatusCode == fuzzPage.StatusCode || CheckIsErrPageAI(req, fuzzPage)
// 06-成功页面, 非异常页面
if !is404Page || 200 == fuzzPage.StatusCode && url404.StatusCode != fuzzPage.StatusCode {
// 1、指纹匹配
technologies1 = Addfingerprintsnormal(payload, technologies1, req, fuzzPage) // 基于200页面文件扫描指纹添加
// 2、成功fuzz路径结果添加
path1 = append(path1, *fuzzPage.Url)
}
if 0 < len(path1) {
async_data <- &FuzzData{Path: &path1, Req: fuzzPage}
}
if 0 < len(technologies1) {
async_technologies <- technologies1
}
} else { // 这里应该元子操作
if nil != err {
//if nil != client && strings.Contains(err.Error(), " connect: connection reset by peer") {
// client.Client = client.GetClient(nil)
// 05-跳转检测,即便是跳转，如果和绝对404不一样，说明检测成功
//if CheckDirckt(fuzzPage, req) && url404.StatusCode != fuzzPage.StatusCode {
// return
//}
//log.Printf("file fuzz %s is err %v\n", szUrl, err)
// 1、状态码和绝对404一样 2、智能识别算出来
is404Page := url404.StatusCode == fuzzPage.StatusCode || CheckIsErrPageAI(req, fuzzPage)
// 06-成功页面, 非异常页面
if !is404Page || 200 == fuzzPage.StatusCode && url404.StatusCode != fuzzPage.StatusCode {
// 1、指纹匹配
technologies1 = Addfingerprintsnormal(payload, technologies1, req, fuzzPage) // 基于200页面文件扫描指纹添加
// 2、成功fuzz路径结果添加
path1 = append(path1, *fuzzPage.Url)
}
if 0 < len(path1) {
async_data <- &FuzzData{Path: &path1, Req: fuzzPage}
}
if 0 < len(technologies1) {
async_technologies <- technologies1
}
} else { // 这里应该元子操作
if nil != err {
//if nil != client && strings.Contains(err.Error(), " connect: connection reset by peer") {
// client.Client = client.GetClient(nil)
//}
//log.Printf("file fuzz %s is err %v\n", szUrl, err)
}
atomic.AddInt32(&errorTimes, 1)
}
atomic.AddInt32(&errorTimes, 1)
return
}
return
}
}
})
}(payload)
}
}
Expand Down Expand Up @@ -455,9 +462,10 @@ var reg1 = regexp.MustCompile("(?i)<meta.*http-equiv\\s*=\\s*\"refresh\".*conten
var reg2 = regexp.MustCompile("(window|self|top)\\.location\\.href\\s*=")

// 跳转检测
// 1、状态码跳转：301 代表永久性转移(Permanently Moved)；302 redirect: 302 代表暂时性转移(Temporarily Moved )
// 2、html刷新跳转
// 3、js 跳转
//
// 1、状态码跳转：301 代表永久性转移(Permanently Moved)；302 redirect: 302 代表暂时性转移(Temporarily Moved )
// 2、html刷新跳转
// 3、js 跳转
func CheckDirckt(fuzzPage *util.Page, req *util.Response) bool {
if nil == fuzzPage || nil == req {
return false
Expand Down
18 changes: 11 additions & 7 deletions brute/fuzzAI.go
View file
Expand Up @@ -33,11 +33,14 @@ var (
)

// 异常、404、500、505 标题、内容 存在到信息库
// 允许正则表达式
//
// 允许正则表达式
//
//go:embed dicts/fuzz404.txt
var fuzz404 string

// 常见404 url 列表,智能学习
//
//go:embed dicts/404url.txt
var sz404Url string

Expand All @@ -63,10 +66,11 @@ func init() {

// 智能学习: 非正常页面，并记录到库中永久使用,使用该方法到页面
// 要么是异常页面，要么是需要学习到指纹，带标记带
// 0、识别学习过的url就跳过
// 1、body 学习
// 2、标题 学习
// 3、url 去重记录
//
// 0、识别学习过的url就跳过
// 1、body 学习
// 2、标题 学习
// 3、url 去重记录
func StudyErrPageAI(req *util.Response, page *util.Page, fingerprintsTag string) {
if nil == req || nil == page || "" == req.Body {
return
Expand Down Expand Up @@ -138,10 +142,10 @@ func CheckIsErrPageAI(req *util.Response, page *util.Page) bool {
}
// 添加到 asz404Url, 保存到库中
if 404 == req.StatusCode {
go func() {
util.DefaultPool.Submit(func() {
asz404Url = append(asz404Url, u01.Path)
util.PutAny[[]string](asz404UrlKey, asz404Url) // 404 path 缓存起来，永久复用
}()
})
}
}
}
Expand Down

0 comments on commit 0520752

Please sign in to comment.