add httpx lib

Util/func.go

@@ -2,9 +2,13 @@ package Util
 
 import (
 	"archive/zip"
+	"bufio"
 	"bytes"
+	"github.com/projectdiscovery/httpx/runner"
+	"io/ioutil"
 	"math/rand"
 	"os"
+	"regexp"
 	"time"
 )
 
@@ -49,3 +53,60 @@ func CreateZipFile(FileName string, FileDate []byte) ([]byte, error) {
 	zipWriter.Close()
 	return buf.Bytes(), nil
 }
+
+func HttpXFileVerify(isFile bool, TagName string, Proxy string, Thread int) [][]string {
+	inputFile := "tmp.ipAddr"
+	if !isFile {
+		_ = os.WriteFile(inputFile, []byte(TagName), 0644)
+		defer os.RemoveAll(inputFile)
+	} else {
+		inputFile = TagName
+	}
+
+	outputFile := "ipAddr.tmp"
+
+	options := runner.Options{
+		Methods:         "GET",
+		InputFile:       inputFile,
+		ExtractTitle:    true,
+		StatusCode:      true,
+		FollowRedirects: true,
+		MaxRedirects:    10,
+		RandomAgent:     true,
+		Timeout:         5,
+		Output:          outputFile,
+		Threads:         Thread,
+	}
+	if Proxy != "" {
+		options.HTTPProxy = Proxy
+	}
+	if err := options.ValidateOptions(); err != nil {
+		panic(err)
+	}
+	HttpXRunner, err := runner.New(&options)
+	if err != nil {
+		panic(err)
+	}
+	defer HttpXRunner.Close()
+
+	HttpXRunner.RunEnumeration()
+	FileDate, _ := ioutil.ReadFile(outputFile)
+	regExp, _ := regexp.Compile(`\x1B\[\d+m`)
+	FileDate = regExp.ReplaceAll(FileDate, []byte(""))
+
+	_ = os.WriteFile(outputFile, FileDate, 0644)
+	defer os.RemoveAll(outputFile)
+
+	rf, err := os.OpenFile(outputFile, os.O_RDWR, 0666) //以读写方式打开文件
+	if nil != err {
+		panic(err)
+	}
+	defer rf.Close()
+	var ret [][]string
+	fc := bufio.NewScanner(rf) //按行读取文件内容
+	regExp, _ = regexp.Compile(`(?U)^(.*) \[(.*)] \[(.*)]`)
+	for fc.Scan() {
+		ret = append(ret, regExp.FindStringSubmatch(fc.Text())[1:])
+	}
+	return ret
+}

go.mod

@@ -3,11 +3,79 @@ module github.com/y-omicron/util
 go 1.18
 
 require (
-	github.com/alecthomas/kong v0.6.1
 	github.com/gookit/color v1.5.1
+	github.com/projectdiscovery/httpx v1.2.3
 )
 
 require (
+	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/akrylysov/pogreb v0.10.1 // indirect
+	github.com/ammario/ipisp/v2 v2.0.0 // indirect
+	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
+	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/bluele/gcache v0.0.2 // indirect
+	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
+	github.com/corpix/uarand v0.2.0 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/gorilla/css v1.0.0 // indirect
+	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
+	github.com/hdm/jarm-go v0.0.7 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
+	github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.19 // indirect
+	github.com/miekg/dns v1.1.48 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d // indirect
+	github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 // indirect
+	github.com/projectdiscovery/clistats v0.0.8 // indirect
+	github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect
+	github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 // indirect
+	github.com/projectdiscovery/fdmax v0.0.3 // indirect
+	github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c // indirect
+	github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c // indirect
+	github.com/projectdiscovery/goflags v0.0.8 // indirect
+	github.com/projectdiscovery/gologger v1.1.4 // indirect
+	github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa // indirect
+	github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 // indirect
+	github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 // indirect
+	github.com/projectdiscovery/mapcidr v1.0.1 // indirect
+	github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
+	github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e // indirect
+	github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect
+	github.com/projectdiscovery/retryabledns v1.0.13 // indirect
+	github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d // indirect
+	github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 // indirect
+	github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 // indirect
+	github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 // indirect
+	github.com/projectdiscovery/wappalyzergo v0.0.50 // indirect
+	github.com/remeh/sizedwaitgroup v1.0.0 // indirect
+	github.com/rs/xid v1.4.0 // indirect
+	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/syndtr/goleveldb v1.0.0 // indirect
+	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
+	github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 // indirect
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
-	golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 // indirect
+	github.com/yl2chen/cidranger v1.0.2 // indirect
+	github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect
+	github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/ratelimit v0.2.0 // indirect
+	golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )