diff --git a/GNUmakefile b/GNUmakefile
index ec24e54..90e5095 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -91,11 +91,13 @@ fetch-rdb:
integration/exploitdb.old fetch exploitdb --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
integration/exploitdb.old fetch githubrepos --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
integration/exploitdb.old fetch inthewild --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
+ integration/exploitdb.old fetch trickest --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
integration/exploitdb.new fetch awesomepoc --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
integration/exploitdb.new fetch exploitdb --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
integration/exploitdb.new fetch githubrepos --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
integration/exploitdb.new fetch inthewild --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
+ integration/exploitdb.new fetch trickest --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
fetch-redis:
docker run --name redis-old -d -p 127.0.0.1:6379:6379 redis
@@ -105,11 +107,13 @@ fetch-redis:
integration/exploitdb.old fetch exploitdb --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
integration/exploitdb.old fetch githubrepos --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
integration/exploitdb.old fetch inthewild --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
+ integration/exploitdb.old fetch trickest --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
integration/exploitdb.new fetch awesomepoc --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
integration/exploitdb.new fetch exploitdb --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
integration/exploitdb.new fetch githubrepos --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
integration/exploitdb.new fetch inthewild --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
+ integration/exploitdb.new fetch trickest --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
diff-cveid:
@ python integration/diff_server_mode.py cveid --sample_rate 0.01
diff --git a/README.md b/README.md
index 91d3ac6..19d83d4 100644
--- a/README.md
+++ b/README.md
@@ -17,9 +17,10 @@ In server mode, a simple Web API can be used.
As the following vulnerabilities database
1. [ExploitDB(OffensiveSecurity)](https://www.exploit-db.com/) by CVE number or Exploit Database ID.
-2. [GitHub Repositories](https://github.com/search?o=desc&q=CVE&s=&type=Repositories)
+2. [GitHub Repositories](https://github.com/nomi-sec/PoC-in-GitHub.git)
3. [Awesome Cve Poc](https://github.com/qazbnm456/awesome-cve-poc#toc473)
4. [inTheWild DB](https://github.com/gmatuz/inthewilddb)
+5. [Trickest CVE](https://github.com/trickest/cve.git)
### Docker Deployment
There's a Docker image available `docker pull vulsio/go-exploitdb`. When using the container, it takes the same arguments as the [normal command line](#Usage).
@@ -56,6 +57,7 @@ Available Commands:
exploitdb Fetch the data of offensive security exploit db
githubrepos Fetch the data of github repos
inthewild Fetch the data of inTheWild Poc
+ trickest Fetch the data of trickest PoCs
Flags:
--batch-size int The number of batch size to insert. (default 500)
diff --git a/commands/fetch-trickest.go b/commands/fetch-trickest.go
new file mode 100644
index 0000000..e2c4ede
--- /dev/null
+++ b/commands/fetch-trickest.go
@@ -0,0 +1,75 @@
+package commands
+
+import (
+ "time"
+
+ "github.com/inconshreveable/log15"
+ "github.com/spf13/cobra"
+ "github.com/spf13/viper"
+ "github.com/vulsio/go-exploitdb/db"
+ "github.com/vulsio/go-exploitdb/fetcher"
+ "github.com/vulsio/go-exploitdb/models"
+ "github.com/vulsio/go-exploitdb/util"
+ "golang.org/x/xerrors"
+)
+
+var fetchTrickestCmd = &cobra.Command{
+ Use: "trickest",
+ Short: "Fetch the data of trickest PoCs",
+ Long: `Fetch the data of trickest PoCs`,
+ RunE: fetchTrickest,
+}
+
+func init() {
+ fetchCmd.AddCommand(fetchTrickestCmd)
+}
+
+func fetchTrickest(_ *cobra.Command, _ []string) (err error) {
+ if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+ return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+ }
+
+ driver, locked, err := db.NewDB(
+ viper.GetString("dbtype"),
+ viper.GetString("dbpath"),
+ viper.GetBool("debug-sql"),
+ db.Option{},
+ )
+ if err != nil {
+ if locked {
+ return xerrors.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %w", err)
+ }
+ return xerrors.Errorf("Failed to open DB. err: %w", err)
+ }
+
+ fetchMeta, err := driver.GetFetchMeta()
+ if err != nil {
+ return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
+ }
+ if fetchMeta.OutDated() {
+ return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
+ }
+ // If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion.
+ if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+ return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+ }
+
+ log15.Info("Fetching Trickest Exploit")
+ var exploits []models.Exploit
+ if exploits, err = fetcher.FetchTrickest(); err != nil {
+ return xerrors.Errorf("Failed to fetch Trickest Exploit. err: %w", err)
+ }
+ log15.Info("Trickest Exploit", "count", len(exploits))
+
+ log15.Info("Insert Exploit into go-exploitdb.", "db", driver.Name())
+ if err := driver.InsertExploit(models.TrickestType, exploits); err != nil {
+ return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+ }
+
+ fetchMeta.LastFetchedAt = time.Now()
+ if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+ return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+ }
+
+ return nil
+}
diff --git a/db/db.go b/db/db.go
index 2468a94..6a4ada9 100644
--- a/db/db.go
+++ b/db/db.go
@@ -26,6 +26,7 @@ type DB interface {
UpsertFetchMeta(*models.FetchMeta) error
}
+// Option :
type Option struct {
RedisTimeout time.Duration
}
diff --git a/db/rdb.go b/db/rdb.go
index f8d8666..c17dacf 100644
--- a/db/rdb.go
+++ b/db/rdb.go
@@ -119,6 +119,10 @@ func (r *RDBDriver) MigrateDB() error {
&models.Paper{},
&models.GitHubRepository{},
&models.InTheWild{},
+ &models.Trickest{},
+ &models.TrickestProduct{},
+ &models.TrickestVersion{},
+ &models.TrickestVulnerability{},
); err != nil {
return xerrors.Errorf("Failed to migrate. err: %w", err)
}
@@ -186,6 +190,16 @@ func (r *RDBDriver) deleteAndInsertExploit(exploitType models.ExploitType, explo
}
}
+ trIDs := []models.Trickest{}
+ if err := tx.Model(&models.Trickest{}).Select("id").Where("exploit_id IN ?", oldIDs[idx.From:idx.To]).Find(&trIDs).Error; err != nil {
+ return xerrors.Errorf("Failed to select old Trickest: %w", err)
+ }
+ if len(trIDs) > 0 {
+ if err := tx.Select(clause.Associations).Delete(&trIDs).Error; err != nil {
+ return xerrors.Errorf("Failed to delete: %w", err)
+ }
+ }
+
if err := tx.Where("id IN ?", oldIDs[idx.From:idx.To]).Delete(&models.Exploit{}).Error; err != nil {
return xerrors.Errorf("Failed to delete: %w", err)
}
@@ -236,7 +250,9 @@ func (r *RDBDriver) GetExploitByID(exploitUniqueID string) ([]models.Exploit, er
return nil, xerrors.Errorf("Failed to get OffensiveSecurity. err: %w", err)
}
case models.GitHubRepositoryType:
- if err := r.conn.Where(&models.GitHubRepository{ExploitID: es[i].ID}).Take(&es[i].GitHubRepository).Error; err != nil {
+ if err := r.conn.
+ Where(&models.GitHubRepository{ExploitID: es[i].ID}).
+ Take(&es[i].GitHubRepository).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, xerrors.Errorf("Failed to get GitHubRepository. DB relationship may be broken, use `$ go-exploitdb fetch githubrepos` to recreate DB. err: %w", err)
}
@@ -249,6 +265,16 @@ func (r *RDBDriver) GetExploitByID(exploitUniqueID string) ([]models.Exploit, er
}
return nil, xerrors.Errorf("Failed to get inTheWild. err: %w", err)
}
+ case models.TrickestType:
+ if err := r.conn.
+ Preload(clause.Associations).
+ Where(&models.Trickest{ExploitID: es[i].ID}).
+ Take(&es[i].Trickest).Error; err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil, xerrors.Errorf("Failed to get Trickest. DB relationship may be broken, use `$ go-exploitdb fetch trickest` to recreate DB. err: %w", err)
+ }
+ return nil, xerrors.Errorf("Failed to get Trickest. err: %w", err)
+ }
}
}
return es, nil
@@ -281,7 +307,9 @@ func (r *RDBDriver) GetExploitAll() ([]models.Exploit, error) {
return nil, xerrors.Errorf("Failed to Get OffensiveSecurity. err: %w", err)
}
case models.GitHubRepositoryType:
- if err := r.conn.Where(&models.GitHubRepository{ExploitID: exploit.ID}).Take(&exploit.GitHubRepository).Error; err != nil {
+ if err := r.conn.
+ Where(&models.GitHubRepository{ExploitID: exploit.ID}).
+ Take(&exploit.GitHubRepository).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, xerrors.Errorf("Failed to get GitHubRepository. DB relationship may be broken, use `$ go-exploitdb fetch githubrepos` to recreate DB. err: %w", err)
}
@@ -294,6 +322,16 @@ func (r *RDBDriver) GetExploitAll() ([]models.Exploit, error) {
}
return nil, xerrors.Errorf("Failed to Get inTheWild. err: %w", err)
}
+ case models.TrickestType:
+ if err := r.conn.
+ Preload(clause.Associations).
+ Where(&models.Trickest{ExploitID: exploit.ID}).
+ Take(&exploit.Trickest).Error; err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil, xerrors.Errorf("Failed to get Trickest. DB relationship may be broken, use `$ go-exploitdb fetch trickest` to recreate DB. err: %w", err)
+ }
+ return nil, xerrors.Errorf("Failed to get Trickest. err: %w", err)
+ }
}
es = append(es, exploit)
}
@@ -335,7 +373,9 @@ func (r *RDBDriver) GetExploitByCveID(cveID string) ([]models.Exploit, error) {
return nil, xerrors.Errorf("Failed to get OffensiveSecurity. err: %w", err)
}
case models.GitHubRepositoryType:
- if err := r.conn.Where(&models.GitHubRepository{ExploitID: es[i].ID}).Take(&es[i].GitHubRepository).Error; err != nil {
+ if err := r.conn.
+ Where(&models.GitHubRepository{ExploitID: es[i].ID}).
+ Take(&es[i].GitHubRepository).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, xerrors.Errorf("Failed to get GitHubRepository. DB relationship may be broken, use `$ go-exploitdb fetch githubrepos` to recreate DB. err: %w", err)
}
@@ -348,6 +388,16 @@ func (r *RDBDriver) GetExploitByCveID(cveID string) ([]models.Exploit, error) {
}
return nil, xerrors.Errorf("Failed to get inTheWild. err: %w", err)
}
+ case models.TrickestType:
+ if err := r.conn.
+ Preload(clause.Associations).
+ Where(&models.Trickest{ExploitID: es[i].ID}).
+ Take(&es[i].Trickest).Error; err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil, xerrors.Errorf("Failed to get Trickest. DB relationship may be broken, use `$ go-exploitdb fetch trickest` to recreate DB. err: %w", err)
+ }
+ return nil, xerrors.Errorf("Failed to get Trickest. err: %w", err)
+ }
}
}
return es, nil
diff --git a/fetcher/githubrepos.go b/fetcher/githubrepos.go
index b4b6441..2e7f313 100644
--- a/fetcher/githubrepos.go
+++ b/fetcher/githubrepos.go
@@ -18,14 +18,14 @@ import (
"golang.org/x/xerrors"
)
-const repoURL string = "https://github.com/nomi-sec/PoC-in-GitHub.git"
+const githubURL string = "https://github.com/nomi-sec/PoC-in-GitHub.git"
// FetchGitHubRepos :
func FetchGitHubRepos(stars, forks int) (exploits []models.Exploit, err error) {
- dir := filepath.Join(util.CacheDir(), "pocs")
- updatedFiles, err := git.CloneOrPull(repoURL, dir)
+ dir := filepath.Join(util.CacheDir(), "github")
+ updatedFiles, err := git.CloneOrPull(githubURL, dir)
if err != nil {
- return nil, xerrors.Errorf("error in pocsrc clone or pull: %w", err)
+ return nil, xerrors.Errorf("Failed to clone or pull nomi-sec/PoC-in-GitHub repository. err: %w", err)
}
var targets []map[string]struct{}
@@ -45,11 +45,11 @@ func FetchGitHubRepos(stars, forks int) (exploits []models.Exploit, err error) {
log15.Debug("PoC-in-GitHub: no updated file")
return nil, nil
}
- log15.Debug(fmt.Sprintf("PoC-in-GitHub updated files: %d", targetFiles))
+ log15.Debug(fmt.Sprintf("PoC-in-GitHub: updated files: %d", targetFiles))
entries := map[string][]models.GitHubRepoJSON{}
for _, target := range targets {
- err = util.FileWalk(dir, target, func(r io.Reader, path string) error {
+ if err := util.FileWalk(dir, target, func(r io.Reader, path string) error {
content, err := ioutil.ReadAll(r)
if err != nil {
return err
@@ -64,9 +64,8 @@ func FetchGitHubRepos(stars, forks int) (exploits []models.Exploit, err error) {
cveID := strings.Split(dirPaths[len(dirPaths)-1], ".")[0]
entries[cveID] = pocs
return nil
- })
- if err != nil {
- return nil, xerrors.Errorf("error in PoC-in-GitHub walk: %w", err)
+ }); err != nil {
+ return nil, xerrors.Errorf("Failed to walk PoC-in-GitHub. err: %w", err)
}
}
diff --git a/fetcher/trickest.go b/fetcher/trickest.go
new file mode 100644
index 0000000..0808c51
--- /dev/null
+++ b/fetcher/trickest.go
@@ -0,0 +1,193 @@
+package fetcher
+
+import (
+ "bufio"
+ "bytes"
+ "crypto/md5"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/url"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/PuerkitoBio/goquery"
+ "github.com/inconshreveable/log15"
+ "github.com/russross/blackfriday/v2"
+ "golang.org/x/xerrors"
+
+ "github.com/vulsio/go-exploitdb/git"
+ "github.com/vulsio/go-exploitdb/models"
+ "github.com/vulsio/go-exploitdb/util"
+)
+
+const trickestURL string = "https://github.com/trickest/cve.git"
+
+type trickestCVE struct {
+ CVEID string
+ Description string
+ Products []string
+ Versions []string
+ Vulnerabilities []string
+ PoCs []string
+}
+
+// FetchTrickest :
+func FetchTrickest() ([]models.Exploit, error) {
+ dir := filepath.Join(util.CacheDir(), "trickest")
+ updatedFiles, err := git.CloneOrPull(trickestURL, dir)
+ if err != nil {
+ return nil, xerrors.Errorf("Failed to clone or pull trickest/cve repository. err: %w", err)
+ }
+
+ var targets []map[string]struct{}
+ targetFiles := 0
+ for year := 1999; year <= time.Now().Year(); year++ {
+ target, err := util.FilterTargets(fmt.Sprint(year), updatedFiles)
+ if err != nil {
+ return nil, xerrors.Errorf("Failed to filter target files: %w", err)
+ }
+
+ if len(target) != 0 {
+ targets = append(targets, target)
+ targetFiles += len(target)
+ }
+ }
+ if targetFiles == 0 {
+ log15.Debug("Trickest CVE: no updated file")
+ return nil, nil
+ }
+ log15.Debug(fmt.Sprintf("Trickest CVE: updated files: %d", targetFiles))
+
+ entries := []trickestCVE{}
+ for _, target := range targets {
+ if err = util.FileWalk(dir, target, func(r io.Reader, path string) error {
+ content, err := ioutil.ReadAll(r)
+ if err != nil {
+ return xerrors.Errorf("Failed to Read file. path: %s, err: %w", path, err)
+ }
+
+ trickest, err := readTrickestHTML(splitTrickestMD(content))
+ if err != nil {
+ return xerrors.Errorf("Failed to read Trickest HTML. err: %w", err)
+ }
+
+ entries = append(entries, trickest)
+ return nil
+ }); err != nil {
+ return nil, xerrors.Errorf("Failed to walk Trickest CVE. err: %w", err)
+ }
+ }
+
+ exploits := []models.Exploit{}
+ for _, e := range entries {
+ products := []models.TrickestProduct{}
+ for _, product := range e.Products {
+ products = append(products, models.TrickestProduct{Product: product})
+ }
+ versions := []models.TrickestVersion{}
+ for _, version := range e.Versions {
+ versions = append(versions, models.TrickestVersion{Version: version})
+ }
+ vulnerabilities := []models.TrickestVulnerability{}
+ for _, vulnerability := range e.Vulnerabilities {
+ vulnerabilities = append(vulnerabilities, models.TrickestVulnerability{Vulnerability: vulnerability})
+ }
+
+ for _, poc := range e.PoCs {
+ exploits = append(exploits, models.Exploit{
+ ExploitType: models.TrickestType,
+ ExploitUniqueID: fmt.Sprintf("%s-%x", models.TrickestType, md5.Sum([]byte(fmt.Sprintf("%s-%s", e.CVEID, poc)))),
+ URL: poc,
+ Description: e.Description,
+ CveID: e.CVEID,
+ Trickest: &models.Trickest{
+ Products: append([]models.TrickestProduct{}, products...),
+ Versions: append([]models.TrickestVersion{}, versions...),
+ Vulnerabilities: append([]models.TrickestVulnerability{}, vulnerabilities...),
+ },
+ })
+ }
+ }
+ return exploits, nil
+}
+
+func splitTrickestMD(content []byte) (string, string, string) {
+ cveHTML := []string{}
+ descHTML := []string{}
+ pocHTML := []string{}
+ mode := "cve"
+ s := bufio.NewScanner(bytes.NewReader(blackfriday.Run(content)))
+ for s.Scan() {
+ t := s.Text()
+ if strings.HasPrefix(t, "Description
") {
+ mode = "desc"
+ }
+ if strings.HasPrefix(t, "POC
") {
+ mode = "poc"
+ }
+ switch mode {
+ case "cve":
+ cveHTML = append(cveHTML, t)
+ case "desc":
+ descHTML = append(descHTML, t)
+ case "poc":
+ pocHTML = append(pocHTML, t)
+ }
+ }
+
+ return strings.Join(cveHTML, ""), strings.Join(descHTML, ""), strings.Join(pocHTML, "")
+}
+
+func readTrickestHTML(cveHTML, descHTML, pocHTML string) (trickestCVE, error) {
+ var trickest trickestCVE
+ cveDoc, err := goquery.NewDocumentFromReader(strings.NewReader(cveHTML))
+ if err != nil {
+ return trickestCVE{}, xerrors.Errorf("Failed to read CVE content. err: %w", err)
+ }
+ trickest.CVEID = cveDoc.Find("h3").Text()
+ cveDoc.Find("img").Each(func(_ int, s *goquery.Selection) {
+ src := s.AttrOr("src", "")
+ if src == "" {
+ return
+ }
+ u, err := url.Parse(src)
+ if err != nil {
+ log15.Debug(fmt.Sprintf("Trickest CVE: failed to parse img src: %s", err))
+ return
+ }
+ l := u.Query().Get("label")
+ m := u.Query().Get("message")
+ if l == "" || m == "" {
+ return
+ }
+
+ switch l {
+ case "Product":
+ trickest.Products = append(trickest.Products, m)
+ case "Version":
+ trickest.Versions = append(trickest.Versions, m)
+ case "Vulnerability":
+ trickest.Vulnerabilities = append(trickest.Vulnerabilities, m)
+ default:
+ log15.Debug(fmt.Sprintf("Trickest CVE: new label: %s", l))
+ }
+ })
+
+ descDoc, err := goquery.NewDocumentFromReader(strings.NewReader(descHTML))
+ if err != nil {
+ return trickestCVE{}, xerrors.Errorf("Failed to read Description content. err: %w", err)
+ }
+ trickest.Description = descDoc.Find("p").Text()
+
+ pocDoc, err := goquery.NewDocumentFromReader(strings.NewReader(pocHTML))
+ if err != nil {
+ return trickestCVE{}, xerrors.Errorf("Failed to read PoCs content. err: %w", err)
+ }
+ pocDoc.Find("a").Each(func(_ int, s *goquery.Selection) {
+ trickest.PoCs = append(trickest.PoCs, s.Text())
+ })
+
+ return trickest, nil
+}
diff --git a/fetcher/trickest_test.go b/fetcher/trickest_test.go
new file mode 100644
index 0000000..5b894c4
--- /dev/null
+++ b/fetcher/trickest_test.go
@@ -0,0 +1,98 @@
+package fetcher
+
+import (
+ "reflect"
+ "testing"
+)
+
+func Test_splitTrickestMD(t *testing.T) {
+ type expected struct {
+ cveHTML string
+ descHTML string
+ pocHTML string
+ }
+
+ var tests = []struct {
+ in string
+ expected expected
+ }{
+ {
+ in: `### [CVE-1999-0001](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-1999-0001)
+
+
+
+
+### Description
+
+ip_input.c in BSD-derived TCP/IP implementations allows remote attackers to cause a denial of service (crash or hang) via crafted packets.
+
+### POC
+
+#### Reference
+No PoCs from references.
+
+#### Github
+- https://github.com/flowground/anchore-io-connector
+- https://github.com/jimmyislive/gocve
+- https://github.com/quentinmayo/get_nvd_data_from_online_to_csv
+
+`,
+ expected: expected{
+ cveHTML: `
`,
+ descHTML: `Description
ip_input.c in BSD-derived TCP/IP implementations allows remote attackers to cause a denial of service (crash or hang) via crafted packets.
`,
+ pocHTML: `POC
Reference
No PoCs from references.
Github
`,
+ },
+ },
+ }
+ for i, tt := range tests {
+ cve, desc, poc := splitTrickestMD([]byte(tt.in))
+ if cve != tt.expected.cveHTML {
+ t.Errorf("[%d] splitTrickestMD expected CVE: %s\n actual: %s\n", i, tt.expected.cveHTML, cve)
+ }
+ if desc != tt.expected.descHTML {
+ t.Errorf("[%d] splitTrickestMD expected Desciption: %s\n actual: %s\n", i, tt.expected.descHTML, desc)
+ }
+ if poc != tt.expected.pocHTML {
+ t.Errorf("[%d] splitTrickestMD expected PoC: %s\n actual: %s\n", i, tt.expected.pocHTML, poc)
+ }
+ }
+}
+
+func Test_readTrickestHTML(t *testing.T) {
+ type args struct {
+ cveHTML string
+ descHTML string
+ pocHTML string
+ }
+
+ var tests = []struct {
+ in args
+ expected trickestCVE
+ }{
+ {
+ in: args{
+ cveHTML: `
`,
+ descHTML: `Description
ip_input.c in BSD-derived TCP/IP implementations allows remote attackers to cause a denial of service (crash or hang) via crafted packets.
`,
+ pocHTML: `POC
Reference
No PoCs from references.
Github
`,
+ },
+ expected: trickestCVE{
+ CVEID: "CVE-1999-0001",
+ Description: "ip_input.c in BSD-derived TCP/IP implementations allows remote attackers to cause a denial of service (crash or hang) via crafted packets.",
+ Products: []string{"n/a"},
+ Versions: []string{"n/a"},
+ Vulnerabilities: []string{"n/a"},
+ PoCs: []string{"https://github.com/flowground/anchore-io-connector", "https://github.com/jimmyislive/gocve", "https://github.com/quentinmayo/get_nvd_data_from_online_to_csv"},
+ },
+ },
+ }
+ for i, tt := range tests {
+ trickestCVE, err := readTrickestHTML(tt.in.cveHTML, tt.in.descHTML, tt.in.pocHTML)
+ if err != nil {
+ t.Errorf("[%d] readTrickestHTML error: %s\n", i, err)
+ continue
+ }
+ if !reflect.DeepEqual(trickestCVE, tt.expected) {
+ t.Errorf("[%d] readTrickestHTML expected: %#v\n actual: %#v\n", i, tt.expected, trickestCVE)
+ }
+ }
+}
diff --git a/git/git.go b/git/git.go
index 11858cc..d064861 100644
--- a/git/git.go
+++ b/git/git.go
@@ -172,7 +172,7 @@ func pullByOSCommand(repoPath string) ([]string, error) {
}
commitHash := strings.TrimSpace(output)
- pullCmd := []string{"pull", "origin", "master"}
+ pullCmd := []string{"pull", "origin", "HEAD"}
_, err = util.Exec("git", append(commandArgs, pullCmd...))
if err != nil {
return nil, xerrors.Errorf("error in git pull: %w", err)
diff --git a/go.mod b/go.mod
index 7a308b5..0a47d96 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.15
require (
github.com/Microsoft/go-winio v0.5.0 // indirect
- github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
+ github.com/PuerkitoBio/goquery v1.8.0
github.com/cheggaaa/pb/v3 v3.0.5
github.com/elazarl/goproxy v0.0.0-20210110162100-a92cc753f88e // indirect
github.com/go-redis/redis/v8 v8.4.10
@@ -17,7 +17,6 @@ require (
github.com/mattn/go-sqlite3 v1.14.7
github.com/mitchellh/go-homedir v1.1.0
github.com/parnurzeal/gorequest v0.2.16
- github.com/pkg/errors v0.9.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0
github.com/sergi/go-diff v1.2.0 // indirect
github.com/spf13/cast v1.4.0 // indirect
@@ -26,7 +25,7 @@ require (
github.com/valyala/fasttemplate v1.2.1 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
- golang.org/x/net v0.0.0-20210614182718-04defd469f4e
+ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
gopkg.in/src-d/go-git.v4 v4.13.1
diff --git a/go.sum b/go.sum
index 38f8d20..8ee403c 100644
--- a/go.sum
+++ b/go.sum
@@ -45,6 +45,8 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU=
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
@@ -57,6 +59,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -68,8 +72,6 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
-github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
-github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
@@ -693,8 +695,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
diff --git a/models/exploit.go b/models/exploit.go
index f378614..c25ec65 100644
--- a/models/exploit.go
+++ b/models/exploit.go
@@ -19,6 +19,8 @@ var (
AwesomePocType ExploitType = "AwesomePoc"
// InTheWildType :
InTheWildType ExploitType = "InTheWild"
+ // TrickestType :
+ TrickestType ExploitType = "Trickest"
)
// Exploit :
@@ -32,6 +34,7 @@ type Exploit struct {
OffensiveSecurity *OffensiveSecurity `json:"offensive_security"`
GitHubRepository *GitHubRepository `json:"github_repository"`
InTheWild *InTheWild `json:"in_the_wild"`
+ Trickest *Trickest `json:"trickest"`
}
// GitHubRepository :
@@ -106,6 +109,36 @@ type InTheWild struct {
Type string `gorm:"type:varchar(255)" json:"type"`
}
+// Trickest :
+type Trickest struct {
+ ID int64 `json:"-"`
+ ExploitID int64 `gorm:"index:idx_trickest_exploit_id" json:"-"`
+ Products []TrickestProduct `json:"products"`
+ Versions []TrickestVersion `json:"versions"`
+ Vulnerabilities []TrickestVulnerability `json:"vulnerabilities"`
+}
+
+// TrickestProduct :
+type TrickestProduct struct {
+ ID int64 `json:"-"`
+ TrickestID int64 `json:"-"`
+ Product string `gorm:"type:varchar(255)" json:"product"`
+}
+
+// TrickestVersion :
+type TrickestVersion struct {
+ ID int64 `json:"-"`
+ TrickestID int64 `json:"-"`
+ Version string `gorm:"type:varchar(255)" json:"version"`
+}
+
+// TrickestVulnerability :
+type TrickestVulnerability struct {
+ ID int64 `json:"-"`
+ TrickestID int64 `json:"-"`
+ Vulnerability string `gorm:"type:varchar(255)" json:"vulnerability"`
+}
+
// MitreXML :
// http://cve.mitre.org/cve/cvrf.html
type MitreXML struct {
diff --git a/util/util.go b/util/util.go
index eb01df9..8a9a2d8 100644
--- a/util/util.go
+++ b/util/util.go
@@ -85,6 +85,7 @@ func CacheDir() string {
return filepath.Join(tmpDir, "go-exploitdb")
}
+// IsCommandAvailable :
func IsCommandAvailable(name string) bool {
cmd := exec.Command(name, "--help")
if err := cmd.Run(); err != nil {
@@ -93,6 +94,7 @@ func IsCommandAvailable(name string) bool {
return true
}
+// Exists :
func Exists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
@@ -104,6 +106,7 @@ func Exists(path string) (bool, error) {
return true, err
}
+// Exec :
func Exec(command string, args []string) (string, error) {
cmd := exec.Command(command, args...)
var stdoutBuf, stderrBuf bytes.Buffer