feat(trickest): support Trickest PoCs

GNUmakefile

@@ -91,11 +91,13 @@ fetch-rdb:
 	integration/exploitdb.old fetch exploitdb --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
 	integration/exploitdb.old fetch githubrepos --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
 	integration/exploitdb.old fetch inthewild --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
+	integration/exploitdb.old fetch trickest --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
 
 	integration/exploitdb.new fetch awesomepoc --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
 	integration/exploitdb.new fetch exploitdb --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
 	integration/exploitdb.new fetch githubrepos --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
 	integration/exploitdb.new fetch inthewild --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
+	integration/exploitdb.new fetch trickest --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
 
 fetch-redis:
 	docker run --name redis-old -d -p 127.0.0.1:6379:6379 redis
@@ -105,11 +107,13 @@ fetch-redis:
 	integration/exploitdb.old fetch exploitdb --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
 	integration/exploitdb.old fetch githubrepos --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
 	integration/exploitdb.old fetch inthewild --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
+	integration/exploitdb.old fetch trickest --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
 
 	integration/exploitdb.new fetch awesomepoc --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
 	integration/exploitdb.new fetch exploitdb --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
 	integration/exploitdb.new fetch githubrepos --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
 	integration/exploitdb.new fetch inthewild --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
+	integration/exploitdb.new fetch trickest --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
 
 diff-cveid:
 	@ python integration/diff_server_mode.py cveid --sample_rate 0.01

README.md

@@ -17,9 +17,10 @@ In server mode, a simple Web API can be used.
 As the following vulnerabilities database
 
 1. [ExploitDB(OffensiveSecurity)](https://www.exploit-db.com/) by CVE number or Exploit Database ID.
-2. [GitHub Repositories](https://github.com/search?o=desc&q=CVE&s=&type=Repositories)
+2. [GitHub Repositories](https://github.com/nomi-sec/PoC-in-GitHub.git)
 3. [Awesome Cve Poc](https://github.com/qazbnm456/awesome-cve-poc#toc473)
 4. [inTheWild DB](https://github.com/gmatuz/inthewilddb)
+5. [Trickest CVE](https://github.com/trickest/cve.git)
 
 ### Docker Deployment
 There's a Docker image available `docker pull vulsio/go-exploitdb`. When using the container, it takes the same arguments as the [normal command line](#Usage).
@@ -56,6 +57,7 @@ Available Commands:
   exploitdb   Fetch the data of offensive security exploit db
   githubrepos Fetch the data of github repos
   inthewild   Fetch the data of inTheWild Poc
+  trickest    Fetch the data of trickest PoCs
 
 Flags:
       --batch-size int   The number of batch size to insert. (default 500)

commands/fetch-trickest.go

@@ -0,0 +1,75 @@
+package commands
+
+import (
+	"time"
+
+	"github.com/inconshreveable/log15"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/vulsio/go-exploitdb/db"
+	"github.com/vulsio/go-exploitdb/fetcher"
+	"github.com/vulsio/go-exploitdb/models"
+	"github.com/vulsio/go-exploitdb/util"
+	"golang.org/x/xerrors"
+)
+
+var fetchTrickestCmd = &cobra.Command{
+	Use:   "trickest",
+	Short: "Fetch the data of trickest PoCs",
+	Long:  `Fetch the data of trickest PoCs`,
+	RunE:  fetchTrickest,
+}
+
+func init() {
+	fetchCmd.AddCommand(fetchTrickestCmd)
+}
+
+func fetchTrickest(_ *cobra.Command, _ []string) (err error) {
+	if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
+	driver, locked, err := db.NewDB(
+		viper.GetString("dbtype"),
+		viper.GetString("dbpath"),
+		viper.GetBool("debug-sql"),
+		db.Option{},
+	)
+	if err != nil {
+		if locked {
+			return xerrors.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %w", err)
+		}
+		return xerrors.Errorf("Failed to open DB. err: %w", err)
+	}
+
+	fetchMeta, err := driver.GetFetchMeta()
+	if err != nil {
+		return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
+	}
+	if fetchMeta.OutDated() {
+		return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
+	}
+	// If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion.
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	log15.Info("Fetching Trickest Exploit")
+	var exploits []models.Exploit
+	if exploits, err = fetcher.FetchTrickest(); err != nil {
+		return xerrors.Errorf("Failed to fetch Trickest Exploit. err: %w", err)
+	}
+	log15.Info("Trickest Exploit", "count", len(exploits))
+
+	log15.Info("Insert Exploit into go-exploitdb.", "db", driver.Name())
+	if err := driver.InsertExploit(models.TrickestType, exploits); err != nil {
+		return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	fetchMeta.LastFetchedAt = time.Now()
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	return nil
+}

db/db.go

@@ -26,6 +26,7 @@ type DB interface {
 	UpsertFetchMeta(*models.FetchMeta) error
 }
 
+// Option :
 type Option struct {
 	RedisTimeout time.Duration
 }

db/rdb.go

@@ -119,6 +119,10 @@ func (r *RDBDriver) MigrateDB() error {
 		&models.Paper{},
 		&models.GitHubRepository{},
 		&models.InTheWild{},
+		&models.Trickest{},
+		&models.TrickestProduct{},
+		&models.TrickestVersion{},
+		&models.TrickestVulnerability{},
 	); err != nil {
 		return xerrors.Errorf("Failed to migrate. err: %w", err)
 	}
@@ -186,6 +190,16 @@ func (r *RDBDriver) deleteAndInsertExploit(exploitType models.ExploitType, explo
 				}
 			}
 
+			trIDs := []models.Trickest{}
+			if err := tx.Model(&models.Trickest{}).Select("id").Where("exploit_id IN ?", oldIDs[idx.From:idx.To]).Find(&trIDs).Error; err != nil {
+				return xerrors.Errorf("Failed to select old Trickest: %w", err)
+			}
+			if len(trIDs) > 0 {
+				if err := tx.Select(clause.Associations).Delete(&trIDs).Error; err != nil {
+					return xerrors.Errorf("Failed to delete: %w", err)
+				}
+			}
+
 			if err := tx.Where("id IN ?", oldIDs[idx.From:idx.To]).Delete(&models.Exploit{}).Error; err != nil {
 				return xerrors.Errorf("Failed to delete: %w", err)
 			}
@@ -236,7 +250,9 @@ func (r *RDBDriver) GetExploitByID(exploitUniqueID string) ([]models.Exploit, er
 				return nil, xerrors.Errorf("Failed to get OffensiveSecurity. err: %w", err)
 			}
 		case models.GitHubRepositoryType:
-			if err := r.conn.Where(&models.GitHubRepository{ExploitID: es[i].ID}).Take(&es[i].GitHubRepository).Error; err != nil {
+			if err := r.conn.
+				Where(&models.GitHubRepository{ExploitID: es[i].ID}).
+				Take(&es[i].GitHubRepository).Error; err != nil {
 				if errors.Is(err, gorm.ErrRecordNotFound) {
 					return nil, xerrors.Errorf("Failed to get GitHubRepository. DB relationship may be broken, use `$ go-exploitdb fetch githubrepos` to recreate DB. err: %w", err)
 				}
@@ -249,6 +265,16 @@ func (r *RDBDriver) GetExploitByID(exploitUniqueID string) ([]models.Exploit, er
 				}
 				return nil, xerrors.Errorf("Failed to get inTheWild. err: %w", err)
 			}
+		case models.TrickestType:
+			if err := r.conn.
+				Preload(clause.Associations).
+				Where(&models.Trickest{ExploitID: es[i].ID}).
+				Take(&es[i].Trickest).Error; err != nil {
+				if errors.Is(err, gorm.ErrRecordNotFound) {
+					return nil, xerrors.Errorf("Failed to get Trickest. DB relationship may be broken, use `$ go-exploitdb fetch trickest` to recreate DB. err: %w", err)
+				}
+				return nil, xerrors.Errorf("Failed to get Trickest. err: %w", err)
+			}
 		}
 	}
 	return es, nil
@@ -281,7 +307,9 @@ func (r *RDBDriver) GetExploitAll() ([]models.Exploit, error) {
 				return nil, xerrors.Errorf("Failed to Get OffensiveSecurity. err: %w", err)
 			}
 		case models.GitHubRepositoryType:
-			if err := r.conn.Where(&models.GitHubRepository{ExploitID: exploit.ID}).Take(&exploit.GitHubRepository).Error; err != nil {
+			if err := r.conn.
+				Where(&models.GitHubRepository{ExploitID: exploit.ID}).
+				Take(&exploit.GitHubRepository).Error; err != nil {
 				if errors.Is(err, gorm.ErrRecordNotFound) {
 					return nil, xerrors.Errorf("Failed to get GitHubRepository. DB relationship may be broken, use `$ go-exploitdb fetch githubrepos` to recreate DB. err: %w", err)
 				}
@@ -294,6 +322,16 @@ func (r *RDBDriver) GetExploitAll() ([]models.Exploit, error) {
 				}
 				return nil, xerrors.Errorf("Failed to Get inTheWild. err: %w", err)
 			}
+		case models.TrickestType:
+			if err := r.conn.
+				Preload(clause.Associations).
+				Where(&models.Trickest{ExploitID: exploit.ID}).
+				Take(&exploit.Trickest).Error; err != nil {
+				if errors.Is(err, gorm.ErrRecordNotFound) {
+					return nil, xerrors.Errorf("Failed to get Trickest. DB relationship may be broken, use `$ go-exploitdb fetch trickest` to recreate DB. err: %w", err)
+				}
+				return nil, xerrors.Errorf("Failed to get Trickest. err: %w", err)
+			}
 		}
 		es = append(es, exploit)
 	}
@@ -335,7 +373,9 @@ func (r *RDBDriver) GetExploitByCveID(cveID string) ([]models.Exploit, error) {
 				return nil, xerrors.Errorf("Failed to get OffensiveSecurity. err: %w", err)
 			}
 		case models.GitHubRepositoryType:
-			if err := r.conn.Where(&models.GitHubRepository{ExploitID: es[i].ID}).Take(&es[i].GitHubRepository).Error; err != nil {
+			if err := r.conn.
+				Where(&models.GitHubRepository{ExploitID: es[i].ID}).
+				Take(&es[i].GitHubRepository).Error; err != nil {
 				if errors.Is(err, gorm.ErrRecordNotFound) {
 					return nil, xerrors.Errorf("Failed to get GitHubRepository. DB relationship may be broken, use `$ go-exploitdb fetch githubrepos` to recreate DB. err: %w", err)
 				}
@@ -348,6 +388,16 @@ func (r *RDBDriver) GetExploitByCveID(cveID string) ([]models.Exploit, error) {
 				}
 				return nil, xerrors.Errorf("Failed to get inTheWild. err: %w", err)
 			}
+		case models.TrickestType:
+			if err := r.conn.
+				Preload(clause.Associations).
+				Where(&models.Trickest{ExploitID: es[i].ID}).
+				Take(&es[i].Trickest).Error; err != nil {
+				if errors.Is(err, gorm.ErrRecordNotFound) {
+					return nil, xerrors.Errorf("Failed to get Trickest. DB relationship may be broken, use `$ go-exploitdb fetch trickest` to recreate DB. err: %w", err)
+				}
+				return nil, xerrors.Errorf("Failed to get Trickest. err: %w", err)
+			}
 		}
 	}
 	return es, nil

fetcher/githubrepos.go

@@ -18,14 +18,14 @@ import (
 	"golang.org/x/xerrors"
 )
 
-const repoURL string = "https://github.com/nomi-sec/PoC-in-GitHub.git"
+const githubURL string = "https://github.com/nomi-sec/PoC-in-GitHub.git"
 
 // FetchGitHubRepos :
 func FetchGitHubRepos(stars, forks int) (exploits []models.Exploit, err error) {
-	dir := filepath.Join(util.CacheDir(), "pocs")
-	updatedFiles, err := git.CloneOrPull(repoURL, dir)
+	dir := filepath.Join(util.CacheDir(), "github")
+	updatedFiles, err := git.CloneOrPull(githubURL, dir)
 	if err != nil {
-		return nil, xerrors.Errorf("error in pocsrc clone or pull: %w", err)
+		return nil, xerrors.Errorf("Failed to clone or pull nomi-sec/PoC-in-GitHub repository. err: %w", err)
 	}
 
 	var targets []map[string]struct{}
@@ -45,11 +45,11 @@ func FetchGitHubRepos(stars, forks int) (exploits []models.Exploit, err error) {
 		log15.Debug("PoC-in-GitHub: no updated file")
 		return nil, nil
 	}
-	log15.Debug(fmt.Sprintf("PoC-in-GitHub updated files: %d", targetFiles))
+	log15.Debug(fmt.Sprintf("PoC-in-GitHub: updated files: %d", targetFiles))
 
 	entries := map[string][]models.GitHubRepoJSON{}
 	for _, target := range targets {
-		err = util.FileWalk(dir, target, func(r io.Reader, path string) error {
+		if err := util.FileWalk(dir, target, func(r io.Reader, path string) error {
 			content, err := ioutil.ReadAll(r)
 			if err != nil {
 				return err
@@ -64,9 +64,8 @@ func FetchGitHubRepos(stars, forks int) (exploits []models.Exploit, err error) {
 			cveID := strings.Split(dirPaths[len(dirPaths)-1], ".")[0]
 			entries[cveID] = pocs
 			return nil
-		})
-		if err != nil {
-			return nil, xerrors.Errorf("error in PoC-in-GitHub walk: %w", err)
+		}); err != nil {
+			return nil, xerrors.Errorf("Failed to walk PoC-in-GitHub. err: %w", err)
 		}
 	}