Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(trickest): support Trickest PoCs #81

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions GNUmakefile
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,13 @@ fetch-rdb:
integration/exploitdb.old fetch exploitdb --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
integration/exploitdb.old fetch githubrepos --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
integration/exploitdb.old fetch inthewild --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
integration/exploitdb.old fetch trickest --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3

integration/exploitdb.new fetch awesomepoc --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
integration/exploitdb.new fetch exploitdb --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
integration/exploitdb.new fetch githubrepos --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
integration/exploitdb.new fetch inthewild --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
integration/exploitdb.new fetch trickest --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3

fetch-redis:
docker run --name redis-old -d -p 127.0.0.1:6379:6379 redis
Expand All @@ -105,11 +107,13 @@ fetch-redis:
integration/exploitdb.old fetch exploitdb --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
integration/exploitdb.old fetch githubrepos --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
integration/exploitdb.old fetch inthewild --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
integration/exploitdb.old fetch trickest --dbtype redis --dbpath "redis://127.0.0.1:6379/0"

integration/exploitdb.new fetch awesomepoc --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
integration/exploitdb.new fetch exploitdb --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
integration/exploitdb.new fetch githubrepos --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
integration/exploitdb.new fetch inthewild --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
integration/exploitdb.new fetch trickest --dbtype redis --dbpath "redis://127.0.0.1:6380/0"

diff-cveid:
@ python integration/diff_server_mode.py cveid --sample_rate 0.01
Expand Down
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@ In server mode, a simple Web API can be used.
As the following vulnerabilities database

1. [ExploitDB(OffensiveSecurity)](https://www.exploit-db.com/) by CVE number or Exploit Database ID.
2. [GitHub Repositories](https://github.com/search?o=desc&q=CVE&s=&type=Repositories)
2. [GitHub Repositories](https://github.com/nomi-sec/PoC-in-GitHub.git)
3. [Awesome Cve Poc](https://github.com/qazbnm456/awesome-cve-poc#toc473)
4. [inTheWild DB](https://github.com/gmatuz/inthewilddb)
5. [Trickest CVE](https://github.com/trickest/cve.git)

### Docker Deployment
There's a Docker image available `docker pull vulsio/go-exploitdb`. When using the container, it takes the same arguments as the [normal command line](#Usage).
Expand Down Expand Up @@ -56,6 +57,7 @@ Available Commands:
exploitdb Fetch the data of offensive security exploit db
githubrepos Fetch the data of github repos
inthewild Fetch the data of inTheWild Poc
trickest Fetch the data of trickest PoCs

Flags:
--batch-size int The number of batch size to insert. (default 500)
Expand Down
75 changes: 75 additions & 0 deletions commands/fetch-trickest.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package commands

import (
"time"

"github.com/inconshreveable/log15"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulsio/go-exploitdb/db"
"github.com/vulsio/go-exploitdb/fetcher"
"github.com/vulsio/go-exploitdb/models"
"github.com/vulsio/go-exploitdb/util"
"golang.org/x/xerrors"
)

var fetchTrickestCmd = &cobra.Command{
Use: "trickest",
Short: "Fetch the data of trickest PoCs",
Long: `Fetch the data of trickest PoCs`,
RunE: fetchTrickest,
}

func init() {
fetchCmd.AddCommand(fetchTrickestCmd)
}

func fetchTrickest(_ *cobra.Command, _ []string) (err error) {
if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
return xerrors.Errorf("Failed to SetLogger. err: %w", err)
}

driver, locked, err := db.NewDB(
viper.GetString("dbtype"),
viper.GetString("dbpath"),
viper.GetBool("debug-sql"),
db.Option{},
)
if err != nil {
if locked {
return xerrors.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %w", err)
}
return xerrors.Errorf("Failed to open DB. err: %w", err)
}

fetchMeta, err := driver.GetFetchMeta()
if err != nil {
return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
}
if fetchMeta.OutDated() {
return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
}
// If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion.
if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
}

log15.Info("Fetching Trickest Exploit")
var exploits []models.Exploit
if exploits, err = fetcher.FetchTrickest(); err != nil {
return xerrors.Errorf("Failed to fetch Trickest Exploit. err: %w", err)
}
log15.Info("Trickest Exploit", "count", len(exploits))

log15.Info("Insert Exploit into go-exploitdb.", "db", driver.Name())
if err := driver.InsertExploit(models.TrickestType, exploits); err != nil {
return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
}

fetchMeta.LastFetchedAt = time.Now()
if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
}

return nil
}
1 change: 1 addition & 0 deletions db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type DB interface {
UpsertFetchMeta(*models.FetchMeta) error
}

// Option :
type Option struct {
RedisTimeout time.Duration
}
Expand Down
56 changes: 53 additions & 3 deletions db/rdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,10 @@ func (r *RDBDriver) MigrateDB() error {
&models.Paper{},
&models.GitHubRepository{},
&models.InTheWild{},
&models.Trickest{},
&models.TrickestProduct{},
&models.TrickestVersion{},
&models.TrickestVulnerability{},
); err != nil {
return xerrors.Errorf("Failed to migrate. err: %w", err)
}
Expand Down Expand Up @@ -186,6 +190,16 @@ func (r *RDBDriver) deleteAndInsertExploit(exploitType models.ExploitType, explo
}
}

trIDs := []models.Trickest{}
if err := tx.Model(&models.Trickest{}).Select("id").Where("exploit_id IN ?", oldIDs[idx.From:idx.To]).Find(&trIDs).Error; err != nil {
return xerrors.Errorf("Failed to select old Trickest: %w", err)
}
if len(trIDs) > 0 {
if err := tx.Select(clause.Associations).Delete(&trIDs).Error; err != nil {
return xerrors.Errorf("Failed to delete: %w", err)
}
}

if err := tx.Where("id IN ?", oldIDs[idx.From:idx.To]).Delete(&models.Exploit{}).Error; err != nil {
return xerrors.Errorf("Failed to delete: %w", err)
}
Expand Down Expand Up @@ -236,7 +250,9 @@ func (r *RDBDriver) GetExploitByID(exploitUniqueID string) ([]models.Exploit, er
return nil, xerrors.Errorf("Failed to get OffensiveSecurity. err: %w", err)
}
case models.GitHubRepositoryType:
if err := r.conn.Where(&models.GitHubRepository{ExploitID: es[i].ID}).Take(&es[i].GitHubRepository).Error; err != nil {
if err := r.conn.
Where(&models.GitHubRepository{ExploitID: es[i].ID}).
Take(&es[i].GitHubRepository).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, xerrors.Errorf("Failed to get GitHubRepository. DB relationship may be broken, use `$ go-exploitdb fetch githubrepos` to recreate DB. err: %w", err)
}
Expand All @@ -249,6 +265,16 @@ func (r *RDBDriver) GetExploitByID(exploitUniqueID string) ([]models.Exploit, er
}
return nil, xerrors.Errorf("Failed to get inTheWild. err: %w", err)
}
case models.TrickestType:
if err := r.conn.
Preload(clause.Associations).
Where(&models.Trickest{ExploitID: es[i].ID}).
Take(&es[i].Trickest).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, xerrors.Errorf("Failed to get Trickest. DB relationship may be broken, use `$ go-exploitdb fetch trickest` to recreate DB. err: %w", err)
}
return nil, xerrors.Errorf("Failed to get Trickest. err: %w", err)
}
}
}
return es, nil
Expand Down Expand Up @@ -281,7 +307,9 @@ func (r *RDBDriver) GetExploitAll() ([]models.Exploit, error) {
return nil, xerrors.Errorf("Failed to Get OffensiveSecurity. err: %w", err)
}
case models.GitHubRepositoryType:
if err := r.conn.Where(&models.GitHubRepository{ExploitID: exploit.ID}).Take(&exploit.GitHubRepository).Error; err != nil {
if err := r.conn.
Where(&models.GitHubRepository{ExploitID: exploit.ID}).
Take(&exploit.GitHubRepository).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, xerrors.Errorf("Failed to get GitHubRepository. DB relationship may be broken, use `$ go-exploitdb fetch githubrepos` to recreate DB. err: %w", err)
}
Expand All @@ -294,6 +322,16 @@ func (r *RDBDriver) GetExploitAll() ([]models.Exploit, error) {
}
return nil, xerrors.Errorf("Failed to Get inTheWild. err: %w", err)
}
case models.TrickestType:
if err := r.conn.
Preload(clause.Associations).
Where(&models.Trickest{ExploitID: exploit.ID}).
Take(&exploit.Trickest).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, xerrors.Errorf("Failed to get Trickest. DB relationship may be broken, use `$ go-exploitdb fetch trickest` to recreate DB. err: %w", err)
}
return nil, xerrors.Errorf("Failed to get Trickest. err: %w", err)
}
}
es = append(es, exploit)
}
Expand Down Expand Up @@ -335,7 +373,9 @@ func (r *RDBDriver) GetExploitByCveID(cveID string) ([]models.Exploit, error) {
return nil, xerrors.Errorf("Failed to get OffensiveSecurity. err: %w", err)
}
case models.GitHubRepositoryType:
if err := r.conn.Where(&models.GitHubRepository{ExploitID: es[i].ID}).Take(&es[i].GitHubRepository).Error; err != nil {
if err := r.conn.
Where(&models.GitHubRepository{ExploitID: es[i].ID}).
Take(&es[i].GitHubRepository).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, xerrors.Errorf("Failed to get GitHubRepository. DB relationship may be broken, use `$ go-exploitdb fetch githubrepos` to recreate DB. err: %w", err)
}
Expand All @@ -348,6 +388,16 @@ func (r *RDBDriver) GetExploitByCveID(cveID string) ([]models.Exploit, error) {
}
return nil, xerrors.Errorf("Failed to get inTheWild. err: %w", err)
}
case models.TrickestType:
if err := r.conn.
Preload(clause.Associations).
Where(&models.Trickest{ExploitID: es[i].ID}).
Take(&es[i].Trickest).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, xerrors.Errorf("Failed to get Trickest. DB relationship may be broken, use `$ go-exploitdb fetch trickest` to recreate DB. err: %w", err)
}
return nil, xerrors.Errorf("Failed to get Trickest. err: %w", err)
}
}
}
return es, nil
Expand Down
17 changes: 8 additions & 9 deletions fetcher/githubrepos.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@ import (
"golang.org/x/xerrors"
)

const repoURL string = "https://github.com/nomi-sec/PoC-in-GitHub.git"
const githubURL string = "https://github.com/nomi-sec/PoC-in-GitHub.git"

// FetchGitHubRepos :
func FetchGitHubRepos(stars, forks int) (exploits []models.Exploit, err error) {
dir := filepath.Join(util.CacheDir(), "pocs")
updatedFiles, err := git.CloneOrPull(repoURL, dir)
dir := filepath.Join(util.CacheDir(), "github")
updatedFiles, err := git.CloneOrPull(githubURL, dir)
if err != nil {
return nil, xerrors.Errorf("error in pocsrc clone or pull: %w", err)
return nil, xerrors.Errorf("Failed to clone or pull nomi-sec/PoC-in-GitHub repository. err: %w", err)
}

var targets []map[string]struct{}
Expand All @@ -45,11 +45,11 @@ func FetchGitHubRepos(stars, forks int) (exploits []models.Exploit, err error) {
log15.Debug("PoC-in-GitHub: no updated file")
return nil, nil
}
log15.Debug(fmt.Sprintf("PoC-in-GitHub updated files: %d", targetFiles))
log15.Debug(fmt.Sprintf("PoC-in-GitHub: updated files: %d", targetFiles))

entries := map[string][]models.GitHubRepoJSON{}
for _, target := range targets {
err = util.FileWalk(dir, target, func(r io.Reader, path string) error {
if err := util.FileWalk(dir, target, func(r io.Reader, path string) error {
content, err := ioutil.ReadAll(r)
if err != nil {
return err
Expand All @@ -64,9 +64,8 @@ func FetchGitHubRepos(stars, forks int) (exploits []models.Exploit, err error) {
cveID := strings.Split(dirPaths[len(dirPaths)-1], ".")[0]
entries[cveID] = pocs
return nil
})
if err != nil {
return nil, xerrors.Errorf("error in PoC-in-GitHub walk: %w", err)
}); err != nil {
return nil, xerrors.Errorf("Failed to walk PoC-in-GitHub. err: %w", err)
}
}

Expand Down