Add EPSS Ingestor command to ingest-worker

lunatrace/bsl/ingest-worker/cmd/ingestworker/epss/epss.go

@@ -0,0 +1,51 @@
+// Copyright by LunaSec (owned by Refinery Labs, Inc)
+//
+// Licensed under the Business Source License v1.1
+// (the "License"); you may not use this file except in compliance with the
+// License. You may obtain a copy of the License at
+//
+// https://github.com/lunasec-io/lunasec/blob/master/licenses/BSL-LunaTrace.txt
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package epss
+
+import (
+  "github.com/ajvpot/clifx"
+  "github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/epss"
+  "github.com/rs/zerolog/log"
+  "github.com/urfave/cli/v2"
+  "go.uber.org/fx"
+)
+
+type Params struct {
+  fx.In
+
+  Ingester epss.EPSSIngester
+}
+
+func NewCommand(p Params) clifx.CommandResult {
+  return clifx.CommandResult{
+    Command: &cli.Command{
+      Name: "epss",
+      Subcommands: []*cli.Command{
+        {
+          Name:        "ingest",
+          Usage:       "[file or directory]",
+          Flags:       []cli.Flag{},
+          Subcommands: []*cli.Command{},
+          Action: func(ctx *cli.Context) error {
+            log.Info().
+              Msg("Updating EPSS Scores")
+            err := p.Ingester.Ingest(ctx.Context)
+            if err == nil {
+              log.Info().
+                Msg("Updated EPSS Scores")
+            }
+            return err
+          },
+        },
+      },
+    },
+  }
+}

lunatrace/bsl/ingest-worker/cmd/ingestworker/main.go

@@ -12,11 +12,13 @@ package main
 
 import (
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/cmd/ingestworker/cwe"
+	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/cmd/ingestworker/epss"
 	packageCommand "github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/cmd/ingestworker/package"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/cmd/ingestworker/vulnerability"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/config/ingestworker"
 	cwe2 "github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/cwe"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/dbfx"
+	epss2 "github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/epss"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/graphqlfx"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/metadata/registry"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/metadata/replicator"
@@ -37,46 +39,48 @@ import (
 )
 
 func main() {
-	// TODO (cthompson) this should be configured with an fx module
-	log.Logger = zerolog.New(os.Stderr).With().Timestamp().Logger()
+  // TODO (cthompson) this should be configured with an fx module
+  log.Logger = zerolog.New(os.Stderr).With().Timestamp().Logger()
 
-	clifx2.Main(
-		fx.Supply(http.DefaultClient),
+  clifx2.Main(
+    fx.Supply(http.DefaultClient),
 
-		graphqlfx.Module,
-		dbfx.Module,
-		registry.NPMModule,
+    graphqlfx.Module,
+    dbfx.Module,
+    registry.NPMModule,
 
-		fx.Provide(
-			cwe2.NewCWEIngester,
-		),
+    fx.Provide(
+      cwe2.NewCWEIngester,
+      epss2.NewEPSSIngester,
+    ),
 
-		// todo make a module
-		fx.Supply(&clifx2.AppConfig{
-			Name:    "ingestworker",
-			Usage:   "LunaTrace Ingest Worker",
-			Version: "0.0.1",
-		}),
-		fx.Provide(
-			ingester.NewPackageSqlIngester,
-			ingester.NewNPMPackageIngester,
-			replicator.NewNPMReplicator,
-		),
-		fx.Provide(
-			ingestworker.NewConfigProvider,
-		),
-		fx.Provide(
-			licensecheck.NewScanner,
-			packagejson.NewScanner,
-			license.NewCommand,
-			vulnmanager.NewFileIngester,
-		),
-		fx.Provide(
-			vulnerability.NewCommand,
-			cwe.NewCommand,
-		),
-		fx.Provide(
-			packageCommand.NewCommand,
-		),
-	)
+    // todo make a module
+    fx.Supply(&clifx2.AppConfig{
+      Name:    "ingestworker",
+      Usage:   "LunaTrace Ingest Worker",
+      Version: "0.0.1",
+    }),
+    fx.Provide(
+      ingester.NewPackageSqlIngester,
+      ingester.NewNPMPackageIngester,
+      replicator.NewNPMReplicator,
+    ),
+    fx.Provide(
+      ingestworker.NewConfigProvider,
+    ),
+    fx.Provide(
+      licensecheck.NewScanner,
+      packagejson.NewScanner,
+      license.NewCommand,
+      vulnmanager.NewFileIngester,
+    ),
+    fx.Provide(
+      vulnerability.NewCommand,
+      cwe.NewCommand,
+      epss.NewCommand,
+    ),
+    fx.Provide(
+      packageCommand.NewCommand,
+    ),
+  )
 }

lunatrace/bsl/ingest-worker/cmd/ingestworker/vulnerability/vulnerability.go

@@ -12,8 +12,10 @@
 package vulnerability
 
 import (
+	"context"
 	"github.com/go-co-op/gocron"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/cwe"
+	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/epss"
 	"github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
 	"go.uber.org/fx"
@@ -27,8 +29,9 @@ import (
 type Params struct {
 	fx.In
 
-	Ingester    vulnerability.FileAdvisoryIngester
-	CWEIngester cwe.CWEIngester
+	Ingester     vulnerability.FileAdvisoryIngester
+	CWEIngester  cwe.CWEIngester
+	EPSSIngester epss.EPSSIngester
 }
 
 func NewCommand(p Params) clifx.CommandResult {
@@ -62,22 +65,52 @@ func NewCommand(p Params) clifx.CommandResult {
 						sourceRelativePath := ctx.String("source-relative-path")
 						cron := ctx.String("cron")
 
-						log.Info().
-							Msg("Updating CWEs")
-						err := p.CWEIngester.Ingest(ctx.Context)
-						if err == nil {
+						runIngestion := func() error {
+							log.Info().
+								Msg("Updating CWEs")
+
+							err := p.CWEIngester.Ingest(ctx.Context)
+
+							if err != nil {
+								log.Error().
+									Err(err).
+									Msg("failed to update CWEs")
+							}
+
 							log.Info().
 								Msg("Updated CWEs")
-						} else {
-							return err
-						}
 
-						runIngestion := func() error {
 							log.Info().
 								Str("source", source).
 								Str("cron", cron).
 								Msg("starting vulnerability ingestion")
-							return p.Ingester.IngestVulnerabilitiesFromSource(advisoryLocation, source, sourceRelativePath)
+							err = p.Ingester.IngestVulnerabilitiesFromSource(advisoryLocation, source, sourceRelativePath)
+
+							if err != nil {
+								log.Error().
+									Err(err).
+									Str("source", source).
+									Str("cron", cron).
+									Msg("failed to ingest vulnerabilities")
+							}
+
+							log.Info().
+								Str("source", source).
+								Str("cron", cron).
+								Msg("starting epss ingestion")
+
+							epssContext := context.Background()
+							err = p.EPSSIngester.Ingest(epssContext)
+
+							if err != nil {
+								log.Error().
+									Err(err).
+									Str("source", source).
+									Str("cron", cron).
+									Msg("failed to ingest epss")
+							}
+
+							return nil
 						}
 
 						if cron != "" {

lunatrace/bsl/ingest-worker/pkg/cwe/ingester.go

@@ -50,6 +50,10 @@ func (s *cweIngester) Ingest(ctx context.Context) error {
 		return err
 	}
 
+	log.Info().
+		Int("count", len(cwes.Weaknesses)).
+		Msg("fetched CWEs from MITRE successfully")
+
 	for _, weakness := range cwes.Weaknesses {
 		weaknessIdStr := weakness.ID
 
@@ -73,7 +77,22 @@ func (s *cweIngester) Ingest(ctx context.Context) error {
 			gql.Vulnerability_cwe_update_columnExtendedDescription,
 			gql.Vulnerability_cwe_update_columnCommonName,
 		})
+
+		if err != nil {
+			log.Error().
+				Err(err).
+				Msg("error inserting CWE")
+			return err
+		}
+
+		log.Info().
+			Int("id", weaknessId).
+			Msg("inserted CWE")
 	}
+
+	log.Info().
+		Msg("ingested CWEs successfully")
+
 	return nil
 }