Add EPSS Ingestor command to ingest-worker

lunatrace/bsl/ingest-worker/cmd/analysiscli/main.go

@@ -8,7 +8,6 @@
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package main
 
 import (

lunatrace/bsl/ingest-worker/cmd/ingestworker/epss/epss.go

@@ -0,0 +1,51 @@
+// Copyright by LunaSec (owned by Refinery Labs, Inc)
+//
+// Licensed under the Business Source License v1.1
+// (the "License"); you may not use this file except in compliance with the
+// License. You may obtain a copy of the License at
+//
+// https://github.com/lunasec-io/lunasec/blob/master/licenses/BSL-LunaTrace.txt
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package epss
+
+import (
+	"github.com/ajvpot/clifx"
+	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/epss"
+	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v2"
+	"go.uber.org/fx"
+)
+
+type Params struct {
+	fx.In
+
+	Ingester epss.EPSSIngester
+}
+
+func NewCommand(p Params) clifx.CommandResult {
+	return clifx.CommandResult{
+		Command: &cli.Command{
+			Name: "epss",
+			Subcommands: []*cli.Command{
+				{
+					Name:        "ingest",
+					Usage:       "[file or directory]",
+					Flags:       []cli.Flag{},
+					Subcommands: []*cli.Command{},
+					Action: func(ctx *cli.Context) error {
+						log.Info().
+							Msg("Updating EPSS Scores")
+						err := p.Ingester.Ingest(ctx.Context)
+						if err == nil {
+							log.Info().
+								Msg("Updated EPSS Scores")
+						}
+						return err
+					},
+				},
+			},
+		},
+	}
+}

lunatrace/bsl/ingest-worker/cmd/ingestworker/license/license.go

@@ -8,7 +8,6 @@
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package license
 
 import (

lunatrace/bsl/ingest-worker/cmd/ingestworker/main.go

@@ -12,11 +12,13 @@ package main
 
 import (
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/cmd/ingestworker/cwe"
+	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/cmd/ingestworker/epss"
 	packageCommand "github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/cmd/ingestworker/package"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/cmd/ingestworker/vulnerability"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/config/ingestworker"
 	cwe2 "github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/cwe"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/dbfx"
+	epss2 "github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/epss"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/graphqlfx"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/metadata/registry"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/metadata/replicator"
@@ -49,6 +51,7 @@ func main() {
 
 		fx.Provide(
 			cwe2.NewCWEIngester,
+			epss2.NewEPSSIngester,
 		),
 
 		// todo make a module
@@ -74,6 +77,7 @@ func main() {
 		fx.Provide(
 			vulnerability.NewCommand,
 			cwe.NewCommand,
+			epss.NewCommand,
 		),
 		fx.Provide(
 			packageCommand.NewCommand,

lunatrace/bsl/ingest-worker/cmd/ingestworker/package/package.go

@@ -8,7 +8,6 @@
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package ingest
 
 import (

lunatrace/bsl/ingest-worker/cmd/ingestworker/vulnerability/vulnerability.go

@@ -8,12 +8,13 @@
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package vulnerability
 
 import (
+	"context"
 	"github.com/go-co-op/gocron"
 	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/cwe"
+	"github.com/lunasec-io/lunasec/lunatrace/bsl/ingest-worker/pkg/epss"
 	"github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
 	"go.uber.org/fx"
@@ -27,8 +28,9 @@ import (
 type Params struct {
 	fx.In
 
-	Ingester    vulnerability.FileAdvisoryIngester
-	CWEIngester cwe.CWEIngester
+	Ingester     vulnerability.FileAdvisoryIngester
+	CWEIngester  cwe.CWEIngester
+	EPSSIngester epss.EPSSIngester
 }
 
 func NewCommand(p Params) clifx.CommandResult {
@@ -62,22 +64,55 @@ func NewCommand(p Params) clifx.CommandResult {
 						sourceRelativePath := ctx.String("source-relative-path")
 						cron := ctx.String("cron")
 
-						log.Info().
-							Msg("Updating CWEs")
-						err := p.CWEIngester.Ingest(ctx.Context)
-						if err == nil {
+						runIngestion := func() error {
+							log.Info().
+								Msg("Updating CWEs")
+
+							err := p.CWEIngester.Ingest(ctx.Context)
+
+							if err != nil {
+								log.Error().
+									Err(err).
+									Msg("failed to update CWEs")
+								return err
+							}
+
 							log.Info().
 								Msg("Updated CWEs")
-						} else {
-							return err
-						}
 
-						runIngestion := func() error {
 							log.Info().
 								Str("source", source).
 								Str("cron", cron).
 								Msg("starting vulnerability ingestion")
-							return p.Ingester.IngestVulnerabilitiesFromSource(advisoryLocation, source, sourceRelativePath)
+							err = p.Ingester.IngestVulnerabilitiesFromSource(advisoryLocation, source, sourceRelativePath)
+
+							if err != nil {
+								log.Error().
+									Err(err).
+									Str("source", source).
+									Str("cron", cron).
+									Msg("failed to ingest vulnerabilities")
+								return err
+							}
+
+							log.Info().
+								Str("source", source).
+								Str("cron", cron).
+								Msg("starting epss ingestion")
+
+							epssContext := context.Background()
+							err = p.EPSSIngester.Ingest(epssContext)
+
+							if err != nil {
+								log.Error().
+									Err(err).
+									Str("source", source).
+									Str("cron", cron).
+									Msg("failed to ingest epss")
+								return err
+							}
+
+							return nil
 						}
 
 						if cron != "" {

lunatrace/bsl/ingest-worker/cmd/queuehandler/main.go

@@ -8,7 +8,6 @@
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package main
 
 import (

lunatrace/bsl/ingest-worker/cmd/registryproxy/main.go

@@ -1,14 +1,13 @@
 // Copyright by LunaSec (owned by Refinery Labs, Inc)
 //
-// Licensed under the Business Source License v1.1 
+// Licensed under the Business Source License v1.1
 // (the "License"); you may not use this file except in compliance with the
 // License. You may obtain a copy of the License at
 //
 // https://github.com/lunasec-io/lunasec/blob/master/licenses/BSL-LunaTrace.txt
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package main
 
 import (

lunatrace/bsl/ingest-worker/pkg/awsfx/config.go

@@ -1,14 +1,13 @@
 // Copyright by LunaSec (owned by Refinery Labs, Inc)
 //
-// Licensed under the Business Source License v1.1 
+// Licensed under the Business Source License v1.1
 // (the "License"); you may not use this file except in compliance with the
 // License. You may obtain a copy of the License at
 //
 // https://github.com/lunasec-io/lunasec/blob/master/licenses/BSL-LunaTrace.txt
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package awsfx
 
 import (

lunatrace/bsl/ingest-worker/pkg/awsfx/module.go

@@ -1,14 +1,13 @@
 // Copyright by LunaSec (owned by Refinery Labs, Inc)
 //
-// Licensed under the Business Source License v1.1 
+// Licensed under the Business Source License v1.1
 // (the "License"); you may not use this file except in compliance with the
 // License. You may obtain a copy of the License at
 //
 // https://github.com/lunasec-io/lunasec/blob/master/licenses/BSL-LunaTrace.txt
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package awsfx
 
 import "go.uber.org/fx"

lunatrace/bsl/ingest-worker/pkg/awsfx/session.go

@@ -8,7 +8,6 @@
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package awsfx
 
 import (

lunatrace/bsl/ingest-worker/pkg/config/ingestworker/config.go

@@ -8,7 +8,6 @@
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package ingestworker
 
 import (

lunatrace/bsl/ingest-worker/pkg/config/queuehandler/config.go

@@ -8,7 +8,6 @@
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package queuehandler
 
 import (

lunatrace/bsl/ingest-worker/pkg/config/registryproxy/config.go

@@ -1,14 +1,13 @@
 // Copyright by LunaSec (owned by Refinery Labs, Inc)
 //
-// Licensed under the Business Source License v1.1 
+// Licensed under the Business Source License v1.1
 // (the "License"); you may not use this file except in compliance with the
 // License. You may obtain a copy of the License at
 //
 // https://github.com/lunasec-io/lunasec/blob/master/licenses/BSL-LunaTrace.txt
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package registryproxy
 
 import (

lunatrace/bsl/ingest-worker/pkg/cwe/fetch.go

@@ -1,14 +1,13 @@
 // Copyright by LunaSec (owned by Refinery Labs, Inc)
 //
-// Licensed under the Business Source License v1.1 
+// Licensed under the Business Source License v1.1
 // (the "License"); you may not use this file except in compliance with the
 // License. You may obtain a copy of the License at
 //
 // https://github.com/lunasec-io/lunasec/blob/master/licenses/BSL-LunaTrace.txt
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package cwe
 
 import (

lunatrace/bsl/ingest-worker/pkg/cwe/fetch_test.go

@@ -1,14 +1,13 @@
 // Copyright by LunaSec (owned by Refinery Labs, Inc)
 //
-// Licensed under the Business Source License v1.1 
+// Licensed under the Business Source License v1.1
 // (the "License"); you may not use this file except in compliance with the
 // License. You may obtain a copy of the License at
 //
 // https://github.com/lunasec-io/lunasec/blob/master/licenses/BSL-LunaTrace.txt
 //
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 package cwe
 
 import (