chore: improve dataflow report for component (#151)

* chore: improve dataflow report for component * fix: improve tests
Bearer · Nov 22, 2022 · 644b7ff · 644b7ff
1 parent 6a9e591
commit 644b7ff
Show file tree

Hide file tree

Showing 7 changed files with 55 additions and 29 deletions.
diff --git a/pkg/classification/db/recipes/atlassian_cloud.json b/pkg/classification/db/recipes/atlassian_cloud.json
@@ -4,7 +4,8 @@
   "type": "external_service",
   "urls": [
     "https://api.atlassian.com",
-    "https://atlassian.net/rest/api/"
+    "https://atlassian.net/rest/api/",
+    "https://auth.atlassian.com"
   ],
   "packages": [
     {

diff --git a/pkg/classification/interfaces/interfaces.go b/pkg/classification/interfaces/interfaces.go
@@ -59,6 +59,14 @@ type RecipeURLMatch struct {
 var ErrInvalidRecipes = errors.New("invalid interface recipe")
 var ErrInvalidInternalDomainRegexp = errors.New("could not parse internal domains as regexp")
 
+func (classification *Classification) Name() string {
+	if classification.RecipeMatch {
+		return classification.RecipeName
+	} else {
+		return classification.URL
+	}
+}
+
 func New(config Config) (*Classifier, error) {
 	// prepare regular expressions for recipes
 	var preparedRecipes []Recipe

diff --git a/pkg/report/output/dataflow/components/components.go b/pkg/report/output/dataflow/components/components.go
@@ -1,8 +1,6 @@
 package components
 
 import (
-	"strings"
-
 	"github.com/bearer/curio/pkg/report/output/dataflow/detectiondecoder"
 	"github.com/bearer/curio/pkg/report/output/dataflow/types"
 
@@ -20,10 +18,12 @@ type component struct {
 	uuid      string
 	detectors map[string]*detector // group detectors by detectorName
 }
+
 type detector struct {
 	name  string
 	files map[string]*fileHolder // group files by filename
 }
+
 type fileHolder struct {
 	name        string
 	lineNumbers map[int]int //group lines by linenumber
@@ -47,7 +47,13 @@ func (holder *Holder) AddInterface(detection interface{}) error {
 	}
 
 	if value.Classification.Decision.State == classify.Valid {
-		holder.addComponent(strings.ToLower(value.Classification.RecipeName), value.Classification.RecipeUUID, string(value.DetectorType), value.Source.Filename, *value.Source.LineNumber)
+		holder.addComponent(
+			value.Classification.Name(),
+			value.Classification.RecipeUUID,
+			string(value.DetectorType),
+			value.Source.Filename,
+			*value.Source.LineNumber,
+		)
 	}
 
 	return nil
@@ -64,7 +70,13 @@ func (holder *Holder) AddDependency(detection interface{}) error {
 	}
 
 	if value.Classification.Decision.State == classify.Valid {
-		holder.addComponent(strings.ToLower(value.Classification.RecipeName), value.Classification.RecipeUUID, string(value.DetectorType), value.Source.Filename, *value.Source.LineNumber)
+		holder.addComponent(
+			value.Classification.RecipeName,
+			value.Classification.RecipeUUID,
+			string(value.DetectorType),
+			value.Source.Filename,
+			*value.Source.LineNumber,
+		)
 	}
 
 	return nil
@@ -81,7 +93,13 @@ func (holder *Holder) AddFramework(detection interface{}) error {
 	}
 
 	if value.Classification.Decision.State == classify.Valid {
-		holder.addComponent(strings.ToLower(value.Classification.RecipeName), value.Classification.RecipeUUID, string(value.DetectorType), value.Source.Filename, *value.Source.LineNumber)
+		holder.addComponent(
+			value.Classification.RecipeName,
+			value.Classification.RecipeUUID,
+			string(value.DetectorType),
+			value.Source.Filename,
+			*value.Source.LineNumber,
+		)
 	}
 
 	return nil

diff --git a/pkg/report/output/dataflow/components/components_test.go b/pkg/report/output/dataflow/components/components_test.go
@@ -19,11 +19,11 @@ func TestDataflowComponents(t *testing.T) {
 		Want        []types.Component
 	}{
 		{
-			Name: "single detection - dependency",
-			FileContent: `{	"detector_type": "gemfile-lock", "type": "dependency_classified", "source": {"filename": "Gemfile.lock", "line_number": 258}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "stripe"}}`,
+			Name:        "single detection - dependency",
+			FileContent: `{	"detector_type": "gemfile-lock", "type": "dependency_classified", "source": {"filename": "Gemfile.lock", "line_number": 258}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "Stripe", "recipe_match": true}}`,
 			Want: []types.Component{
 				{
-					Name: "stripe",
+					Name: "Stripe",
 					Locations: []types.ComponentLocation{
 						{
 							Detector:   "gemfile-lock",
@@ -35,16 +35,16 @@ func TestDataflowComponents(t *testing.T) {
 			},
 		},
 		{
-			Name: "single detection - dependency - no classification",
+			Name:        "single detection - dependency - no classification",
 			FileContent: `{	"detector_type": "gemfile-lock", "type": "dependency_classified", "source": {"filename": "Gemfile.lock", "line_number": 258}}`,
-			Want: []types.Component{},
+			Want:        []types.Component{},
 		},
 		{
-			Name: "single detection - interface",
-			FileContent: `{	"detector_type": "ruby", "type": "interface_classified", "source": {"filename": "billing.rb", "line_number": 2}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "Stripe"}}`,
+			Name:        "single detection - interface",
+			FileContent: `{	"detector_type": "ruby", "type": "interface_classified", "source": {"filename": "billing.rb", "line_number": 2}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "Stripe", "recipe_match": true}}`,
 			Want: []types.Component{
 				{
-					Name: "stripe",
+					Name: "Stripe",
 					Locations: []types.ComponentLocation{
 						{
 							Detector:   "ruby",
@@ -56,17 +56,17 @@ func TestDataflowComponents(t *testing.T) {
 			},
 		},
 		{
-			Name: "single detection - interface - no classification",
+			Name:        "single detection - interface - no classification",
 			FileContent: `{	"detector_type": "ruby", "type": "interface_classified", "source": {"filename": "billing.rb", "line_number": 2}}`,
-			Want: []types.Component{},
+			Want:        []types.Component{},
 		},
 		{
 			Name: "single detection - duplicates",
-			FileContent: `{	"detector_type": "ruby", "type": "interface_classified", "source": {"filename": "billing.rb", "line_number": 2}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "Stripe"}}
-{ "detector_type": "ruby", "type": "interface_classified", "source": {"filename": "billing.rb", "line_number": 2}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "Stripe"}}`,
+			FileContent: `{	"detector_type": "ruby", "type": "interface_classified", "source": {"filename": "billing.rb", "line_number": 2}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "Stripe", "recipe_match": true}}
+{ "detector_type": "ruby", "type": "interface_classified", "source": {"filename": "billing.rb", "line_number": 2}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "Stripe", "recipe_match": true}}`,
 			Want: []types.Component{
 				{
-					Name: "stripe",
+					Name: "Stripe",
 					Locations: []types.ComponentLocation{
 						{
 							Detector:   "ruby",
@@ -79,11 +79,11 @@ func TestDataflowComponents(t *testing.T) {
 		},
 		{
 			Name: "multiple detections - deterministic output",
-			FileContent: `{	"detector_type": "ruby", "type": "interface_classified", "source": {"filename": "billing.rb", "line_number": 2}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "Stripe"}}
-{"detector_type": "gemfile-lock", "type": "dependency_classified", "source": {"filename": "Gemfile.lock", "line_number": 258}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "stripe"}}`,
+			FileContent: `{	"detector_type": "ruby", "type": "interface_classified", "source": {"filename": "billing.rb", "line_number": 2}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "Stripe", "recipe_match": true}}
+{"detector_type": "gemfile-lock", "type": "dependency_classified", "source": {"filename": "Gemfile.lock", "line_number": 258}, "classification": { "Decision": { "state": "valid" }, "recipe_name": "Stripe", "recipe_match": true}}`,
 			Want: []types.Component{
 				{
-					Name: "stripe",
+					Name: "Stripe",
 					Locations: []types.ComponentLocation{
 						{
 							Detector:   "gemfile-lock",

diff --git a/pkg/report/output/dataflow/dataflow.go b/pkg/report/output/dataflow/dataflow.go
@@ -89,7 +89,6 @@ func GetOutput(input []interface{}, config settings.Config, isInternal bool) (*D
 				return nil, err
 			}
 		}
-
 	}
 
 	dataflow := &DataFlow{

diff --git a/pkg/report/output/dataflow/types/components.go b/pkg/report/output/dataflow/types/components.go
@@ -1,13 +1,13 @@
 package types
 
 type Component struct {
-	Name      string
+	Name      string              `json:"name"`
 	UUID      string              `json:"uuid,omitempty" yaml:"uuid,omitempty"`
-	Locations []ComponentLocation
+	Locations []ComponentLocation `json:"locations"`
 }
 
 type ComponentLocation struct {
-	Detector   string
-	Filename   string
-	LineNumber int
+	Detector   string `json:"detector"`
+	Filename   string `json:"filename"`
+	LineNumber int    `json:"line_number"`
 }
diff --git a/pkg/util/file/file.go b/pkg/util/file/file.go
@@ -96,7 +96,7 @@ func (path *Path) Exists() bool {
 	}
 
 	if err != nil {
-		log.Error().Msgf("error checking if file '%s' exists: %s", path.AbsolutePath, err)
+		log.Debug().Msgf("file '%s' doesn't exist", path.AbsolutePath)
 		return false
 	}