chore: Run classifier for schema detections

pkg/detectors/graphql/graphql.go

@@ -67,7 +67,7 @@ func (detector *detector) ExtractFromSchema(
 
 	uuidHolder := parserschema.NewUUIDHolder()
 
-	return tree.Query(graphqlSchemaQuery, func(captures parser.Captures) error {
+	err = tree.Query(graphqlSchemaQuery, func(captures parser.Captures) error {
 		objectNode := captures["object_name"]
 		objectName := stripQuotes(objectNode.Content())
 		fieldType := stripQuotes(captures["field_type"].Content())
@@ -78,20 +78,37 @@ func (detector *detector) ExtractFromSchema(
 		objectUUID := uuidHolder.Assign(objectNode.ID(), detector.idGenerator)
 		fieldUUID := uuidHolder.Assign(fieldNode.ID(), detector.idGenerator)
 
-		report.AddSchema(detectors.DetectorGraphQL,
-			schema.Schema{
-				ObjectName:      objectName,
-				ObjectUUID:      objectUUID,
-				FieldName:       fieldName,
-				FieldType:       fieldType,
-				FieldUUID:       fieldUUID,
-				SimpleFieldType: convertType(fieldType),
-			},
-			fieldNode.Source(true),
+		currentSchema := schema.Schema{
+			ObjectName:      objectName,
+			ObjectUUID:      objectUUID,
+			FieldName:       fieldName,
+			FieldType:       fieldType,
+			FieldUUID:       fieldUUID,
+			SimpleFieldType: convertType(fieldType),
+		}
+
+		if !report.SchemaGroupIsOpen() {
+			source := objectNode.Source(true)
+			report.SchemaGroupBegin(
+				detectors.DetectorGraphQL,
+				objectNode,
+				currentSchema,
+				&source,
+			)
+		}
+		source := fieldNode.Source(true)
+		report.SchemaGroupAddItem(
+			fieldNode,
+			currentSchema,
+			&source,
 		)
 
 		return nil
 	})
+
+	report.SchemaGroupEnd(detector.idGenerator)
+
+	return err
 }
 
 func stripQuotes(value string) string {

pkg/detectors/internal/testhelper/testhelper.go

@@ -82,6 +82,7 @@ type InMemoryReport struct {
 	Errors           []*detections.ErrorDetection
 	SecretLeaks      []*detections.Detection
 	CreateView       []*detections.Detection
+	SchemaGroupDetectorType reportdetectors.Type
 }
 
 func (report *InMemoryReport) AddDetection(
@@ -103,20 +104,27 @@ func (report *InMemoryReport) AddDetection(
 	}
 }
 
-func (report *InMemoryReport) AddSchema(
-	detectorType reportdetectors.Type,
-	schema schema.Schema,
-	source source.Source,
-) {
+func (report *InMemoryReport) SchemaGroupBegin(detectorType reportdetectors.Type, node *parser.Node, schema schema.Schema, source *source.Source) {
+	report.SchemaGroupDetectorType = detectorType
+}
+
+func (report *InMemoryReport) SchemaGroupIsOpen() bool {
+	return report.SchemaGroupDetectorType != ""
+}
 
+func (report *InMemoryReport) SchemaGroupAddItem(node *parser.Node, schema schema.Schema, source *source.Source) {
 	report.Detections = append(report.Detections, &detections.Detection{
-		DetectorType: detectorType,
+		DetectorType: report.SchemaGroupDetectorType,
 		Value:        schema,
-		Source:       source,
+		Source:       *source,
 		Type:         detections.TypeSchema,
 	})
 }
 
+func (report *InMemoryReport) SchemaGroupEnd(idGenerator nodeid.Generator) {
+	report.SchemaGroupDetectorType = ""
+}
+
 func (report *InMemoryReport) AddDataType(
 	detectionType detections.DetectionType,
 	detectorType reportdetectors.Type,
@@ -125,7 +133,7 @@ func (report *InMemoryReport) AddDataType(
 	parent *parser.Node,
 ) {
 
-	datatype.Export(report, detectionType, detectorType, idGenerator, values)
+	datatype.ExportClassified(report, detectionType, detectorType, idGenerator, values, nil)
 }
 
 func (report *InMemoryReport) AddCreateView(

pkg/detectors/openapi/reportadder/reportadder.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/bearer/curio/pkg/parser"
+	"github.com/bearer/curio/pkg/parser/nodeid"
 	reporttypes "github.com/bearer/curio/pkg/report"
 	"github.com/bearer/curio/pkg/report/detectors"
 	"github.com/bearer/curio/pkg/report/operations"
@@ -17,26 +18,64 @@ import (
 
 var regexpPathVariable = regexp.MustCompile(`\{.+\}`)
 
-func AddSchema(file *file.FileInfo, report reporttypes.Report, foundValues map[parser.Node]*schemahelper.Schema) {
+type SortableSchema struct {
+	Key parser.Node
+	Value *schemahelper.Schema
+}
+type SortableSchemaList []SortableSchema
+func (s SortableSchemaList) Len() int {
+	return len(s)
+}
+func (s SortableSchemaList) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+func (s SortableSchemaList) Less(i, j int) bool {
+	lineNumberA := s[i].Value.Source.LineNumber
+	lineNumberB := s[j].Value.Source.LineNumber
+	return *lineNumberA < *lineNumberB
+}
+
+func AddSchema(file *file.FileInfo, report reporttypes.Report, foundValues map[parser.Node]*schemahelper.Schema, idGenerator nodeid.Generator) {
 	// we need sorted schemas so our reports are consistent and repeatable
-	var sortedSchemas []*schemahelper.Schema
-	for _, schema := range foundValues {
-		sortedSchemas = append(sortedSchemas, schema)
+	sortedSchemas := make(SortableSchemaList, len(foundValues))
+	i := 0
+	for k, v := range(foundValues) {
+		sortedSchemas[i] = SortableSchema{Key: k, Value: v}
+		i++
 	}
-	sort.Slice(sortedSchemas, func(i, j int) bool {
-		lineNumberA := sortedSchemas[i].Source.LineNumber
-		lineNumberB := sortedSchemas[j].Source.LineNumber
-		return *lineNumberA < *lineNumberB
-	})
-	for _, schema := range sortedSchemas {
+	sort.Sort(sortedSchemas)
+
+	for _, sortableSchema := range sortedSchemas {
+		node := sortableSchema.Key
+		schema := sortableSchema.Value
+
 		schema.Source.Language = file.Language
 		schema.Source.LanguageType = file.LanguageTypeString()
 		schema.Value.FieldName = stringutil.StripQuotes(schema.Value.FieldName)
 		schema.Value.FieldType = stringutil.StripQuotes(schema.Value.FieldType)
 		schema.Value.ObjectName = stringutil.StripQuotes(schema.Value.ObjectName)
 		schema.Value.SimpleFieldType = convertSchema(schema.Value.FieldType)
-		report.AddSchema(detectors.DetectorOpenAPI, schema.Value, schema.Source)
+
+		if !report.SchemaGroupIsOpen() {
+			// @todo FIXME: Transition to new API
+			var objectNode *parser.Node
+			report.SchemaGroupBegin(
+				detectors.DetectorOpenAPI,
+				objectNode,
+				schema.Value,
+				&schema.Source,
+			)
+		}
+
+		fieldNode := node
+		report.SchemaGroupAddItem(
+			&fieldNode,
+			schema.Value,
+			&schema.Source,
+		)
 	}
+
+	report.SchemaGroupEnd(idGenerator)
 }
 
 func convertSchema(value string) string {

pkg/detectors/openapi/v2json/v2json.go

@@ -68,7 +68,7 @@ func ProcessFile(idGenerator nodeid.Generator, file *file.FileInfo, report repor
 		return false, err
 	}
 
-	reportadder.AddSchema(file, report, foundSchemas)
+	reportadder.AddSchema(file, report, foundSchemas, idGenerator)
 
 	return true, err
 }

pkg/detectors/openapi/v2yaml/v2yml.go

@@ -68,7 +68,7 @@ func ProcessFile(idGenerator nodeid.Generator, file *file.FileInfo, report repor
 		return false, err
 	}
 
-	reportadder.AddSchema(file, report, foundValues)
+	reportadder.AddSchema(file, report, foundValues, idGenerator)
 
 	return true, err
 }

pkg/detectors/openapi/v3json/v3json.go

@@ -67,7 +67,7 @@ func ProcessFile(idGenerator nodeid.Generator, file *file.FileInfo, report repor
 		return false, err
 	}
 
-	reportadder.AddSchema(file, report, foundSchemas)
+	reportadder.AddSchema(file, report, foundSchemas, idGenerator)
 
 	return true, err
 }

pkg/detectors/openapi/v3yaml/v3yml.go

@@ -66,7 +66,7 @@ func ProcessFile(idGenerator nodeid.Generator, file *file.FileInfo, report repor
 		return false, err
 	}
 
-	reportadder.AddSchema(file, report, foundSchemas)
+	reportadder.AddSchema(file, report, foundSchemas, idGenerator)
 
 	return true, err
 }

pkg/detectors/proto/proto.go

@@ -69,7 +69,7 @@ func (detector *detector) ExtractFromSchema(
 
 	uuidHolder := parserschema.NewUUIDHolder()
 
-	return tree.Query(protoSchemaQuery, func(captures parser.Captures) error {
+	err = tree.Query(protoSchemaQuery, func(captures parser.Captures) error {
 		objectNode := captures["object_name"]
 		objectName := stripQuotes(objectNode.Content())
 
@@ -81,21 +81,37 @@ func (detector *detector) ExtractFromSchema(
 		objectUUID := uuidHolder.Assign(objectNode.ID(), detector.idGenerator)
 		fieldUUID := uuidHolder.Assign(fieldNode.ID(), detector.idGenerator)
 
-		report.AddSchema(
-			detectors.DetectorProto,
-			schema.Schema{
-				ObjectName:      objectName,
-				ObjectUUID:      objectUUID,
-				FieldName:       fieldName,
-				FieldUUID:       fieldUUID,
-				FieldType:       columnType,
-				SimpleFieldType: convertToSimpleType(columnType),
-			},
-			fieldNode.Source(true),
+		currentSchema := schema.Schema{
+			ObjectName:      objectName,
+			ObjectUUID:      objectUUID,
+			FieldName:       fieldName,
+			FieldUUID:       fieldUUID,
+			FieldType:       columnType,
+			SimpleFieldType: convertToSimpleType(columnType),
+		}
+
+		if !report.SchemaGroupIsOpen() {
+			source := objectNode.Source(true)
+			report.SchemaGroupBegin(
+				detectors.DetectorProto,
+				objectNode,
+				currentSchema,
+				&source,
+			)
+		}
+		source := fieldNode.Source(true)
+		report.SchemaGroupAddItem(
+			fieldNode,
+			currentSchema,
+			&source,
 		)
 
 		return nil
 	})
+
+	report.SchemaGroupEnd(detector.idGenerator)
+
+	return err
 }
 
 func stripQuotes(value string) string {

pkg/detectors/rails/.snapshots/TestBuildReportDatabaseSchema

@@ -8,7 +8,7 @@
       Language: (string) (len=4) "Ruby",
       LanguageType: (string) (len=11) "programming",
       LineNumber: (*int)(20),
-      ColumnNumber: (*int)(7),
+      ColumnNumber: (*int)(14),
       Text: (*string)(<nil>)
     },
     Value: (schema.Schema) {
@@ -31,7 +31,7 @@
       Language: (string) (len=4) "Ruby",
       LanguageType: (string) (len=11) "programming",
       LineNumber: (*int)(21),
-      ColumnNumber: (*int)(7),
+      ColumnNumber: (*int)(14),
       Text: (*string)(<nil>)
     },
     Value: (schema.Schema) {
@@ -54,7 +54,7 @@
       Language: (string) (len=4) "Ruby",
       LanguageType: (string) (len=11) "programming",
       LineNumber: (*int)(22),
-      ColumnNumber: (*int)(7),
+      ColumnNumber: (*int)(14),
       Text: (*string)(<nil>)
     },
     Value: (schema.Schema) {
@@ -77,7 +77,7 @@
       Language: (string) (len=4) "Ruby",
       LanguageType: (string) (len=11) "programming",
       LineNumber: (*int)(23),
-      ColumnNumber: (*int)(7),
+      ColumnNumber: (*int)(14),
       Text: (*string)(<nil>)
     },
     Value: (schema.Schema) {
@@ -100,7 +100,7 @@
       Language: (string) (len=4) "Ruby",
       LanguageType: (string) (len=11) "programming",
       LineNumber: (*int)(24),
-      ColumnNumber: (*int)(7),
+      ColumnNumber: (*int)(16),
       Text: (*string)(<nil>)
     },
     Value: (schema.Schema) {

pkg/detectors/rails/personal_data/personal_data.go

@@ -45,27 +45,47 @@ func ExtractFromDatabaseSchema(
 
 	uuidHolder := parserschema.NewUUIDHolder()
 
-	return tree.Query(rubyDatabaseSchemaQuery, func(captures parser.Captures) error {
+	err = tree.Query(rubyDatabaseSchemaQuery, func(captures parser.Captures) error {
 		tableNode := captures["table_name"]
 		tableName := stripQuotes(tableNode.Content())
-		columnName := stripQuotes(captures["column_name"].Content())
+		columnNode := captures["column_name"]
+		columnName := stripQuotes(columnNode.Content())
 		columnTypeNode := captures["type"]
 		columnType := columnTypeNode.Content()
 
 		objectUUID := uuidHolder.Assign(tableNode.ID(), idGenerator)
 		fieldUUID := uuidHolder.Assign(columnTypeNode.ID(), idGenerator)
 
-		report.AddSchema(detectors.DetectorRails, schema.Schema{
+		currentSchema := schema.Schema{
 			ObjectName:      tableName,
 			ObjectUUID:      objectUUID,
 			FieldName:       columnName,
 			FieldUUID:       fieldUUID,
 			FieldType:       columnType,
 			SimpleFieldType: convertToSimpleType(columnType),
-		}, columnTypeNode.Source(false))
+		}
+		if !report.SchemaGroupIsOpen() {
+			source := tableNode.Source(false)
+			report.SchemaGroupBegin(
+				detectors.DetectorRails,
+				tableNode,
+				currentSchema,
+				&source,
+			)
+		}
+		source := columnNode.Source(false)
+		report.SchemaGroupAddItem(
+			columnNode,
+			currentSchema,
+			&source,
+		)
 
 		return nil
 	})
+
+	report.SchemaGroupEnd(idGenerator)
+
+	return err
 }
 
 func stripQuotes(value string) string {