chore: clean up and add data category UUID to data type (#141)

* feat: update data category UUIDs

* feat: update data type JSON to include category UUID

* feat: clean up Data Type type

* chore: add temp folder for scripts

* feat: update code for data type changes and to prefer UUIDs

* feat: add internal option for dataflow output

feat: clean up data type uuid output

* chore: update snapshots

* fix: update data type use

.gitignore

@@ -7,4 +7,6 @@ curio-build
 .DS_Store
 
 # curio binaries
-curio
+curio
+
+temp/

battle_tests/sheet/sheet.go

@@ -116,7 +116,7 @@ func (client *GoogleSheets) CreateDocument(tagName string, parentFolderId string
 		for index := range data_types {
 			headerCells = append(headerCells, &sheets.CellData{
 				UserEnteredValue: &sheets.ExtendedValue{
-					StringValue: &data_types[index].DataCategoryName,
+					StringValue: &data_types[index].Name,
 				},
 			})
 		}
@@ -253,10 +253,10 @@ func (client *GoogleSheets) InsertMetrics(documentID string, metrics *metricssca
 
 	empty_value := float64(0)
 	for _, data_type := range db.Default().DataTypes {
-		if _, exists := holder[data_type.DataCategoryName]; exists {
+		if _, exists := holder[data_type.Name]; exists {
 			values = append(values, &sheets.CellData{
 				UserEnteredValue: &sheets.ExtendedValue{
-					NumberValue: holder[data_type.DataCategoryName].occurrences,
+					NumberValue: holder[data_type.Name].occurrences,
 				},
 			})
 		} else {

integration/flags/.snapshots/TestReportFlags-domain-resolution-disabled

@@ -1,4 +1,4 @@
-[{"detector_type":"detect_ruby_logger","source":{"column_number":1,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"data_category_name":"Unique Identifier","default_category":"Identification","id":86,"uuid":"12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9"},"decision":{"reason":"valid_object_with_invalid_properties","state":"invalid"},"name":"user"},"field_name":"user","field_type":"","field_type_simple":"unknown","object_name":""}},{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":29,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":39,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]
+[{"detector_type":"detect_ruby_logger","source":{"column_number":1,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"14124881-6b92-4fc5-8005-ea7c1c09592e","name":"Unique Identifier","uuid":"12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9"},"decision":{"reason":"valid_object_with_invalid_properties","state":"invalid"},"name":"user"},"field_name":"user","field_type":"","field_type_simple":"unknown","object_name":""}},{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":29,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":39,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]
 
 --
 

integration/flags/.snapshots/TestReportFlags-format-json

@@ -1,4 +1,4 @@
-[{"detector_type":"detect_ruby_logger","source":{"column_number":1,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"data_category_name":"Unique Identifier","default_category":"Identification","id":86,"uuid":"12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9"},"decision":{"reason":"valid_object_with_invalid_properties","state":"invalid"},"name":"user"},"field_name":"user","field_type":"","field_type_simple":"unknown","object_name":""}},{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":29,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":39,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]
+[{"detector_type":"detect_ruby_logger","source":{"column_number":1,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"14124881-6b92-4fc5-8005-ea7c1c09592e","name":"Unique Identifier","uuid":"12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9"},"decision":{"reason":"valid_object_with_invalid_properties","state":"invalid"},"name":"user"},"field_name":"user","field_type":"","field_type_simple":"unknown","object_name":""}},{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":29,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":39,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]
 
 --
 

integration/flags/.snapshots/TestReportFlags-format-yaml

@@ -10,9 +10,8 @@
   value:
     classification:
         data_type:
-            data_category_name: Unique Identifier
-            default_category: Identification
-            id: 86
+            category_uuid: 14124881-6b92-4fc5-8005-ea7c1c09592e
+            name: Unique Identifier
             uuid: 12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9
         decision:
             reason: valid_object_with_invalid_properties