feat: add support for ellipsis (#167)

* feat: add support for ellipsis * chore: add data types in both datatypes and risks * chore: add more file to custom detection * chore: update examples
Bearer · Nov 25, 2022 · ea8b692 · ea8b692
1 parent ff51229
commit ea8b692
Show file tree

Hide file tree

Showing 17 changed files with 125 additions and 108 deletions.
diff --git a/integration/flags/.snapshots/TestInitCommand-init b/integration/flags/.snapshots/TestInitCommand-init
@@ -24,6 +24,20 @@ scan:
             root_lowercase: true
             metavars: {}
             stored: false
+        detect_rails_logger:
+            disabled: false
+            type: risk
+            languages:
+                - ruby
+            patterns:
+                - |
+                  Rails.logger.info(<$ARGUMENT>)
+            param_parenting: false
+            processors: []
+            root_singularize: false
+            root_lowercase: false
+            metavars: {}
+            stored: false
         detect_ruby_logger:
             disabled: false
             type: risk
@@ -94,6 +108,30 @@ scan:
             root_lowercase: true
             metavars: {}
             stored: false
+        ruby_file_detection:
+            disabled: false
+            type: risk
+            languages:
+                - ruby
+            patterns:
+                - |
+                  CSV.open(...) { <$DATA_TYPE> }
+                - |
+                  CSV.open(...) do
+                    <$DATA_TYPE>
+                  end
+                - |
+                  File.open(...) do
+                    <$DATA_TYPE>
+                  end
+                - |
+                  File.open(...) { <$DATA_TYPE> }
+            param_parenting: true
+            processors: []
+            root_singularize: false
+            root_lowercase: false
+            metavars: {}
+            stored: false
     debug: false
     disable-domain-resolution: false
     domain-resolution-timeout: 3s

diff --git a/integration/flags/.snapshots/TestReportFlags-domain-resolution-disabled b/integration/flags/.snapshots/TestReportFlags-domain-resolution-disabled
@@ -1,4 +1,4 @@
-[{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info('testing', user.something.something3)","line_number":1}}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something","parent":{"content":"logger.info('testing', user.something.something3)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":29,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":39,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]
+[{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
 
 --
 
diff --git a/integration/flags/.snapshots/TestReportFlags-format-json b/integration/flags/.snapshots/TestReportFlags-format-json
@@ -1,4 +1,4 @@
-[{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info('testing', user.something.something3)","line_number":1}}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something","parent":{"content":"logger.info('testing', user.something.something3)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":29,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":39,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]
+[{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
 
 --
 
diff --git a/integration/flags/.snapshots/TestReportFlags-format-yaml b/integration/flags/.snapshots/TestReportFlags-format-yaml
@@ -9,33 +9,20 @@
   type: custom_classified
   value:
     classification:
+        data_type:
+            category_uuid: cef587dd-76db-430b-9e18-7b031e1a193b
+            name: Email Address
+            uuid: 22e24c62-82d3-4b72-827c-e261533331bd
         decision:
-            reason: invalid_property
-            state: invalid
-        name: something
-    field_name: something
+            reason: known_pattern
+            state: valid
+        name: email
+    field_name: email
     field_type: ""
     field_type_simple: unknown
     object_name: user
     parent:
-        content: logger.info('testing', user.something.something3)
-        line_number: 1
-- detector_type: detect_ruby_logger
-  source:
-    column_number: 16
-    filename: main.rb
-    language: Ruby
-    language_type: programming
-    line_number: 1
-    text: null
-  type: custom_classified
-  value:
-    field_name: something3
-    field_type: ""
-    field_type_simple: unknown
-    object_name: something
-    parent:
-        content: logger.info('testing', user.something.something3)
+        content: logger.info("user info", user.email)
         line_number: 1
 - detector_type: ruby
   source:
@@ -53,7 +40,7 @@
     object_name: logger
 - detector_type: ruby
   source:
-    column_number: 29
+    column_number: 31
     filename: main.rb
     language: Ruby
     language_type: programming
@@ -62,75 +49,18 @@
   type: schema_classified
   value:
     classification:
+        data_type:
+            category_uuid: cef587dd-76db-430b-9e18-7b031e1a193b
+            name: Email Address
+            uuid: 22e24c62-82d3-4b72-827c-e261533331bd
         decision:
-            reason: invalid_property
-            state: invalid
-        name: something
-    field_name: something
+            reason: known_pattern
+            state: valid
+        name: email
+    field_name: email
     field_type: ""
     field_type_simple: unknown
     object_name: user
-- detector_type: ruby
-  source:
-    column_number: 39
-    filename: main.rb
-    language: Ruby
-    language_type: programming
-    line_number: 1
-    text: null
-  type: schema_classified
-  value:
-    field_name: something3
-    field_type: ""
-    field_type_simple: unknown
-    object_name: something
-- detector_type: ruby
-  source:
-    column_number: 14
-    filename: main.rb
-    language: Ruby
-    language_type: programming
-    line_number: 3
-    text: null
-  type: schema_classified
-  value:
-    field_name: info
-    field_type: ""
-    field_type_simple: unknown
-    object_name: logger
-- detector_type: ruby
-  source:
-    column_number: 24
-    filename: main.rb
-    language: Ruby
-    language_type: programming
-    line_number: 3
-    text: null
-  type: schema_classified
-  value:
-    classification:
-        decision:
-            reason: invalid_property
-            state: invalid
-        name: something
-    field_name: something
-    field_type: ""
-    field_type_simple: unknown
-    object_name: user
-- detector_type: ruby
-  source:
-    column_number: 34
-    filename: main.rb
-    language: Ruby
-    language_type: programming
-    line_number: 3
-    text: null
-  type: schema_classified
-  value:
-    field_name: something3
-    field_type: ""
-    field_type_simple: unknown
-    object_name: something
 
 
 --

diff --git a/integration/flags/.snapshots/TestReportFlags-health-context b/integration/flags/.snapshots/TestReportFlags-health-context
@@ -1,4 +1,4 @@
-[{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info('testing', user.something.something3)","line_number":1}}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something","parent":{"content":"logger.info('testing', user.something.something3)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":29,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":39,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]
+[{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
 
 --