refactor: remove ellipsis support from Ruby (#176)

* refactor: remove ellipsis from ruby * test: add coverage for ruby logger and csv custom detectors
Bearer · Nov 29, 2022 · 9cf39d2 · 9cf39d2
1 parent fd22b9f
commit 9cf39d2
Show file tree

Hide file tree

Showing 9 changed files with 178 additions and 15 deletions.
diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_ruby_logger b/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_ruby_logger
@@ -0,0 +1,39 @@
+data_types:
+    - name: Email Address
+      detectors:
+        - name: detect_ruby_logger
+          locations:
+            - filename: testdata/ruby/detect_ruby_logger.rb
+              line_number: 3
+        - name: ruby
+          locations:
+            - filename: testdata/ruby/detect_ruby_logger.rb
+              line_number: 3
+    - name: Physical Address
+      detectors:
+        - name: detect_ruby_logger
+          locations:
+            - filename: testdata/ruby/detect_ruby_logger.rb
+              line_number: 4
+        - name: ruby
+          locations:
+            - filename: testdata/ruby/detect_ruby_logger.rb
+              line_number: 4
+risks:
+    - detector_id: detect_ruby_logger
+      data_types:
+        - name: Email Address
+          stored: false
+          locations:
+            - filename: testdata/ruby/detect_ruby_logger.rb
+              line_number: 3
+        - name: Physical Address
+          stored: false
+          locations:
+            - filename: testdata/ruby/detect_ruby_logger.rb
+              line_number: 4
+components: []
+
+
+--
+
diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_file_detection b/integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_file_detection
@@ -0,0 +1,84 @@
+data_types:
+    - name: Email Address
+      detectors:
+        - name: ruby
+          locations:
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 5
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 12
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 16
+        - name: ruby_file_detection
+          locations:
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 5
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 12
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 16
+    - name: Emails
+      detectors:
+        - name: ruby
+          locations:
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 14
+    - name: Firstname
+      detectors:
+        - name: ruby
+          locations:
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 6
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 16
+        - name: ruby_file_detection
+          locations:
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 6
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 16
+    - name: Lastname
+      detectors:
+        - name: ruby
+          locations:
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 7
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 16
+        - name: ruby_file_detection
+          locations:
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 7
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 16
+risks:
+    - detector_id: ruby_file_detection
+      data_types:
+        - name: Email Address
+          stored: false
+          locations:
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 5
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 12
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 16
+        - name: Firstname
+          stored: false
+          locations:
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 6
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 16
+        - name: Lastname
+          stored: false
+          locations:
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 7
+            - filename: testdata/ruby/ruby_file_detection.rb
+              line_number: 16
+components: []
+
+
+--
+
diff --git a/integration/custom_detectors/custom_detectors_test.go b/integration/custom_detectors/custom_detectors_test.go
@@ -0,0 +1,24 @@
+package integration_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/bearer/curio/integration/internal/testhelper"
+)
+
+func newScanTest(language, name, filename string) testhelper.TestCase {
+	arguments := []string{"scan", filepath.Join("testdata", language, filename), "--report=dataflow", "--format=yaml"}
+	options := testhelper.TestCaseOptions{StartWorker: true}
+
+	return testhelper.NewTestCase(name, arguments, options)
+}
+
+func TestCustomDetectors(t *testing.T) {
+	tests := []testhelper.TestCase{
+		newScanTest("ruby", "detect_ruby_logger", "detect_ruby_logger.rb"),
+		newScanTest("ruby", "ruby_file_detection", "ruby_file_detection.rb"),
+	}
+
+	testhelper.RunTests(t, tests)
+}
diff --git a/integration/custom_detectors/testdata/ruby/detect_ruby_logger.rb b/integration/custom_detectors/testdata/ruby/detect_ruby_logger.rb
@@ -0,0 +1,5 @@
+logger.info(
+  "user info are:",
+  user.email,
+  user.address
+)
diff --git a/integration/custom_detectors/testdata/ruby/ruby_file_detection.rb b/integration/custom_detectors/testdata/ruby/ruby_file_detection.rb
@@ -0,0 +1,18 @@
+CSV.open("path/to/user.csv", "wb") do |csv|
+  csv << ["email", "first_name", "last_name"]
+	users.each do |user|
+		csv << [
+			user.email,
+			user.first_name,
+			user.last_name
+		]
+	end
+end
+
+File.open("users.log", "w") { |f| f.write "#{Time.now} - User #{user.email} logged in\n" }
+
+File.open(user.emails, "users.csv", "w") do |f|
+	users.each do |user|
+		f.write "#{user.email},#{user.first_name},#{user.last_name}"
+	end
+end
diff --git a/integration/flags/.snapshots/TestInitCommand-init b/integration/flags/.snapshots/TestInitCommand-init
@@ -115,17 +115,17 @@ scan:
                 - ruby
             patterns:
                 - |
-                  CSV.open(...) { <$DATA_TYPE> }
+                  CSV.open { <$DATA_TYPE> }
                 - |
-                  CSV.open(...) do
+                  CSV.open do
                     <$DATA_TYPE>
                   end
                 - |
-                  File.open(...) do
+                  File.open do
                     <$DATA_TYPE>
                   end
                 - |
-                  File.open(...) { <$DATA_TYPE> }
+                  File.open { <$DATA_TYPE> }
             param_parenting: true
             processors: []
             root_singularize: false

diff --git a/pkg/commands/process/settings/custom_detector.yml b/pkg/commands/process/settings/custom_detector.yml
@@ -18,17 +18,17 @@ ruby_file_detection:
     - ruby
   patterns:
     - |
-      CSV.open(...) { <$DATA_TYPE> }
+      CSV.open { <$DATA_TYPE> }
     - |
-      CSV.open(...) do
+      CSV.open do
         <$DATA_TYPE>
       end
     - |
-      File.open(...) do
+      File.open do
         <$DATA_TYPE>
       end
     - |
-      File.open(...) { <$DATA_TYPE> }
+      File.open { <$DATA_TYPE> }
   param_parenting: true
   metavars: {}
   stored: false

diff --git a/pkg/detectors/ruby/custom_detector/compile_pattern.go b/pkg/detectors/ruby/custom_detector/compile_pattern.go
@@ -13,12 +13,10 @@ import (
 var classNameRegex = regexp.MustCompile(`\$CLASS_NAME`)
 var argumentsRegex = regexp.MustCompile(`<\$ARGUMENT>`)
 var dataTypeRegex = regexp.MustCompile(`<\$DATA_TYPE>`)
-var ellipsisRegex = regexp.MustCompile(`\.\.\.`)
 
 func (detector *Detector) CompilePattern(Rule string, idGenerator nodeid.Generator) (config.CompiledRule, error) {
 	reworkedRule := classNameRegex.ReplaceAll([]byte(Rule), []byte("Var_Class_Name"+idGenerator.GenerateId()))
 	reworkedRule = argumentsRegex.ReplaceAll([]byte(reworkedRule), []byte("Var_Arguments"+idGenerator.GenerateId()))
-	reworkedRule = ellipsisRegex.ReplaceAll([]byte(reworkedRule), []byte("Var_Ellipsis"+idGenerator.GenerateId()))
 	reworkedRule = dataTypeRegex.ReplaceAll([]byte(reworkedRule), []byte("Var_DataTypes"+idGenerator.GenerateId()))
 
 	tree, err := parser.ParseBytes(&file.FileInfo{}, &file.Path{}, []byte(reworkedRule), language, 0)

diff --git a/pkg/detectors/ruby/custom_detector/custom_detector.go b/pkg/detectors/ruby/custom_detector/custom_detector.go
@@ -24,11 +24,6 @@ func (detector *Detector) IsParam(node *parser.Node) (isTerminating bool, should
 			return
 		}
 
-		if strings.Index(node.Content(), "Var_Ellipsis") == 0 {
-			shouldIgnore = true
-			return
-		}
-
 		// get simple string identifiers
 		param = &config.Param{
 			StringMatch: node.Content(),