chore: address random failures (#190)

* chore: address random failures

* fix: improve custom detector performance

* feat: disable domain resolution by default

* fix: fix detections

Co-authored-by: vjerci <27707350+vjerci@users.noreply.github.com>

integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_file_detection

@@ -56,15 +56,15 @@ risks:
               line_number: 12
               parent:
                 line_number: 12
-                content: 'File.open("users.log", "w") { |f| f.write "#{Time.now} - User #{user.email} logged in\n" }'
+                content: 'File.open("users.log", "w") { |f| f.write "#{Time.now} - User #{user_2.email} logged in\n" }'
             - filename: testdata/ruby/ruby_file_detection.rb
               line_number: 16
               parent:
                 line_number: 14
                 content: |-
-                    File.open(user.emails, "users.csv", "w") do |f|
-                    	users.each do |user|
-                    		f.write "#{user.email},#{user.first_name},#{user.last_name}"
+                    File.open(user_3.emails, "users.csv", "w") do |f|
+                    	users.each do |user_4|
+                    		f.write "#{user_4.email},#{user_4.first_name},#{user_4.last_name}"
                     	end
                     end
         - name: Firstname
@@ -90,9 +90,9 @@ risks:
               parent:
                 line_number: 14
                 content: |-
-                    File.open(user.emails, "users.csv", "w") do |f|
-                    	users.each do |user|
-                    		f.write "#{user.email},#{user.first_name},#{user.last_name}"
+                    File.open(user_3.emails, "users.csv", "w") do |f|
+                    	users.each do |user_4|
+                    		f.write "#{user_4.email},#{user_4.first_name},#{user_4.last_name}"
                     	end
                     end
         - name: Lastname
@@ -118,9 +118,9 @@ risks:
               parent:
                 line_number: 14
                 content: |-
-                    File.open(user.emails, "users.csv", "w") do |f|
-                    	users.each do |user|
-                    		f.write "#{user.email},#{user.first_name},#{user.last_name}"
+                    File.open(user_3.emails, "users.csv", "w") do |f|
+                    	users.each do |user_4|
+                    		f.write "#{user_4.email},#{user_4.first_name},#{user_4.last_name}"
                     	end
                     end
 components: []

integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_http_detection

@@ -60,8 +60,8 @@ risks:
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 3
               parent:
-                line_number: 94
-                content: HTTPX.get("http://my.api.com/users/search?first_name=#{user.first_name}")
+                line_number: 7
+                content: URI.encode_www_form(user_1)
         - name: Firstname
           stored: false
           locations:
@@ -70,11 +70,6 @@ risks:
               parent:
                 line_number: 7
                 content: URI.encode_www_form(user_1)
-            - filename: testdata/ruby/ruby_http_detection.rb
-              line_number: 12
-              parent:
-                line_number: 3
-                content: URI("http://my.api.com/users/search?ethnic_origin=#{user.ethnic_origin}")
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 19
               parent:
@@ -84,7 +79,7 @@ risks:
               line_number: 39
               parent:
                 line_number: 39
-                content: Typhoeus.get("http://my.api.com/users/search?first_name=#{user_2.first_name}")
+                content: Typhoeus.get("http://my.api.com/users/search?first_name=#{user_9.first_name}")
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 44
               parent:
@@ -104,12 +99,12 @@ risks:
               line_number: 77
               parent:
                 line_number: 77
-                content: Faraday.get("http://my.api.com/users/search?first_name=#{user.first_name}")
+                content: Faraday.get("http://my.api.com/users/search?first_name=#{user_2.first_name}")
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 94
               parent:
                 line_number: 94
-                content: HTTPX.get("http://my.api.com/users/search?first_name=#{user.first_name}")
+                content: HTTPX.get("http://my.api.com/users/search?first_name=#{user_2.first_name}")
         - name: Lastname
           stored: false
           locations:
@@ -122,7 +117,7 @@ risks:
               line_number: 12
               parent:
                 line_number: 94
-                content: HTTPX.get("http://my.api.com/users/search?first_name=#{user.first_name}")
+                content: HTTPX.get("http://my.api.com/users/search?first_name=#{user_2.first_name}")
     - detector_id: ruby_http_post_detection
       data_types:
         - name: Ethnic Origin
@@ -131,8 +126,8 @@ risks:
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 3
               parent:
-                line_number: 29
-                content: 'RestClient.post("http://my.api.com/users/create", { user: { first_name: "John", last_name: "Doe" } })'
+                line_number: 7
+                content: URI.encode_www_form(user_1)
         - name: Firstname
           stored: false
           locations:
@@ -145,22 +140,22 @@ risks:
               line_number: 12
               parent:
                 line_number: 12
-                content: 'Net::HTTP.post_form(uri, { user: { first_name: "John", last_name: "Doe" } })'
+                content: 'Net::HTTP.post_form(uri, { user_2: { first_name: "John", last_name: "Doe" } })'
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 21
               parent:
                 line_number: 22
-                content: Curl.post("http://my.api.com/users/create", user_3)
+                content: Curl.post("http://my.api.com/users/create", user_4)
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 24
               parent:
                 line_number: 24
-                content: 'Curl.post("http://my.api.com/users/create", { user: { first_name: "John", last_name: "Doe" } })'
+                content: 'Curl.post("http://my.api.com/users/create", { user_5: { first_name: "John", last_name: "Doe" } })'
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 29
               parent:
                 line_number: 29
-                content: 'RestClient.post("http://my.api.com/users/create", { user: { first_name: "John", last_name: "Doe" } })'
+                content: 'RestClient.post("http://my.api.com/users/create", { user_6: { first_name: "John", last_name: "Doe" } })'
         - name: Lastname
           stored: false
           locations:
@@ -173,22 +168,22 @@ risks:
               line_number: 12
               parent:
                 line_number: 12
-                content: 'Net::HTTP.post_form(uri, { user: { first_name: "John", last_name: "Doe" } })'
+                content: 'Net::HTTP.post_form(uri, { user_2: { first_name: "John", last_name: "Doe" } })'
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 21
               parent:
                 line_number: 22
-                content: Curl.post("http://my.api.com/users/create", user_3)
+                content: Curl.post("http://my.api.com/users/create", user_4)
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 24
               parent:
                 line_number: 24
-                content: 'Curl.post("http://my.api.com/users/create", { user: { first_name: "John", last_name: "Doe" } })'
+                content: 'Curl.post("http://my.api.com/users/create", { user_5: { first_name: "John", last_name: "Doe" } })'
             - filename: testdata/ruby/ruby_http_detection.rb
               line_number: 29
               parent:
                 line_number: 29
-                content: 'RestClient.post("http://my.api.com/users/create", { user: { first_name: "John", last_name: "Doe" } })'
+                content: 'RestClient.post("http://my.api.com/users/create", { user_6: { first_name: "John", last_name: "Doe" } })'
 components: []
 
 

integration/custom_detectors/custom_detectors_test.go

@@ -21,7 +21,7 @@ func TestCustomDetectors(t *testing.T) {
 		newScanTest("ruby", "detect_ruby_logger", "detect_ruby_logger.rb"),
 		newScanTest("ruby", "ruby_file_detection", "ruby_file_detection.rb"),
 		newScanTest("ruby", "ssl_certificate_verification_disabled", "ssl_certificate_verification_disabled.rb"),
-		// newScanTest("ruby", "ruby_http_detection", "ruby_http_detection.rb"),
+		newScanTest("ruby", "ruby_http_detection", "ruby_http_detection.rb"),
 	}
 
 	testhelper.RunTests(t, tests)

integration/custom_detectors/testdata/ruby/ruby_file_detection.rb

@@ -9,10 +9,10 @@
 	end
 end
 
-File.open("users.log", "w") { |f| f.write "#{Time.now} - User #{user.email} logged in\n" }
+File.open("users.log", "w") { |f| f.write "#{Time.now} - User #{user_2.email} logged in\n" }
 
-File.open(user.emails, "users.csv", "w") do |f|
-	users.each do |user|
-		f.write "#{user.email},#{user.first_name},#{user.last_name}"
+File.open(user_3.emails, "users.csv", "w") do |f|
+	users.each do |user_4|
+		f.write "#{user_4.email},#{user_4.first_name},#{user_4.last_name}"
 	end
 end

integration/custom_detectors/testdata/ruby/ruby_http_detection.rb

@@ -1,6 +1,6 @@
 ## URI
 
-uri = URI("http://my.api.com/users/search?ethnic_origin=#{user.ethnic_origin}")
+uri = URI("http://my.api.com/users/search?ethnic_origin=#{user_1.ethnic_origin}")
 
 uri = URI('http://my.api.com/users/search')
 user_1 = { first_name: "John", last_name: "Doe" }
@@ -9,34 +9,34 @@
 
 ## Net::HTTP
 
-response = Net::HTTP.post_form(uri, { user: { first_name: "John", last_name: "Doe" } })
+response = Net::HTTP.post_form(uri, { user_2: { first_name: "John", last_name: "Doe" } })
 
 
 ## Curl
 
 User = Struct.new(:first_name, :last_name, keyword_init: true)
-user_2 = User.new(first_name: "first", last_name: "last")
+user_3 = User.new(first_name: "first", last_name: "last")
 response = Curl.get("http://my.api.com/users/search?first_name=#{user_2.first_name}")
 
-user_3 = { first_name: "John", last_name: "Doe" }
-response = Curl.post("http://my.api.com/users/create", user_3)
+user_4 = { first_name: "John", last_name: "Doe" }
+response = Curl.post("http://my.api.com/users/create", user_4)
 
-response = Curl.post("http://my.api.com/users/create", { user: { first_name: "John", last_name: "Doe" } })
+response = Curl.post("http://my.api.com/users/create", { user_5: { first_name: "John", last_name: "Doe" } })
 
 
 ## RestClient
 
-RestClient.post("http://my.api.com/users/create", { user: { first_name: "John", last_name: "Doe" } })
+RestClient.post("http://my.api.com/users/create", { user_6: { first_name: "John", last_name: "Doe" } })
 
 
 ## Typhoeus
 
-options = { body: { user: { first_name: "John", last_name: "Doe" } } }
+options = { body: { user_7: { first_name: "John", last_name: "Doe" } } }
 response = Typhoeus.post("http://my.api.com/users/create", options)
 
-response = Typhoeus.post("http://my.api.com/users/create", { body: { user: { first_name: "John", last_name: "Doe" } } })
+response = Typhoeus.post("http://my.api.com/users/create", { body: { user_8: { first_name: "John", last_name: "Doe" } } })
 
-Typhoeus.get("http://my.api.com/users/search?first_name=#{user_2.first_name}")
+Typhoeus.get("http://my.api.com/users/search?first_name=#{user_9.first_name}")
 
 
 ## HTTParty
@@ -60,23 +60,23 @@
 
 HTTP.get("http://my.api.com/users/search?first_name=#{user_2.first_name}")
 
-HTTP.get("http://my.api.com/users/search", params: { user: { first_name: "John" } })
+HTTP.get("http://my.api.com/users/search", params: { user_8: { first_name: "John" } })
 
-HTTP.post("http://my.api.com/users/create", form: { user: { first_name: "John", last_name: "Doe" } })
+HTTP.post("http://my.api.com/users/create", form: { user_9: { first_name: "John", last_name: "Doe" } })
 
 
 ## Excon
 
 Excon.get("http://my.api.com/users/search?first_name=#{user_2.first_name}")
 
-Excon.post("http://my.api.com/users/create", body: { user: { first_name: "John", last_name: "Doe" } })
+Excon.post("http://my.api.com/users/create", body: { user_10: { first_name: "John", last_name: "Doe" } })
 
 
 ## Faraday
 
-Faraday.get("http://my.api.com/users/search?first_name=#{user.first_name}")
+Faraday.get("http://my.api.com/users/search?first_name=#{user_2.first_name}")
 
-params_2 = { user: { first_name: "John", last_name: "Doe" } }
+params_2 = { user_11: { first_name: "John", last_name: "Doe" } }
 
 encoded_params = URI.encode_www_form(params_2)
 
@@ -89,6 +89,6 @@
 
 ## HTTPX
 
-HTTPX.post("http://my.api.com/users/create", json: { user: { first_name: "John", last_name: "Doe" } })
+HTTPX.post("http://my.api.com/users/create", json: { user_12: { first_name: "John", last_name: "Doe" } })
 
-HTTPX.get("http://my.api.com/users/search?first_name=#{user.first_name}")
+HTTPX.get("http://my.api.com/users/search?first_name=#{user_2.first_name}")

integration/flags/.snapshots/TestInitCommand-init

@@ -272,18 +272,18 @@ scan:
             stored: false
             detect_presence: true
     debug: false
-    disable-domain-resolution: false
+    disable-domain-resolution: true
     domain-resolution-timeout: 3s
     internal-domains: []
     quiet: false
     skip-path: []
 worker:
     existing-worker: ""
-    file-size-max: 25000000
+    file-size-max: 100000
     files-to-batch: 1
     memory-max: 800000000
     timeout: 10m0s
-    timeout-file-max: 5m0s
+    timeout-file-max: 30s
     timeout-file-min: 5s
     timeout-file-second-per-bytes: 10000
     timeout-worker-online: 1m0s

integration/flags/.snapshots/TestMetadataFlags-help-scan

@@ -12,7 +12,7 @@ Examples:
 Scan Flags
       --context string                       Expand context of schema classification e.g., --context=health, to include data types particular to health
       --debug                                Enable debug logs
-      --disable-domain-resolution            Do not attempt to resolve detected domains during classification (default false), e.g. --disable-domain-resolution=true
+      --disable-domain-resolution            Do not attempt to resolve detected domains during classification (default false), e.g. --disable-domain-resolution=true (default true)
       --domain-resolution-timeout duration   Set timeout when attempting to resolve detected domains during classification (default 3 seconds), e.g. --domain-resolution-timeout=3s (default 3s)
       --internal-domains strings             Define regular expressions for better classification of private or unreachable domains e.g. --internal-domains="*.my-company.com,private.sh"
       --quiet                                Suppress non-essential messages
@@ -24,11 +24,11 @@ Policy Flags
 
 Worker Flags
       --existing-worker string              URL of an existing worker
-      --file-size-max int                   ignore files with file size larger than this config (default 25000000)
+      --file-size-max int                   ignore files with file size larger than this config (default 100000)
       --files-to-batch int                  number of files to batch to worker (default 1)
       --memory-max int                      if memory needed to scan a file surpasses this limit, skip the file (default 800000000)
       --timeout duration                    time allowed to complete scan (default 10m0s)
-      --timeout-file-max duration           maximum timeout assigned to scanning file, this config superseeds timeout-second-per-bytes (default 5m0s)
+      --timeout-file-max duration           maximum timeout assigned to scanning file, this config superseeds timeout-second-per-bytes (default 30s)
       --timeout-file-min duration           minimum timeout assigned to scanning file, this config superseeds timeout-second-per-bytes (default 5s)
       --timeout-file-second-per-bytes int   number of file size bytes producing a second of timeout assigned to scanning a file (default 10000)
       --timeout-worker-online duration      maximum time for worker process to come online (default 1m0s)

integration/flags/.snapshots/TestMetadataFlags-scan-help

@@ -12,7 +12,7 @@ Examples:
 Scan Flags
       --context string                       Expand context of schema classification e.g., --context=health, to include data types particular to health
       --debug                                Enable debug logs
-      --disable-domain-resolution            Do not attempt to resolve detected domains during classification (default false), e.g. --disable-domain-resolution=true
+      --disable-domain-resolution            Do not attempt to resolve detected domains during classification (default false), e.g. --disable-domain-resolution=true (default true)
       --domain-resolution-timeout duration   Set timeout when attempting to resolve detected domains during classification (default 3 seconds), e.g. --domain-resolution-timeout=3s (default 3s)
       --internal-domains strings             Define regular expressions for better classification of private or unreachable domains e.g. --internal-domains="*.my-company.com,private.sh"
       --quiet                                Suppress non-essential messages
@@ -24,11 +24,11 @@ Policy Flags
 
 Worker Flags
       --existing-worker string              URL of an existing worker
-      --file-size-max int                   ignore files with file size larger than this config (default 25000000)
+      --file-size-max int                   ignore files with file size larger than this config (default 100000)
       --files-to-batch int                  number of files to batch to worker (default 1)
       --memory-max int                      if memory needed to scan a file surpasses this limit, skip the file (default 800000000)
       --timeout duration                    time allowed to complete scan (default 10m0s)
-      --timeout-file-max duration           maximum timeout assigned to scanning file, this config superseeds timeout-second-per-bytes (default 5m0s)
+      --timeout-file-max duration           maximum timeout assigned to scanning file, this config superseeds timeout-second-per-bytes (default 30s)
       --timeout-file-min duration           minimum timeout assigned to scanning file, this config superseeds timeout-second-per-bytes (default 5s)
       --timeout-file-second-per-bytes int   number of file size bytes producing a second of timeout assigned to scanning a file (default 10000)
       --timeout-worker-online duration      maximum time for worker process to come online (default 1m0s)

integration/policies/.snapshots/TestPolicies-http_get_parameters

@@ -4,12 +4,16 @@ critical:
       line_number: 4
       filename: testdata/ruby/http_get_parameters.rb
       category_group: Personal data
+      parent_line_number: 5
+      parent_content: URI.encode_www_form(user)
 high:
     - policy_name: HTTP GET parameters
       policy_description: Sending data as HTTP GET parameters
       line_number: 1
       filename: testdata/ruby/http_get_parameters.rb
       category_group: Sensitive data
+      parent_line_number: 1
+      parent_content: URI("http://my.api.com/users/search?ethnic_origin=#{user_1.ethnic_origin}")
 
 
 --