From 9cf39d2ff401c33807a02d7a8239754b9f9cbbcd Mon Sep 17 00:00:00 2001 From: David Roe Date: Tue, 29 Nov 2022 15:07:01 +0000 Subject: [PATCH] refactor: remove ellipsis support from Ruby (#176) * refactor: remove ellipsis from ruby * test: add coverage for ruby logger and csv custom detectors --- .../TestCustomDetectors-detect_ruby_logger | 39 +++++++++ .../TestCustomDetectors-ruby_file_detection | 84 +++++++++++++++++++ .../custom_detectors/custom_detectors_test.go | 24 ++++++ .../testdata/ruby/detect_ruby_logger.rb | 5 ++ .../testdata/ruby/ruby_file_detection.rb | 18 ++++ .../flags/.snapshots/TestInitCommand-init | 8 +- .../process/settings/custom_detector.yml | 8 +- .../ruby/custom_detector/compile_pattern.go | 2 - .../ruby/custom_detector/custom_detector.go | 5 -- 9 files changed, 178 insertions(+), 15 deletions(-) create mode 100644 integration/custom_detectors/.snapshots/TestCustomDetectors-detect_ruby_logger create mode 100644 integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_file_detection create mode 100644 integration/custom_detectors/custom_detectors_test.go create mode 100644 integration/custom_detectors/testdata/ruby/detect_ruby_logger.rb create mode 100644 integration/custom_detectors/testdata/ruby/ruby_file_detection.rb diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_ruby_logger b/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_ruby_logger new file mode 100644 index 000000000..988d4e2a4 --- /dev/null +++ b/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_ruby_logger @@ -0,0 +1,39 @@ +data_types: + - name: Email Address + detectors: + - name: detect_ruby_logger + locations: + - filename: testdata/ruby/detect_ruby_logger.rb + line_number: 3 + - name: ruby + locations: + - filename: testdata/ruby/detect_ruby_logger.rb + line_number: 3 + - name: Physical Address + detectors: + - name: detect_ruby_logger + locations: + - filename: testdata/ruby/detect_ruby_logger.rb + line_number: 4 + - name: ruby + locations: + - filename: testdata/ruby/detect_ruby_logger.rb + line_number: 4 +risks: + - detector_id: detect_ruby_logger + data_types: + - name: Email Address + stored: false + locations: + - filename: testdata/ruby/detect_ruby_logger.rb + line_number: 3 + - name: Physical Address + stored: false + locations: + - filename: testdata/ruby/detect_ruby_logger.rb + line_number: 4 +components: [] + + +-- + diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_file_detection b/integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_file_detection new file mode 100644 index 000000000..ce721a3a3 --- /dev/null +++ b/integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_file_detection @@ -0,0 +1,84 @@ +data_types: + - name: Email Address + detectors: + - name: ruby + locations: + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 5 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 12 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 16 + - name: ruby_file_detection + locations: + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 5 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 12 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 16 + - name: Emails + detectors: + - name: ruby + locations: + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 14 + - name: Firstname + detectors: + - name: ruby + locations: + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 6 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 16 + - name: ruby_file_detection + locations: + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 6 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 16 + - name: Lastname + detectors: + - name: ruby + locations: + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 7 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 16 + - name: ruby_file_detection + locations: + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 7 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 16 +risks: + - detector_id: ruby_file_detection + data_types: + - name: Email Address + stored: false + locations: + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 5 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 12 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 16 + - name: Firstname + stored: false + locations: + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 6 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 16 + - name: Lastname + stored: false + locations: + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 7 + - filename: testdata/ruby/ruby_file_detection.rb + line_number: 16 +components: [] + + +-- + diff --git a/integration/custom_detectors/custom_detectors_test.go b/integration/custom_detectors/custom_detectors_test.go new file mode 100644 index 000000000..1ad9084ed --- /dev/null +++ b/integration/custom_detectors/custom_detectors_test.go @@ -0,0 +1,24 @@ +package integration_test + +import ( + "path/filepath" + "testing" + + "github.com/bearer/curio/integration/internal/testhelper" +) + +func newScanTest(language, name, filename string) testhelper.TestCase { + arguments := []string{"scan", filepath.Join("testdata", language, filename), "--report=dataflow", "--format=yaml"} + options := testhelper.TestCaseOptions{StartWorker: true} + + return testhelper.NewTestCase(name, arguments, options) +} + +func TestCustomDetectors(t *testing.T) { + tests := []testhelper.TestCase{ + newScanTest("ruby", "detect_ruby_logger", "detect_ruby_logger.rb"), + newScanTest("ruby", "ruby_file_detection", "ruby_file_detection.rb"), + } + + testhelper.RunTests(t, tests) +} diff --git a/integration/custom_detectors/testdata/ruby/detect_ruby_logger.rb b/integration/custom_detectors/testdata/ruby/detect_ruby_logger.rb new file mode 100644 index 000000000..d8fe43bbe --- /dev/null +++ b/integration/custom_detectors/testdata/ruby/detect_ruby_logger.rb @@ -0,0 +1,5 @@ +logger.info( + "user info are:", + user.email, + user.address +) diff --git a/integration/custom_detectors/testdata/ruby/ruby_file_detection.rb b/integration/custom_detectors/testdata/ruby/ruby_file_detection.rb new file mode 100644 index 000000000..834b1e771 --- /dev/null +++ b/integration/custom_detectors/testdata/ruby/ruby_file_detection.rb @@ -0,0 +1,18 @@ +CSV.open("path/to/user.csv", "wb") do |csv| + csv << ["email", "first_name", "last_name"] + users.each do |user| + csv << [ + user.email, + user.first_name, + user.last_name + ] + end +end + +File.open("users.log", "w") { |f| f.write "#{Time.now} - User #{user.email} logged in\n" } + +File.open(user.emails, "users.csv", "w") do |f| + users.each do |user| + f.write "#{user.email},#{user.first_name},#{user.last_name}" + end +end diff --git a/integration/flags/.snapshots/TestInitCommand-init b/integration/flags/.snapshots/TestInitCommand-init index 8cdbabd68..c1f7431df 100644 --- a/integration/flags/.snapshots/TestInitCommand-init +++ b/integration/flags/.snapshots/TestInitCommand-init @@ -115,17 +115,17 @@ scan: - ruby patterns: - | - CSV.open(...) { <$DATA_TYPE> } + CSV.open { <$DATA_TYPE> } - | - CSV.open(...) do + CSV.open do <$DATA_TYPE> end - | - File.open(...) do + File.open do <$DATA_TYPE> end - | - File.open(...) { <$DATA_TYPE> } + File.open { <$DATA_TYPE> } param_parenting: true processors: [] root_singularize: false diff --git a/pkg/commands/process/settings/custom_detector.yml b/pkg/commands/process/settings/custom_detector.yml index 733ba2928..8d92384a9 100644 --- a/pkg/commands/process/settings/custom_detector.yml +++ b/pkg/commands/process/settings/custom_detector.yml @@ -18,17 +18,17 @@ ruby_file_detection: - ruby patterns: - | - CSV.open(...) { <$DATA_TYPE> } + CSV.open { <$DATA_TYPE> } - | - CSV.open(...) do + CSV.open do <$DATA_TYPE> end - | - File.open(...) do + File.open do <$DATA_TYPE> end - | - File.open(...) { <$DATA_TYPE> } + File.open { <$DATA_TYPE> } param_parenting: true metavars: {} stored: false diff --git a/pkg/detectors/ruby/custom_detector/compile_pattern.go b/pkg/detectors/ruby/custom_detector/compile_pattern.go index d8fcd7538..b144c59da 100644 --- a/pkg/detectors/ruby/custom_detector/compile_pattern.go +++ b/pkg/detectors/ruby/custom_detector/compile_pattern.go @@ -13,12 +13,10 @@ import ( var classNameRegex = regexp.MustCompile(`\$CLASS_NAME`) var argumentsRegex = regexp.MustCompile(`<\$ARGUMENT>`) var dataTypeRegex = regexp.MustCompile(`<\$DATA_TYPE>`) -var ellipsisRegex = regexp.MustCompile(`\.\.\.`) func (detector *Detector) CompilePattern(Rule string, idGenerator nodeid.Generator) (config.CompiledRule, error) { reworkedRule := classNameRegex.ReplaceAll([]byte(Rule), []byte("Var_Class_Name"+idGenerator.GenerateId())) reworkedRule = argumentsRegex.ReplaceAll([]byte(reworkedRule), []byte("Var_Arguments"+idGenerator.GenerateId())) - reworkedRule = ellipsisRegex.ReplaceAll([]byte(reworkedRule), []byte("Var_Ellipsis"+idGenerator.GenerateId())) reworkedRule = dataTypeRegex.ReplaceAll([]byte(reworkedRule), []byte("Var_DataTypes"+idGenerator.GenerateId())) tree, err := parser.ParseBytes(&file.FileInfo{}, &file.Path{}, []byte(reworkedRule), language, 0) diff --git a/pkg/detectors/ruby/custom_detector/custom_detector.go b/pkg/detectors/ruby/custom_detector/custom_detector.go index 5c4e4f829..2a2bcb69c 100644 --- a/pkg/detectors/ruby/custom_detector/custom_detector.go +++ b/pkg/detectors/ruby/custom_detector/custom_detector.go @@ -24,11 +24,6 @@ func (detector *Detector) IsParam(node *parser.Node) (isTerminating bool, should return } - if strings.Index(node.Content(), "Var_Ellipsis") == 0 { - shouldIgnore = true - return - } - // get simple string identifiers param = &config.Param{ StringMatch: node.Content(),