feat: application level encryption (#157)

* feat: non encrypted properties

* feat: refractor encryption fails processor

* feat: add support for rego processor

* feat: reconciliation of sql and ruby detector

* feat: add pluralize

* merge: fix merge conflicts

* feat: add support for custom detector type

* feat: skip adding verifiers

* chore: go mod tidy

* feat: add verified by fields

* fix: manually resolve merge conflicts

* chore: refractor verifiers

* test: add integration tests for verified by

* fix: linting issue

* fix: manually resolve merge conflicts

* chore: snake case rego variables

* chore: remove tanker from default processors

* fix: tests failing

* feat: add root singularize option

* fix: correct typo

* fix: tests

* fix: remove debug log

* fix: tests

* feat: remove detect ruby classes

go.mod

@@ -5,6 +5,7 @@ go 1.18
 require (
 	github.com/TwiN/go-color v1.4.0
 	github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
+	github.com/gertd/go-pluralize v0.2.1
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-enry/go-enry/v2 v2.8.3
 	github.com/google/uuid v1.3.0

go.sum

@@ -94,6 +94,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/gertd/go-pluralize v0.2.1 h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlLgiA=
+github.com/gertd/go-pluralize v0.2.1/go.mod h1:rbYaKDbsXxmRfr8uygAEKhOWsjyrrqrkHVpZvoOp8zk=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gitleaks/go-gitdiff v0.8.0 h1:7aExTZm+K/M/EQKOyYcub8rIAdWK6ONxPGuRzxmWW+0=

integration/flags/.snapshots/TestInitCommand-init

@@ -8,14 +8,90 @@ report:
 scan:
     context: ""
     custom_detector:
+        detect_encrypted_ruby_class_properties:
+            disabled: false
+            type: verifier
+            languages:
+                - ruby
+            patterns:
+                - |
+                  class $CLASS_NAME < ApplicationRecord
+                    encrypts <$ARGUMENT>
+                  end
+            param_parenting: true
+            processors: []
+            root_singularize: true
+            root_lowercase: true
+            metavars: {}
+            stored: false
         detect_ruby_logger:
             disabled: false
+            type: risk
             languages:
                 - ruby
             patterns:
                 - |
                   logger.info(<$ARGUMENT>)
             param_parenting: false
+            processors: []
+            root_singularize: false
+            root_lowercase: false
+            metavars: {}
+            stored: false
+        detect_sql_create_public_table:
+            disabled: false
+            type: data_type
+            languages:
+                - sql
+            patterns:
+                - |
+                  CREATE TABLE public.$TABLE_NAME (
+                    <$COLUMN>
+                  )
+            param_parenting: true
+            processors:
+                - query: |
+                    verified_by = data.bearer.encrypted_verified.verified_by
+                    encrypted = data.bearer.encrypted_verified.encrypted
+                  modules:
+                    - name: bearer.encrypted_verified
+                      content: |-
+                        package bearer.encrypted_verified
+
+                        import future.keywords
+
+
+                        default encrypted := false
+
+
+                        ruby_encrypted[location] {
+                            some detection in input.all_detections
+                            detection.detector_type == "detect_encrypted_ruby_class_properties"
+                            detection.value.classification.decision.state == "valid"
+                            location = detection
+                        }
+
+                        encrypted = true {
+                            some detection in ruby_encrypted
+                            detection.value.object_name == input.target.value.object_name
+                            detection.value.field_name == input.target.value.field_name
+                            input.target.value.field_name != ""
+                            input.target.value.object_name != ""
+                        }
+
+                        verified_by[verification] {
+                            some detection in ruby_encrypted
+                            detection.value.object_name == input.target.value.object_name
+                            detection.value.field_name == input.target.value.field_name
+
+                            verification = {
+                                "detector": "detect_encrypted_ruby_class_properties",
+                                "filename": detection.source.filename,
+                                "line_number": detection.source.line_number
+                            }
+                        }
+            root_singularize: true
+            root_lowercase: true
             metavars: {}
             stored: false
     debug: false

integration/flags/.snapshots/TestReportFlags-report-dataflow-verified-by

@@ -0,0 +1,69 @@
+data_types:
+    - name: Country
+      detectors:
+        - name: detect_sql_create_public_table
+          locations:
+            - filename: schema.sql
+              line_number: 8
+              encrypted: true
+              verified_by:
+                - detector: detect_encrypted_ruby_class_properties
+                  filename: user.rb
+                  line_number: 2
+        - name: ruby
+          locations:
+            - filename: user.rb
+              line_number: 2
+    - name: Date of birth
+      detectors:
+        - name: detect_sql_create_public_table
+          locations:
+            - filename: schema.sql
+              line_number: 6
+    - name: Email Address
+      detectors:
+        - name: detect_sql_create_public_table
+          locations:
+            - filename: schema.sql
+              line_number: 5
+              encrypted: true
+              verified_by:
+                - detector: detect_encrypted_ruby_class_properties
+                  filename: user.rb
+                  line_number: 2
+        - name: ruby
+          locations:
+            - filename: user.rb
+              line_number: 2
+    - name: Firstname
+      detectors:
+        - name: detect_sql_create_public_table
+          locations:
+            - filename: schema.sql
+              line_number: 3
+    - name: Lastname
+      detectors:
+        - name: detect_sql_create_public_table
+          locations:
+            - filename: schema.sql
+              line_number: 4
+    - name: Physical Address
+      detectors:
+        - name: detect_sql_create_public_table
+          locations:
+            - filename: schema.sql
+              line_number: 7
+              encrypted: true
+              verified_by:
+                - detector: detect_encrypted_ruby_class_properties
+                  filename: user.rb
+                  line_number: 2
+        - name: ruby
+          locations:
+            - filename: user.rb
+              line_number: 2
+components: []
+
+
+--
+

integration/flags/report_flags_test.go

@@ -33,6 +33,7 @@ func TestReportFlags(t *testing.T) {
 		newScanTest("format-yaml", []string{"--format=yaml"}, ""),
 		newScanTest("report-detectors", []string{"--report=detectors"}, ""),
 		newScanTest("report-dataflow", []string{"--report=dataflow"}, ""),
+		newScanProject("report-dataflow-verified-by", []string{"--report=dataflow", "--format=yaml"}, "", "verified_by"),
 		newScanProject("report-policies", []string{"--report=policies", "--format=yaml"}, "", "policies"),
 		newScanTest("output", []string{"--output=" + outputPath}, outputPath),
 		newScanTest("health-context", []string{"--context=health"}, ""),

integration/flags/testdata/verified_by/schema.sql

@@ -0,0 +1,9 @@
+CREATE TABLE public.Users (
+  id bigint NOT NULL,
+  first_name character varying NOT NULL,
+  last_name character varying NOT NULL,
+  email character varying NOT NULL,
+  tanker_encrypted_date_of_birth character varying NOT NULL,
+  city character varying NOT NULL,
+  country character varying NOT NULL,
+);

integration/flags/testdata/verified_by/user.rb

@@ -0,0 +1,3 @@
+class Users < ApplicationRecord
+    encrypts :email, :country, :city # requires the detection of the structure too
+end

pkg/commands/process/settings/custom_detector.yml

@@ -1,6 +1,38 @@
 detect_ruby_logger:
+  type: "risk"
   patterns:
     - |
       logger.info(<$ARGUMENT>)
   languages:
     - ruby
+detect_encrypted_ruby_class_properties:
+  type: "verifier"
+  patterns:
+    - |
+      class $CLASS_NAME < ApplicationRecord
+        encrypts <$ARGUMENT>
+      end
+  param_parenting: true
+  root_singularize: true
+  root_lowercase: true
+  languages:
+    - ruby
+detect_sql_create_public_table:
+  type: "data_type"
+  patterns:
+    - |
+      CREATE TABLE public.$TABLE_NAME (
+        <$COLUMN>
+      )
+  param_parenting: true
+  root_singularize: true
+  root_lowercase: true
+  languages:
+    - sql
+  processors:
+    - query: |
+        verified_by = data.bearer.encrypted_verified.verified_by
+        encrypted = data.bearer.encrypted_verified.encrypted
+      modules:
+        - path: processors/encrypted_verified.rego
+          name: bearer.encrypted_verified

pkg/commands/process/settings/processors/encrypted_verified.rego

@@ -0,0 +1,34 @@
+package bearer.encrypted_verified
+
+import future.keywords
+
+
+default encrypted := false
+
+
+ruby_encrypted[location] {
+    some detection in input.all_detections
+    detection.detector_type == "detect_encrypted_ruby_class_properties"
+    detection.value.classification.decision.state == "valid"
+    location = detection
+}
+
+encrypted = true {
+    some detection in ruby_encrypted
+    detection.value.object_name == input.target.value.object_name
+    detection.value.field_name == input.target.value.field_name
+    input.target.value.field_name != ""
+    input.target.value.object_name != ""
+}
+
+verified_by[verification] {
+    some detection in ruby_encrypted
+    detection.value.object_name == input.target.value.object_name
+    detection.value.field_name == input.target.value.field_name
+
+    verification = {
+        "detector": "detect_encrypted_ruby_class_properties",
+        "filename": detection.source.filename,
+        "line_number": detection.source.line_number
+    }
+}

pkg/commands/process/settings/settings.go

@@ -9,6 +9,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/bearer/curio/pkg/flag"
+	"github.com/bearer/curio/pkg/util/rego"
 )
 
 type Config struct {
@@ -26,28 +27,51 @@ var LevelHigh = "high"
 var LevelMedium = "medium"
 var LevelLow = "low"
 
+type Modules []*PolicyModule
+
 type Policy struct {
 	Query       string
-	Name        string
 	Id          string
+	Name        string
 	Description string
-	Modules     []*PolicyModule
 	Level       PolicyLevel
+	Modules     Modules
 }
 
 type PolicyModule struct {
-	Path    string
+	Path    string `yaml:"path,omitempty"`
 	Name    string
 	Content string
 }
 
+func (modules Modules) ToRegoModules() (output []rego.Module) {
+	for _, module := range modules {
+		output = append(output, rego.Module{
+			Name:    module.Name,
+			Content: module.Content,
+		})
+	}
+	return
+}
+
 type Rule struct {
 	Disabled       bool
+	Type           string
 	Languages      []string
 	Patterns       []string
 	ParamParenting bool `yaml:"param_parenting"`
-	Metavars       map[string]MetaVar
-	Stored         bool
+	Processors     []Processor
+
+	RootSingularize bool `yaml:"root_singularize"`
+	RootLowercase   bool `yaml:"root_lowercase"`
+
+	Metavars map[string]MetaVar
+	Stored   bool
+}
+
+type Processor struct {
+	Query   string
+	Modules Modules
 }
 
 type MetaVar struct {
@@ -65,6 +89,9 @@ var defaultPolicies []byte
 //go:embed policies/*
 var policiesFs embed.FS
 
+//go:embed processors/*
+var processorsFs embed.FS
+
 var CustomDetectorKey string = "scan.custom_detector"
 var PoliciesKey string = "scan.policies"
 
@@ -77,9 +104,24 @@ func FromOptions(opts flag.Options) (Config, error) {
 		}
 	}
 
+	for _, customDetector := range rules {
+		for _, processor := range customDetector.Processors {
+			for _, module := range processor.Modules {
+				if module.Path != "" {
+					content, err := processorsFs.ReadFile(module.Path)
+					if err != nil {
+						return Config{}, err
+					}
+					module.Content = string(content)
+					module.Path = ""
+				}
+			}
+		}
+	}
+
 	policies := DefaultPolicies()
 	if viper.IsSet(PoliciesKey) {
-		err := viper.UnmarshalKey(PoliciesKey, &rules)
+		err := viper.UnmarshalKey(PoliciesKey, &policies)
 		if err != nil {
 			return Config{}, err
 		}

pkg/detectors/custom/config/config.go

@@ -3,12 +3,14 @@ package config
 import "github.com/bearer/curio/pkg/commands/process/settings"
 
 type CompiledRule struct {
-	RuleName       string
-	Tree           string
-	Params         []Param
-	Metavars       map[string]settings.MetaVar
-	ParamParenting bool
-	Languages      []string
+	RuleName        string
+	Tree            string
+	Params          []Param
+	Metavars        map[string]settings.MetaVar
+	ParamParenting  bool
+	RootSingularize bool
+	RootLowercase   bool
+	Languages       []string
 }
 
 type Param struct {