Skip to content

Commit

Permalink
feat: implement rego from dataflow (#123)
Browse files Browse the repository at this point in the history
* feat: got opa query working

* chore: add opa dependency

* feat: load embeded policies

* feat: add locations to logger leaks policy

* fix: remove comment

* docs: update commands documentation

* Update pkg/classification/dependencies/dependencies.go

Co-authored-by: Cédric Fabianski <cedric@bearer.sh>

* Update pkg/report/output/output.go

Co-authored-by: Cédric Fabianski <cedric@bearer.sh>

* fix: report output

* chore: fix output file

* docs: update docs

* test: update tests

* test: update tests

* test: add test for policies

* fix: split query into 2 lines

Co-authored-by: Cédric Fabianski <cedric@bearer.sh>
  • Loading branch information
vjerci and cfabianski committed Nov 15, 2022
1 parent 21f31c0 commit 8be0e07
Show file tree
Hide file tree
Showing 27 changed files with 272 additions and 67 deletions.
17 changes: 13 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/go-enry/go-enry/v2 v2.8.2
github.com/google/uuid v1.3.0
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00
github.com/open-policy-agent/opa v0.46.1
github.com/rs/zerolog v1.28.0
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/schollz/progressbar/v3 v3.11.0
Expand All @@ -25,12 +26,15 @@ require (
)

require (
github.com/OneOfOne/xxhash v1.2.8 // indirect
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/charmbracelet/lipgloss v0.5.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fatih/semgroup v1.2.0 // indirect
github.com/gitleaks/go-gitdiff v0.8.0 // indirect
github.com/go-enry/go-oniguruma v1.2.1 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/h2non/filetype v1.1.3 // indirect
github.com/hhatto/gocloc v0.4.3 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
Expand All @@ -43,30 +47,35 @@ require (
github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 // indirect
github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0 // indirect
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
github.com/rivo/uniseg v0.4.2 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/tchap/go-patricia/v2 v2.3.1 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/yashtewari/glob-intersection v0.1.0 // indirect
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
golang.org/x/term v0.1.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)

require (
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml v1.9.5
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/cobra v1.6.0
github.com/spf13/cobra v1.6.1
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.13.0
github.com/subosito/gotenv v1.4.1 // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/text v0.4.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0
Expand Down
64 changes: 55 additions & 9 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion integration/flags/.snapshots/TestInitCommand-init
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ scan:
- ruby
patterns:
- |
log.info(<$ARGUMENT>)
logger.info(<$ARGUMENT>)
param_parenting: false
metavars: {}
stored: false
Expand Down
2 changes: 1 addition & 1 deletion integration/flags/.snapshots/TestMetadataFlags-help-scan
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ Worker Flags
Report Flags
-f, --format string format (json, yaml) (default "json")
--output string path where to save report
--report string specify the kind of report (detectors, dataflow, stats) (default "detectors")
--report string specify the kind of report (detectors, dataflow, policies, stats) (default "detectors")

General Flags
--config-file string file from which to load configurations
Expand Down
2 changes: 1 addition & 1 deletion integration/flags/.snapshots/TestMetadataFlags-scan-help
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ Worker Flags
Report Flags
-f, --format string format (json, yaml) (default "json")
--output string path where to save report
--report string specify the kind of report (detectors, dataflow, stats) (default "detectors")
--report string specify the kind of report (detectors, dataflow, policies, stats) (default "detectors")

General Flags
--config-file string file from which to load configurations
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
[{"detector_type":"detect_ruby_logger","source":{"column_number":1,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"data_category_name":"Unique Identifier","default_category":"Identification","id":86,"uuid":"12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9"},"decision":{"reason":"valid_object_with_invalid_properties","state":"invalid"},"name":"user"},"field_name":"user","field_type":"","field_type_simple":"unknown","object_name":""}},{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":5,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"log"}},{"detector_type":"ruby","source":{"column_number":26,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":36,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]
[{"detector_type":"detect_ruby_logger","source":{"column_number":1,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"data_category_name":"Unique Identifier","default_category":"Identification","id":86,"uuid":"12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9"},"decision":{"reason":"valid_object_with_invalid_properties","state":"invalid"},"name":"user"},"field_name":"user","field_type":"","field_type_simple":"unknown","object_name":""}},{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":29,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":39,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]

--

2 changes: 1 addition & 1 deletion integration/flags/.snapshots/TestReportFlags-format-json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
[{"detector_type":"detect_ruby_logger","source":{"column_number":1,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"data_category_name":"Unique Identifier","default_category":"Identification","id":86,"uuid":"12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9"},"decision":{"reason":"valid_object_with_invalid_properties","state":"invalid"},"name":"user"},"field_name":"user","field_type":"","field_type_simple":"unknown","object_name":""}},{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":5,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"log"}},{"detector_type":"ruby","source":{"column_number":26,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":36,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]
[{"detector_type":"detect_ruby_logger","source":{"column_number":1,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"data_category_name":"Unique Identifier","default_category":"Identification","id":86,"uuid":"12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9"},"decision":{"reason":"valid_object_with_invalid_properties","state":"invalid"},"name":"user"},"field_name":"user","field_type":"","field_type_simple":"unknown","object_name":""}},{"detector_type":"detect_ruby_logger","source":{"column_number":6,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"detect_ruby_logger","source":{"column_number":16,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":29,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":39,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}},{"detector_type":"ruby","source":{"column_number":14,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":24,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"invalid_property","state":"invalid"},"name":"something"},"field_name":"something","field_type":"","field_type_simple":"unknown","object_name":"user"}},{"detector_type":"ruby","source":{"column_number":34,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":3,"text":null},"type":"schema_classified","value":{"classification":null,"field_name":"something3","field_type":"","field_type_simple":"unknown","object_name":"something"}}]

--

8 changes: 4 additions & 4 deletions integration/flags/.snapshots/TestReportFlags-format-yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
object_name: something
- detector_type: ruby
source:
column_number: 5
column_number: 8
filename: main.rb
language: Ruby
language_type: programming
Expand All @@ -70,10 +70,10 @@
field_name: info
field_type: ""
field_type_simple: unknown
object_name: log
object_name: logger
- detector_type: ruby
source:
column_number: 26
column_number: 29
filename: main.rb
language: Ruby
language_type: programming
Expand All @@ -92,7 +92,7 @@
object_name: user
- detector_type: ruby
source:
column_number: 36
column_number: 39
filename: main.rb
language: Ruby
language_type: programming
Expand Down
Loading

0 comments on commit 8be0e07

Please sign in to comment.