From e3e82a0337470696933ae0331b756dca1f360477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Fabianski?= Date: Wed, 13 Mar 2024 10:56:52 +0100 Subject: [PATCH 1/4] chore: rename main.go to bearer.go --- .github/workflows/e2e_test.yml | 4 ++-- .github/workflows/version_comparison.yml | 15 +++++++-------- .gitignore | 3 --- cmd/bearer/{main.go => bearer.go} | 0 docs/contributing/code.md | 2 +- e2e/internal/testhelper/testhelper.go | 2 +- scripts/run_tests.sh | 2 +- 7 files changed, 12 insertions(+), 16 deletions(-) rename cmd/bearer/{main.go => bearer.go} (100%) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 2ad026292..4ec83714a 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -9,7 +9,7 @@ on: - synchronize - reopened paths-ignore: - - 'docs/**' + - "docs/**" permissions: contents: read jobs: @@ -23,7 +23,7 @@ jobs: with: go-version: 1.21 - name: Build binary for integration tests - run: go build -a -o ./bearer ./cmd/bearer/main.go + run: go build -a -o ./bearer ./cmd/bearer/bearer.go - name: Run tests run: go test -v ./e2e/... -p 8 timeout-minutes: 10 diff --git a/.github/workflows/version_comparison.yml b/.github/workflows/version_comparison.yml index c33f62536..75304f02f 100644 --- a/.github/workflows/version_comparison.yml +++ b/.github/workflows/version_comparison.yml @@ -3,13 +3,13 @@ on: workflow_dispatch: inputs: baseRef: - description: 'Base CLI ref (tag/branch/SHA)' + description: "Base CLI ref (tag/branch/SHA)" baseRulesRef: - description: 'Base rules ref' + description: "Base rules ref" testRef: - description: 'Test CLI ref (tag/branch/SHA)' + description: "Test CLI ref (tag/branch/SHA)" testRulesRef: - description: 'Test rules ref' + description: "Test rules ref" jobs: setup: @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v4 - id: load_repo_list name: Load KPI repository list - run : | + run: | echo "matrix=$(npx --yes json5 ./kpi_scan/kpi_repo_list.json5)" >> $GITHUB_OUTPUT - name: Set up Go uses: actions/setup-go@v5 @@ -49,7 +49,7 @@ jobs: - name: Build base CLI run: | cd ./base-cli - go build -o ../bearer-comparison/base-bearer ./cmd/bearer/main.go + go build -o ../bearer-comparison/base-bearer ./cmd/bearer/bearer.go - name: Checkout test CLI uses: actions/checkout@v4 with: @@ -65,7 +65,7 @@ jobs: - name: Build test CLI run: | cd ./test-cli - go build -o ../bearer-comparison/test-bearer ./cmd/bearer/main.go + go build -o ../bearer-comparison/test-bearer ./cmd/bearer/bearer.go - name: Cache CLIs and rules uses: actions/cache/save@v4 with: @@ -113,4 +113,3 @@ jobs: | jq > test.json - run: | diff -u base.json test.json - diff --git a/.gitignore b/.gitignore index 7fb3c4add..2e89dc9fd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,6 @@ bearer-build .DS_Store -# binaries -bearer - temp/ diff --git a/cmd/bearer/main.go b/cmd/bearer/bearer.go similarity index 100% rename from cmd/bearer/main.go rename to cmd/bearer/bearer.go diff --git a/docs/contributing/code.md b/docs/contributing/code.md index 1ffa98a24..e6f7e8a29 100644 --- a/docs/contributing/code.md +++ b/docs/contributing/code.md @@ -40,7 +40,7 @@ Finally, we use [direnv](https://direnv.net/) to manage env vars in development. To run Bearer CLI from source use the following command from the `bearer` directory: ```bash -go run ./cmd/bearer/main.go [COMMAND] +go run ./cmd/bearer/bearer.go [COMMAND] ``` Use commands and flags as normal in place of `[COMMAND]`. diff --git a/e2e/internal/testhelper/testhelper.go b/e2e/internal/testhelper/testhelper.go index 4a718172b..f26bbd5f0 100644 --- a/e2e/internal/testhelper/testhelper.go +++ b/e2e/internal/testhelper/testhelper.go @@ -96,7 +96,7 @@ func CreateCommand(arguments []string) (*exec.Cmd, context.CancelFunc) { if os.Getenv("USE_BINARY") != "" { cmd = exec.CommandContext(ctx, executablePath(), arguments...) } else { - arguments = append([]string{"run", GetCWD() + "/cmd/bearer/main.go"}, arguments...) + arguments = append([]string{"run", GetCWD() + "/cmd/bearer/bearer.go"}, arguments...) cmd = exec.CommandContext(ctx, "go", arguments...) } diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 03e33fee2..30c5c9114 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -24,7 +24,7 @@ do_cleanup() { trap do_cleanup 1 2 3 6 do_info "Building binary..." -go build -a -o ./bearer ./cmd/bearer/main.go || do_error "Failed to build binary" +go build -a -o ./bearer ./cmd/bearer/bearer.go || do_error "Failed to build binary" [ -f bearer ] || do_error "No binary found" From 7a4d16ddf4dc85ab34071c6b15f1eaa544613691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Fabianski?= Date: Wed, 13 Mar 2024 11:15:44 +0100 Subject: [PATCH 2/4] fix: do not write unclassified to the report --- e2e/flags/.snapshots/TestOuputFlag | 2 +- internal/report/schema/datatype/datatype.go | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/e2e/flags/.snapshots/TestOuputFlag b/e2e/flags/.snapshots/TestOuputFlag index b198e435e..068719d7d 100644 --- a/e2e/flags/.snapshots/TestOuputFlag +++ b/e2e/flags/.snapshots/TestOuputFlag @@ -1,2 +1,2 @@ -[{"detector_type":"ruby","source":{"end_column_number":12,"end_line_number":1,"filename":"main.rb","full_filename":"","language":"Ruby","language_type":"programming","start_column_number":8,"start_line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"decision":{"reason":"","state":""},"name":"info"},"field_name":"info","field_type":"","field_type_simple":"unknown","normalized_field_name":"info","normalized_object_name":"logger","object_name":"logger"}},{"detector_type":"ruby","source":{"end_column_number":36,"end_line_number":1,"filename":"main.rb","full_filename":"","language":"Ruby","language_type":"programming","start_column_number":31,"start_line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category":{"groups":{"172d90e3-cb9a-46b6-90e5-dd7169c3af54":{"name":"PII","uuid":"172d90e3-cb9a-46b6-90e5-dd7169c3af54"},"e1d3135b-3c0f-4b55-abce-19f27a26cbb3":{"name":"Personal Data","uuid":"e1d3135b-3c0f-4b55-abce-19f27a26cbb3"}},"name":"Contact","uuid":"cef587dd-76db-430b-9e18-7b031e1a193b"},"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email","subject_name":"User"},"field_name":"email","field_type":"","field_type_simple":"unknown","normalized_field_name":"email","normalized_object_name":"user","object_name":"user"}},{"filenames":["main.rb"],"type":"file_list"}] +[{"detector_type":"ruby","source":{"end_column_number":36,"end_line_number":1,"filename":"main.rb","full_filename":"","language":"Ruby","language_type":"programming","start_column_number":31,"start_line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category":{"groups":{"172d90e3-cb9a-46b6-90e5-dd7169c3af54":{"name":"PII","uuid":"172d90e3-cb9a-46b6-90e5-dd7169c3af54"},"e1d3135b-3c0f-4b55-abce-19f27a26cbb3":{"name":"Personal Data","uuid":"e1d3135b-3c0f-4b55-abce-19f27a26cbb3"}},"name":"Contact","uuid":"cef587dd-76db-430b-9e18-7b031e1a193b"},"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email","subject_name":"User"},"field_name":"email","field_type":"","field_type_simple":"unknown","normalized_field_name":"email","normalized_object_name":"user","object_name":"user"}},{"filenames":["main.rb"],"type":"file_list"}] diff --git a/internal/report/schema/datatype/datatype.go b/internal/report/schema/datatype/datatype.go index 859fc0d73..80bf51f18 100644 --- a/internal/report/schema/datatype/datatype.go +++ b/internal/report/schema/datatype/datatype.go @@ -10,6 +10,7 @@ import ( "github.com/bearer/bearer/internal/report/detections" "github.com/bearer/bearer/internal/report/detectors" "github.com/bearer/bearer/internal/report/schema" + "github.com/bearer/bearer/internal/util/classify" "github.com/bearer/bearer/internal/util/normalize_key" "github.com/bearer/bearer/internal/util/pluralize" ) @@ -195,6 +196,10 @@ func dataTypeToSchema[D DataTypable](report detections.ReportDetection, detectio return } + if classification, ok := dataType.GetClassification().(classificationschema.Classification); ok && classification.Decision.State != classify.Valid { + return + } + selfUUID := dataType.GetUUID() if selfUUID == "" { selfUUID = idGenerator.GenerateId() @@ -213,10 +218,9 @@ func dataTypeToSchema[D DataTypable](report detections.ReportDetection, detectio EndColumnNumber: parent.EndColumnNumber(), } } - normalizedObjectName := "" - normalizedFieldName := "" - normalizedObjectName = pluralize.Singular(strings.ToLower(parentName)) - normalizedFieldName = pluralize.Singular(strings.ToLower(selfName)) + + normalizedObjectName := pluralize.Singular(strings.ToLower(parentName)) + normalizedFieldName := pluralize.Singular(strings.ToLower(selfName)) report.AddDetection( detectionType, From 3cdde9233cff6ce09bae70e0b980837637664ab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Fabianski?= Date: Wed, 13 Mar 2024 15:22:58 +0100 Subject: [PATCH 3/4] chore: refactor datatype --- internal/report/schema/datatype/datatype.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/report/schema/datatype/datatype.go b/internal/report/schema/datatype/datatype.go index 80bf51f18..3f39232ba 100644 --- a/internal/report/schema/datatype/datatype.go +++ b/internal/report/schema/datatype/datatype.go @@ -196,7 +196,7 @@ func dataTypeToSchema[D DataTypable](report detections.ReportDetection, detectio return } - if classification, ok := dataType.GetClassification().(classificationschema.Classification); ok && classification.Decision.State != classify.Valid { + if classification := dataType.GetClassification().(classificationschema.Classification); classification.Decision.State != classify.Valid { return } From 9cda95cddd1648d8437910df32f4ebff9dc6840d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Fabianski?= Date: Wed, 13 Mar 2024 19:18:18 +0100 Subject: [PATCH 4/4] fixup: uncommitted changes remaining on CI --- .envrc.example | 2 +- .github/workflows/e2e_test.yml | 2 +- .gitignore | 1 + internal/report/schema/datatype/datatype.go | 2 +- scripts/run_tests.sh | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.envrc.example b/.envrc.example index 4ea4849c4..4ff592814 100644 --- a/.envrc.example +++ b/.envrc.example @@ -10,5 +10,5 @@ export SCAN_DIR=/Users/username/OWASP export BEARER_DISABLE_VERSION_CHECK=true export BEARER_DISABLE_DEFAULT_RULES=true export BEARER_EXTERNAL_RULE_DIR=$PWD/../bearer-rules/rules -export BEARER_FORCE=true +# export BEARER_FORCE=true # export BEARER_IGNORE_GIT=true diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 4ec83714a..171bd4359 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -23,7 +23,7 @@ jobs: with: go-version: 1.21 - name: Build binary for integration tests - run: go build -a -o ./bearer ./cmd/bearer/bearer.go + run: go build -a ./cmd/bearer/bearer.go - name: Run tests run: go test -v ./e2e/... -p 8 timeout-minutes: 10 diff --git a/.gitignore b/.gitignore index 2e89dc9fd..e4068bdf4 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ bearer-build temp/ +bearer .envrc /*.pprof diff --git a/internal/report/schema/datatype/datatype.go b/internal/report/schema/datatype/datatype.go index 3f39232ba..80bf51f18 100644 --- a/internal/report/schema/datatype/datatype.go +++ b/internal/report/schema/datatype/datatype.go @@ -196,7 +196,7 @@ func dataTypeToSchema[D DataTypable](report detections.ReportDetection, detectio return } - if classification := dataType.GetClassification().(classificationschema.Classification); classification.Decision.State != classify.Valid { + if classification, ok := dataType.GetClassification().(classificationschema.Classification); ok && classification.Decision.State != classify.Valid { return } diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 30c5c9114..302dd6848 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -24,7 +24,7 @@ do_cleanup() { trap do_cleanup 1 2 3 6 do_info "Building binary..." -go build -a -o ./bearer ./cmd/bearer/bearer.go || do_error "Failed to build binary" +go build -a ./cmd/bearer/bearer.go || do_error "Failed to build binary" [ -f bearer ] || do_error "No binary found"