Skip to content

Commit

Permalink
feat: add status output (#110)
Browse files Browse the repository at this point in the history
feat: add stats output
  • Loading branch information
cfabianski committed Nov 14, 2022
1 parent 7ba6125 commit e4c1a31
Show file tree
Hide file tree
Showing 11 changed files with 196 additions and 37 deletions.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ require (
github.com/gitleaks/go-gitdiff v0.8.0 // indirect
github.com/go-enry/go-oniguruma v1.2.1 // indirect
github.com/h2non/filetype v1.1.3 // indirect
github.com/hhatto/gocloc v0.4.3 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/jessevdk/go-flags v1.5.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
Expand Down
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gitleaks/go-gitdiff v0.8.0 h1:7aExTZm+K/M/EQKOyYcub8rIAdWK6ONxPGuRzxmWW+0=
github.com/gitleaks/go-gitdiff v0.8.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
github.com/go-enry/go-enry/v2 v2.8.0/go.mod h1:GVzIiAytiS5uT/QiuakK7TF1u4xDab87Y8V5EJRpsIQ=
github.com/go-enry/go-enry/v2 v2.8.2 h1:uiGmC+3K8sVd/6DOe2AOJEOihJdqda83nPyJNtMR8RI=
github.com/go-enry/go-enry/v2 v2.8.2/go.mod h1:GVzIiAytiS5uT/QiuakK7TF1u4xDab87Y8V5EJRpsIQ=
github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo=
Expand Down Expand Up @@ -144,10 +145,15 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hhatto/gocloc v0.4.3 h1:bWbEi+cOKDAvWwPsP2lT30638Bg37X+Ru00TG7adpGg=
github.com/hhatto/gocloc v0.4.3/go.mod h1:EPoonh5stxIeraUU70Ogyj9yIpvV6Xirnjhyx+3/cHM=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
Expand Down Expand Up @@ -210,6 +216,7 @@ github.com/schollz/progressbar/v3 v3.11.0 h1:3nIBUF1Zw/pGUaRHP7PZWmARP7ZQbWQ6vL6
github.com/schollz/progressbar/v3 v3.11.0/go.mod h1:R2djRgv58sn00AGysc4fN0ip4piOGd3z88K+zVBjczs=
github.com/smacker/go-tree-sitter v0.0.0-20220829074436-0a7a807924f2 h1:p+xxTsHssBdE21bzntBWAKjNyZ7BpuxynngfAe4hTHg=
github.com/smacker/go-tree-sitter v0.0.0-20220829074436-0a7a807924f2/go.mod h1:q99oHDsbP0xRwmn7Vmob8gbSMNyvJ83OauXPSuHQuKE=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo=
github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
Expand Down Expand Up @@ -390,6 +397,7 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down Expand Up @@ -557,6 +565,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
Expand Down
3 changes: 2 additions & 1 deletion pkg/flag/report_flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ var (

ReportDetectors = "detectors"
ReportDataFlow = "dataflow"
ReportStats = "stats"
)

var (
Expand All @@ -22,7 +23,7 @@ var (
Name: "report",
ConfigName: "report.report",
Value: ReportDetectors,
Usage: "specify the kind of report (detectors, dataflow)",
Usage: "specify the kind of report (detectors, dataflow, stats)",
}
OutputFlag = Flag{
Name: "output",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package output_test
package components_test

import (
"os"
"testing"

"github.com/bearer/curio/pkg/commands/process/settings"
"github.com/bearer/curio/pkg/report/output"
"github.com/bearer/curio/pkg/report/output/dataflow"
"github.com/bearer/curio/pkg/report/output/dataflow/types"
"github.com/bearer/curio/pkg/report/output/detectors"
globaltypes "github.com/bearer/curio/pkg/types"
"github.com/stretchr/testify/assert"
)
Expand Down Expand Up @@ -116,15 +116,15 @@ func TestDataflowComponents(t *testing.T) {
}
file.Close()

detections, err := output.GetDetectorsOutput(globaltypes.Report{
detections, err := detectors.GetOutput(globaltypes.Report{
Path: file.Name(),
})
if err != nil {
t.Fatalf("failed to get detectors output %s", err)
return
}

dataflow, err := dataflow.GetOuput(detections, settings.Config{})
dataflow, err := dataflow.GetOutput(detections, settings.Config{})
if err != nil {
t.Fatalf("failed to get detectors output %s", err)
return
Expand Down
2 changes: 1 addition & 1 deletion pkg/report/output/dataflow/dataflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type DataFlow struct {

var allowedDetections []detections.DetectionType = []detections.DetectionType{detections.TypeSchemaClassified, detections.TypeCustomClassified, detections.TypeDependencyClassified, detections.TypeInterfaceClassified}

func GetOuput(input []interface{}, config settings.Config) (*DataFlow, error) {
func GetOutput(input []interface{}, config settings.Config) (*DataFlow, error) {
dataTypesHolder := datatypes.New()
risksHolder := risks.New(config)
componentsHolder := components.New()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package output_test
package datatypes_test

import (
"os"
"testing"

"github.com/bearer/curio/pkg/commands/process/settings"
"github.com/bearer/curio/pkg/report/output"
"github.com/bearer/curio/pkg/report/output/dataflow"
"github.com/bearer/curio/pkg/report/output/dataflow/types"
"github.com/bearer/curio/pkg/report/output/detectors"
globaltypes "github.com/bearer/curio/pkg/types"
"github.com/stretchr/testify/assert"
)
Expand Down Expand Up @@ -161,15 +161,15 @@ func TestDataflowDataType(t *testing.T) {
}
file.Close()

detections, err := output.GetDetectorsOutput(globaltypes.Report{
detections, err := detectors.GetOutput(globaltypes.Report{
Path: file.Name(),
})
if err != nil {
t.Fatalf("failed to get detectors output %s", err)
return
}

dataflow, err := dataflow.GetOuput(detections, test.Config)
dataflow, err := dataflow.GetOutput(detections, test.Config)
if err != nil {
t.Fatalf("failed to get detectors output %s", err)
return
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package output_test
package risks_test

import (
"os"
"testing"

"github.com/bearer/curio/pkg/commands/process/settings"
"github.com/bearer/curio/pkg/report/output"
"github.com/bearer/curio/pkg/report/output/dataflow"
"github.com/bearer/curio/pkg/report/output/dataflow/types"
"github.com/bearer/curio/pkg/report/output/detectors"
globaltypes "github.com/bearer/curio/pkg/types"
"github.com/stretchr/testify/assert"
)
Expand Down Expand Up @@ -153,15 +153,15 @@ func TestDataflowRisks(t *testing.T) {
}
file.Close()

detections, err := output.GetDetectorsOutput(globaltypes.Report{
detections, err := detectors.GetOutput(globaltypes.Report{
Path: file.Name(),
})
if err != nil {
t.Fatalf("failed to get detectors output %s", err)
return
}

dataflow, err := dataflow.GetOuput(detections, test.Config)
dataflow, err := dataflow.GetOutput(detections, test.Config)
if err != nil {
t.Fatalf("failed to get detectors output %s", err)
return
Expand Down
26 changes: 26 additions & 0 deletions pkg/report/output/detectors/detectors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package detectors

import (
"fmt"
"os"

"github.com/bearer/curio/pkg/types"
"github.com/rs/zerolog/log"
"github.com/wlredeye/jsonlines"
)

func GetOutput(report types.Report) ([]interface{}, error) {
var detections []interface{}
f, err := os.Open(report.Path)
if err != nil {
return nil, fmt.Errorf("failed to open report: %w", err)
}

err = jsonlines.Decode(f, &detections)
if err != nil {
return nil, fmt.Errorf("failed to decode report: %w", err)
}
log.Debug().Msgf("got %d detections", len(detections))

return detections, nil
}
49 changes: 26 additions & 23 deletions pkg/report/output/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,25 @@ package output
import (
"encoding/json"
"fmt"
"os"

"github.com/bearer/curio/pkg/commands/process/settings"
"github.com/bearer/curio/pkg/flag"
"github.com/bearer/curio/pkg/report/output/dataflow"
"github.com/bearer/curio/pkg/report/output/detectors"
"github.com/bearer/curio/pkg/report/output/stats"
"github.com/bearer/curio/pkg/types"
"gopkg.in/yaml.v3"

"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/wlredeye/jsonlines"
)

func ReportJSON(report types.Report, output *zerolog.Event, config settings.Config) error {
ouputDetections, err := getReportOutput(report, config)
outputDetections, err := getReportOutput(report, config)
if err != nil {
return err
}

jsonBytes, err := json.Marshal(&ouputDetections)
jsonBytes, err := json.Marshal(&outputDetections)
if err != nil {
return fmt.Errorf("failed to json marshal detections: %w", err)
}
Expand All @@ -49,41 +48,45 @@ func ReportYAML(report types.Report, output *zerolog.Event, config settings.Conf
}

func getReportOutput(report types.Report, config settings.Config) (any, error) {
var ouputDetections any
var output any
var err error

if config.Report.Report == flag.ReportDetectors {
ouputDetections, err = GetDetectorsOutput(report)
output, err = detectors.GetOutput(report)
if err != nil {
return nil, err
}
} else if config.Report.Report == flag.ReportDataFlow {
detections, err := GetDetectorsOutput(report)
detectorsOutput, err := detectors.GetOutput(report)
if err != nil {
return nil, err
}

ouputDetections, err = dataflow.GetOuput(detections, config)
output, err = dataflow.GetOutput(detectorsOutput, config)
if err != nil {
return nil, err
}
} else if config.Report.Report == flag.ReportStats {
lineOfCodeOutput, err := stats.GoclocDetectorOutput(config.Scan.Target)
if err != nil {
return nil, err
}
}

return ouputDetections, nil
}
detectorsOutput, err := detectors.GetOutput(report)
if err != nil {
return nil, err
}

func GetDetectorsOutput(report types.Report) ([]interface{}, error) {
var detections []interface{}
f, err := os.Open(report.Path)
if err != nil {
return nil, fmt.Errorf("failed to open report: %w", err)
}
dataflowOutput, err := dataflow.GetOutput(detectorsOutput, config)
if err != nil {
return nil, err
}

err = jsonlines.Decode(f, &detections)
if err != nil {
return nil, fmt.Errorf("failed to decode report: %w", err)
output, err = stats.GetOutput(lineOfCodeOutput, dataflowOutput, config)
if err != nil {
return nil, err
}
}
log.Debug().Msgf("got %d detections", len(detections))

return detections, nil
return output, nil
}
77 changes: 77 additions & 0 deletions pkg/report/output/stats/gocloc_detector.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package stats

import (
"regexp"
"strings"

"github.com/hhatto/gocloc"
"github.com/jessevdk/go-flags"
)

type CmdOptions struct {
Byfile bool `long:"by-file" description:"report results for every encountered source file"`
SortTag string `long:"sort" default:"code" description:"sort based on a certain column"`
OutputType string `long:"output-type" default:"default" description:"output type [values: default,cloc-xml,sloccount,json]"`
ExcludeExt string `long:"exclude-ext" description:"exclude file name extensions (separated commas)"`
IncludeLang string `long:"include-lang" description:"include language name (separated commas)"`
Match string `long:"match" description:"include file name (regex)"`
NotMatch string `long:"not-match" description:"exclude file name (regex)"`
MatchDir string `long:"match-d" description:"include dir name (regex)"`
NotMatchDir string `long:"not-match-d" description:"exclude dir name (regex)"`
Debug bool `long:"debug" description:"dump debug log for developer"`
SkipDuplicated bool `long:"skip-duplicated" description:"skip duplicated files"`
ShowLang bool `long:"show-lang" description:"print about all languages and extensions"`
}

func GoclocDetectorOutput(path string) (*gocloc.Result, error) {
var opts CmdOptions
clocOpts := gocloc.NewClocOptions()
args := []string{
"--output-type=json",
path,
}

paths, err := flags.ParseArgs(&opts, args)
if err != nil {
return nil, err
}

languages := gocloc.NewDefinedLanguages()

// setup option for exclude extensions
for _, ext := range strings.Split(opts.ExcludeExt, ",") {
e, ok := gocloc.Exts[ext]
if ok {
clocOpts.ExcludeExts[e] = struct{}{}
} else {
clocOpts.ExcludeExts[ext] = struct{}{}
}
}

// directory and file matching options
if opts.Match != "" {
clocOpts.ReMatch = regexp.MustCompile(opts.Match)
}
if opts.NotMatch != "" {
clocOpts.ReNotMatch = regexp.MustCompile(opts.NotMatch)
}
if opts.MatchDir != "" {
clocOpts.ReMatchDir = regexp.MustCompile(opts.MatchDir)
}
if opts.NotMatchDir != "" {
clocOpts.ReNotMatchDir = regexp.MustCompile(opts.NotMatchDir)
}

// setup option for include languages
for _, lang := range strings.Split(opts.IncludeLang, ",") {
if _, ok := languages.Langs[lang]; ok {
clocOpts.IncludeLangs[lang] = struct{}{}
}
}

clocOpts.Debug = opts.Debug
clocOpts.SkipDuplicated = opts.SkipDuplicated

processor := gocloc.NewProcessor(languages, clocOpts)
return processor.Analyze(paths)
}
Loading

0 comments on commit e4c1a31

Please sign in to comment.