Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Preserve original path for archives; add to output #180

Closed
wants to merge 11 commits into from
Closed

Preserve original path for archives; add to output #180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
914d5f3
Preserve original path for archives; add to output
May 7, 2024
dcfdfd8
Display scanned file next to original path
May 3, 2024
66637ea
Add original path of moved file to diff output
May 3, 2024
7e16599
Add from/to original paths to diff output
May 3, 2024
bf2e861
Fix tests
May 7, 2024
8cb6eff
Formatting fix
May 3, 2024
b8eb7eb
Address PR comments
May 5, 2024
ad7f455
Improve rules based on LightSpy + add Huntress to third_party (#169)
tstromberg May 7, 2024
3e5926c
Make --ignore-self more precise (#194)
May 7, 2024
c7ca09d
Fix errant change
May 7, 2024
cd28a28
More rebase fixes
May 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 6 additions & 3 deletions pkg/action/diff.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ func Diff(ctx context.Context, c Config) (*bincapz.Report, error) {

rbs := bincapz.FileReport{
Path: tr.Path,
OriginalAbsPath: tr.OriginalAbsPath,
Behaviors: map[string]bincapz.Behavior{},
PreviousRiskScore: fr.RiskScore,
PreviousRiskLevel: fr.RiskLevel,
Expand Down Expand Up @@ -109,12 +110,12 @@ func Diff(ctx context.Context, c Config) (*bincapz.Report, error) {

abs := bincapz.FileReport{
Path: tr.Path,
OriginalAbsPath: tr.OriginalAbsPath,
Behaviors: map[string]bincapz.Behavior{},
PreviousRiskScore: fr.RiskScore,
PreviousRiskLevel: fr.RiskLevel,

RiskScore: tr.RiskScore,
RiskLevel: tr.RiskLevel,
RiskScore: tr.RiskScore,
RiskLevel: tr.RiskLevel,
}

// if destination behavior is not in the source
Expand Down Expand Up @@ -153,7 +154,9 @@ func Diff(ctx context.Context, c Config) (*bincapz.Report, error) {
// We think that this file moved from rpath to apath.
abs := bincapz.FileReport{
Path: tr.Path,
OriginalAbsPath: tr.OriginalAbsPath,
PreviousRelPath: rpath,
PreviousAbsPath: fr.OriginalAbsPath,
PreviousRelPathScore: score,

Behaviors: map[string]bincapz.Behavior{},
Expand Down
35 changes: 23 additions & 12 deletions pkg/action/scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,13 @@ func findFilesRecursively(ctx context.Context, root string, c Config) ([]string,
return files, err
}

func scanSinglePath(ctx context.Context, c Config, yrs *yara.Rules, path string) (*bincapz.FileReport, error) {
func scanSinglePath(
ctx context.Context,
c Config,
yrs *yara.Rules,
path string,
aPath string,
) (*bincapz.FileReport, error) {
logger := clog.FromContext(ctx)
var mrs yara.MatchRules
logger = logger.With("path", path)
Expand All @@ -70,10 +76,10 @@ func scanSinglePath(ctx context.Context, c Config, yrs *yara.Rules, path string)

if err := yrs.ScanFile(path, 0, 0, &mrs); err != nil {
logger.Info("skipping", slog.Any("error", err))
return &bincapz.FileReport{Path: path, Error: fmt.Sprintf("scanfile: %v", err)}, nil
return &bincapz.FileReport{Path: path, OriginalAbsPath: aPath, Error: fmt.Sprintf("scanfile: %v", err)}, nil
}

fr, err := report.Generate(ctx, path, mrs, c.IgnoreTags, c.MinResultScore)
fr, err := report.Generate(ctx, path, aPath, mrs, c.IgnoreTags, c.MinResultScore)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -128,7 +134,7 @@ func recursiveScan(ctx context.Context, c Config) (*bincapz.Report, error) {
return nil, err
}
} else {
err = processFile(ctx, c, yrs, r, p, logger)
err = processFile(ctx, c, yrs, r, p, "", logger)
if err != nil {
return nil, err
}
Expand All @@ -153,35 +159,40 @@ func processArchive(
logger *clog.Logger,
) error {
var err error
p, err = archive(ctx, p)
var ap string
ap, err = archive(ctx, p)
if err != nil {
return fmt.Errorf("failed to prepare archive for scanning: %w", err)
}
var ap []string
ap, err = findFilesRecursively(ctx, p, c)
var af []string
af, err = findFilesRecursively(ctx, ap, c)
if err != nil {
return fmt.Errorf("find files: %w", err)
}
for _, a := range ap {
err = processFile(ctx, c, yrs, r, a, logger)
for _, a := range af {
// a is the scan path (within the temp directory)
// p is the original path to the archive file
err = processFile(ctx, c, yrs, r, a, p, logger)
if err != nil {
return err
}
}
if err := os.RemoveAll(p); err != nil {
if err := os.RemoveAll(ap); err != nil {
logger.Errorf("remove %s: %v", p, err)
}
return nil
}

func processFile(
ctx context.Context,
c Config, yrs *yara.Rules,
c Config,
yrs *yara.Rules,
r *bincapz.Report,
p string,
a string,
logger *clog.Logger,
) error {
fr, err := scanSinglePath(ctx, c, yrs, p)
fr, err := scanSinglePath(ctx, c, yrs, p, a)
if err != nil {
logger.Errorf("scan path: %v", err)
return nil
Expand Down
5 changes: 5 additions & 0 deletions pkg/bincapz/bincapz.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type FileReport struct {

// The relative path we think this moved from.
PreviousRelPath string `json:",omitempty" yaml:",omitempty"`
PreviousAbsPath string `json:",omitempty" yaml:",omitempty"`
// The levenshtein distance between the previous path and the current path
PreviousRelPathScore float64 `json:",omitempty" yaml:",omitempty"`
PreviousRiskScore int `json:",omitempty" yaml:",omitempty"`
Expand All @@ -48,6 +49,10 @@ type FileReport struct {
PackageRisk []string `json:",omitempty" yaml:",omitempty"`

IsBincapz bool `json:",omitempty" yaml:",omitempty"`

// The original path for scanned archive files
// When not scanning archives, this will be empty
OriginalAbsPath string
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add json:",omitempty" yaml:",omitempty" so that the empty field doesn't show up in JSON output.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm unclear on the relationship between Path and OriginalAbsPath. If Path is the original path requested by the user, is this just the absolute version of it?

On a related note - we should make sure that we're not storing any temporary file paths within the struct, as they aren't useful to the reader, as that file location no longer exists when they read the report.

We should however store the relative location within an archive somewhere so that they can investigate further.

}

type DiffReport struct {
Expand Down
36 changes: 33 additions & 3 deletions pkg/render/markdown.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,28 +42,51 @@ func matchFragmentLink(s string) string {
}

func (r Markdown) File(ctx context.Context, fr bincapz.FileReport) error {
markdownTable(ctx, &fr, r.w, tableConfig{Title: fmt.Sprintf("## %s [%s]", fr.Path, mdRisk(fr.RiskScore, fr.RiskLevel))})
tableCfg := tableConfig{Title: fmt.Sprintf("## %s [%s]", fr.Path, mdRisk(fr.RiskScore, fr.RiskLevel))}
if fr.OriginalAbsPath != "" {
fileName := fr.Path[strings.LastIndex(fr.Path, "/")+1:]
tableCfg.SubTitle = fmt.Sprintf("### Original Path (From): %s > %s\n### Original Path (To): %s > %s", fr.PreviousAbsPath, fileName, fr.OriginalAbsPath, fileName)
}
markdownTable(ctx, &fr, r.w, tableCfg)
return nil
}

func (r Markdown) Full(ctx context.Context, rep bincapz.Report) error {
for f, fr := range rep.Diff.Removed {
fr := fr
markdownTable(ctx, &fr, r.w, tableConfig{Title: fmt.Sprintf("## Deleted: %s [%s]", f, mdRisk(fr.RiskScore, fr.RiskLevel)), DiffRemoved: true})
fileName := fr.Path[strings.LastIndex(fr.Path, "/")+1:]
tableCfg := tableConfig{Title: fmt.Sprintf("## Deleted: %s [%s]", f, mdRisk(fr.RiskScore, fr.RiskLevel)), DiffRemoved: true}
if fr.OriginalAbsPath != "" {
tableCfg.SubTitle = fmt.Sprintf("### Original Path (From): %s > %s\n### Original Path (To): %s > %s", fr.PreviousAbsPath, fileName, fr.OriginalAbsPath, fileName)
}
markdownTable(ctx, &fr, r.w, tableCfg)
}

for f, fr := range rep.Diff.Added {
fr := fr
markdownTable(ctx, &fr, r.w, tableConfig{Title: fmt.Sprintf("## Added: %s [%s]", f, mdRisk(fr.RiskScore, fr.RiskLevel)), DiffAdded: true})
fileName := fr.Path[strings.LastIndex(fr.Path, "/")+1:]
tableCfg := tableConfig{Title: fmt.Sprintf("## Added: %s [%s]", f, mdRisk(fr.RiskScore, fr.RiskLevel)), DiffAdded: true}
if fr.OriginalAbsPath != "" {
tableCfg.SubTitle = fmt.Sprintf("### Original Path (From): %s > %s\n### Original Path (To): %s > %s", fr.PreviousAbsPath, fileName, fr.OriginalAbsPath, fileName)
}
markdownTable(ctx, &fr, r.w, tableCfg)
}

for f, fr := range rep.Diff.Modified {
fr := fr
var title string
var subtitle string
fileName := fr.Path[strings.LastIndex(fr.Path, "/")+1:]
if fr.PreviousRelPath != "" {
title = fmt.Sprintf("## Moved: %s -> %s (similarity: %0.2f)", fr.PreviousRelPath, f, fr.PreviousRelPathScore)
if fr.OriginalAbsPath != "" {
subtitle = fmt.Sprintf("### Original Path (From): %s > %s\n### Original Path (To): %s > %s", fr.PreviousAbsPath, fileName, fr.OriginalAbsPath, fileName)
}
} else {
title = fmt.Sprintf("## Changed: %s", f)
if fr.OriginalAbsPath != "" {
subtitle = fmt.Sprintf("### Original Path (From): %s > %s\n### Original Path (To): %s > %s", fr.PreviousAbsPath, fileName, fr.OriginalAbsPath, fileName)
}
}
if fr.RiskScore != fr.PreviousRiskScore {
title = fmt.Sprintf("%s [%s → %s]",
Expand All @@ -73,6 +96,9 @@ func (r Markdown) Full(ctx context.Context, rep bincapz.Report) error {
}

fmt.Fprint(r.w, title+"\n\n")
if subtitle != "" {
fmt.Fprintf(r.w, subtitle+"\n\n")
}
added := 0
removed := 0
for _, b := range fr.Behaviors {
Expand Down Expand Up @@ -129,6 +155,10 @@ func markdownTable(_ context.Context, fr *bincapz.FileReport, w io.Writer, rc ta
fmt.Fprintf(w, "%s\n\n", rc.Title)
}

if fr.OriginalAbsPath != "" {
fmt.Fprintf(w, "%s\n\n", rc.SubTitle)
}

sort.Slice(kbs, func(i, j int) bool {
if kbs[i].Behavior.RiskScore == kbs[j].Behavior.RiskScore {
return kbs[i].Key < kbs[j].Key
Expand Down
3 changes: 3 additions & 0 deletions pkg/render/simple.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ func NewSimple(w io.Writer) Simple {

func (r Simple) File(_ context.Context, fr bincapz.FileReport) error {
fmt.Fprintf(r.w, "# %s\n", fr.Path)
if fr.OriginalAbsPath != "" {
fmt.Fprintf(r.w, "## Original Path: %s\n", fr.OriginalAbsPath)
}
bs := []string{}

for k := range fr.Behaviors {
Expand Down
61 changes: 47 additions & 14 deletions pkg/render/terminal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ type KeyedBehavior struct {

type tableConfig struct {
Title string
SubTitle string
ShowTitle bool
DiffRemoved bool
DiffAdded bool
Expand Down Expand Up @@ -92,46 +93,74 @@ func ShortRisk(s string) string {
}

func (r Terminal) File(ctx context.Context, fr bincapz.FileReport) error {
renderTable(ctx, &fr, r.w,
tableConfig{
Title: fmt.Sprintf("%s %s", fr.Path, darkBrackets(decorativeRisk(fr.RiskScore, fr.RiskLevel))),
},
)
fileName := fr.Path[strings.LastIndex(fr.Path, "/")+1:]
tableCfg := tableConfig{
Title: fmt.Sprintf("%s %s", fr.Path, darkBrackets(decorativeRisk(fr.RiskScore, fr.RiskLevel))),
}
if fr.OriginalAbsPath != "" {
tableCfg.SubTitle = fmt.Sprintf("Original Path: %s > %s", fr.OriginalAbsPath, fileName)
}

renderTable(ctx, &fr, r.w, tableCfg)
return nil
}

func (r Terminal) Full(ctx context.Context, rep bincapz.Report) error {
for f, fr := range rep.Diff.Removed {
fr := fr
renderTable(ctx, &fr, r.w, tableConfig{
fileName := fr.Path[strings.LastIndex(fr.Path, "/")+1:]
tableCfg := tableConfig{
Title: fmt.Sprintf("Deleted: %s %s", f, darkBrackets(decorativeRisk(fr.RiskScore, fr.RiskLevel))),
DiffRemoved: true,
})
}
if fr.OriginalAbsPath != "" {
tableCfg.SubTitle = fmt.Sprintf("Original Path: %s > %s", fr.OriginalAbsPath, fileName)
}

renderTable(ctx, &fr, r.w, tableCfg)
}

for f, fr := range rep.Diff.Added {
fr := fr
renderTable(ctx, &fr, r.w, tableConfig{
fileName := fr.Path[strings.LastIndex(fr.Path, "/")+1:]
tableCfg := tableConfig{
Title: fmt.Sprintf("Added: %s %s", f, darkBrackets(decorativeRisk(fr.RiskScore, fr.RiskLevel))),
DiffAdded: true,
})
}
if fr.OriginalAbsPath != "" {
tableCfg.SubTitle = fmt.Sprintf("Original Path: %s > %s", fr.OriginalAbsPath, fileName)
}

renderTable(ctx, &fr, r.w, tableCfg)
}

for f, fr := range rep.Diff.Modified {
fr := fr
fileName := fr.Path[strings.LastIndex(fr.Path, "/")+1:]
var title string
var subtitle string
if fr.PreviousRelPath != "" {
title = fmt.Sprintf("Moved: %s -> %s (score: %f)", fr.PreviousRelPath, f, fr.PreviousRelPathScore)
title = fmt.Sprintf("Moved: %s -> %s (score: %f)\n", fr.PreviousRelPath, f, fr.PreviousRelPathScore)
if fr.OriginalAbsPath != "" {
subtitle = fmt.Sprintf("Original Path (From): %s > %s\nOriginal Path (To): %s > %s\n", fr.PreviousAbsPath, fileName, fr.OriginalAbsPath, fileName)
}
} else {
title = fmt.Sprintf("Changed: %s", f)
title = fmt.Sprintf("Changed: %s\n", f)
subtitle = fmt.Sprintf("Original Path: %s > %s\n", fr.OriginalAbsPath, fileName)
}

if fr.RiskScore != fr.PreviousRiskScore {
title = fmt.Sprintf("%s %s\n\n", title,
darkBrackets(fmt.Sprintf("%s %s %s", decorativeRisk(fr.PreviousRiskScore, fr.PreviousRiskLevel), color.HiWhiteString("→"), decorativeRisk(fr.RiskScore, fr.RiskLevel))))
darkBrackets(fmt.Sprintf("%s %s %s\n", decorativeRisk(fr.PreviousRiskScore, fr.PreviousRiskLevel), color.HiWhiteString("→"), decorativeRisk(fr.RiskScore, fr.RiskLevel))))
if fr.OriginalAbsPath != "" {
subtitle = fmt.Sprintf("Original Path (From): %s > %s\nOriginal Path (To): %s > %s\n", fr.PreviousAbsPath, fileName, fr.OriginalAbsPath, fileName)
}
}

fmt.Fprint(r.w, title)
fmt.Fprintf(r.w, "%s\n", title)
if subtitle != "" {
fmt.Fprintf(r.w, "%s\n", subtitle)
}
added := 0
removed := 0
for _, b := range fr.Behaviors {
Expand Down Expand Up @@ -182,6 +211,7 @@ func darkenText(s string) string {

func renderTable(ctx context.Context, fr *bincapz.FileReport, w io.Writer, rc tableConfig) {
title := rc.Title
subtitle := rc.SubTitle

path := fr.Path
if fr.Error != "" {
Expand Down Expand Up @@ -295,7 +325,10 @@ func renderTable(ctx context.Context, fr *bincapz.FileReport, w io.Writer, rc ta
}

if title != "" {
fmt.Fprintf(w, "%s", title)
fmt.Fprintf(w, "%s\n", title)
}
if subtitle != "" {
fmt.Fprintf(w, "%s\n", subtitle)
}
fmt.Fprintf(w, "\n")

Expand Down
18 changes: 13 additions & 5 deletions pkg/report/report.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,14 @@ func fixURL(s string) string {
return strings.ReplaceAll(s, " ", "%20")
}

func Generate(ctx context.Context, path string, mrs yara.MatchRules, ignoreTags []string, minScore int) (bincapz.FileReport, error) {
func Generate(
ctx context.Context,
path string,
aPath string,
mrs yara.MatchRules,
ignoreTags []string,
minScore int,
) (bincapz.FileReport, error) {
ignore := map[string]bool{}
for _, t := range ignoreTags {
ignore[t] = true
Expand All @@ -283,10 +290,11 @@ func Generate(ctx context.Context, path string, mrs yara.MatchRules, ignoreTags
}

fr := bincapz.FileReport{
Path: path,
SHA256: ptCheck,
Meta: map[string]string{},
Behaviors: map[string]bincapz.Behavior{},
Path: path,
OriginalAbsPath: aPath,
SHA256: ptCheck,
Meta: map[string]string{},
Behaviors: map[string]bincapz.Behavior{},
}

pledges := []string{}
Expand Down