Skip to content
Permalink
Browse files

added FileCmp analyzer to compare current vs saved file

  • Loading branch information...
Collin Mulliner
Collin Mulliner committed Oct 11, 2019
1 parent 8bd37e0 commit 996312e48c2fc9979cd7f191d891c2087912efb3
Showing with 519 additions and 8 deletions.
  1. +1 −1 Makefile
  2. +35 −2 Readme.md
  3. +7 −5 cmd/fwanalyzer/fwanalyzer.go
  4. +193 −0 pkg/analyzer/filecmp/filecmp.go
  5. +274 −0 pkg/analyzer/filecmp/filecmp_test.go
  6. +9 −0 scripts/diff.sh
@@ -16,7 +16,7 @@ build:
test: lint build
gunzip -c test/test.img.gz >test/test.img
gunzip -c test/ubifs.img.gz >test/ubifs.img
go test -count=3 -cover ./...
PATH=$(PATH):`pwd`/scripts go test -count=3 -cover ./...
PATH=./test:$(PATH):./scripts:./build ./test/test.py


@@ -65,7 +65,7 @@ Command line options
- `-cfgpath` : string, path to config file and included files (can be repated)
- `-in` : string, filesystem image file or path to directory
- `-out` : string, output report to file or stdout using '-'
- `-tree` : string, overwrite directory to read the filetree file from
- `-extra` : string, overwrite directory to read extra data from (e.g. filetree, filecmp)
- `-ee` : exit with error if offenders are present
- `-invertMatch` : invert regex matches (for testing)

@@ -294,6 +294,39 @@ Example Output:
}
```

### File Compare Check

The `FileCmp` (File Compare) check is a mechanism to compare a file from a previous
run with the file from the current run. The main idea behind this check is to provide
more insights into file changes, since it allows comparing two versions of a file rather than
comparing only a digest.

This works by saving the file as the `OldFilePath`
(if it does not exist) and skipping the check at the first run. In consecutive runs
the current file and the saved old file will be copied to a temp directory. The script will
be executed passing the original filename, the path to the old file and the path to the current file
as arguments. If the script prints output the check will be marked as failed.

- `File` : string, the full path of the file
- `Script`: string, path to the script
- `ScriptOptions`: string, argument passed to the script
- `OldFilePath`: string, filename (absolute or relative) to use to store old file
- `InformationalOnly` : bool, (optional) the result of the check will be Informational only (default: false)

Script runs as:
```sh
script.sh <OrigFilename> <oldFile> <newFile> -- <argument>
```

Example:
```toml
[FileCmp."test.txt"]
File = "/test.txt"
Script = "diff.sh"
OldFilePath = "test.txt"
InformationalOnly = true
```

#### Json Field Compare

- `File` : string, the full path of the file
@@ -338,7 +371,7 @@ The `FileTree` check generates a full filesystem tree (a list of every file and
the newly generated filetree file is OldFileTreePath with ".new" appeneded to it.

The `OldFileTreePath` is relative to the configuration file. This means for '-cfg testdir/test.toml' with OldTreeFilePath = "test.json" fwanalyzer will
try to read 'testdir/test.json'. The `-tree` command line option can be used to overwrite the path: '-cfg testdir/test.toml -tree test1' will try to
try to read 'testdir/test.json'. The `-extra` command line option can be used to overwrite the path: '-cfg testdir/test.toml -extra test1' will try to
read 'test1/test.json'. Similar the newly generated filetree file will be stored in the same directory.

File modification check can be customized with:
@@ -29,6 +29,7 @@ import (
"github.com/cruise-automation/fwanalyzer/pkg/analyzer"
"github.com/cruise-automation/fwanalyzer/pkg/analyzer/dataextract"
"github.com/cruise-automation/fwanalyzer/pkg/analyzer/dircontent"
"github.com/cruise-automation/fwanalyzer/pkg/analyzer/filecmp"
"github.com/cruise-automation/fwanalyzer/pkg/analyzer/filecontent"
"github.com/cruise-automation/fwanalyzer/pkg/analyzer/filepathowner"
"github.com/cruise-automation/fwanalyzer/pkg/analyzer/filestatcheck"
@@ -90,7 +91,7 @@ func main() {
var cfgpath arrayFlags
var in = flag.String("in", "", "filesystem image file or path to directory")
var out = flag.String("out", "-", "output to file (use - for stdout)")
var tree = flag.String("tree", "", "overwrite directory to read the filetree file from")
var extra = flag.String("extra", "", "overwrite directory to read extra data from (filetree, cmpfile, ...)")
var cfg = flag.String("cfg", "", "config file")
flag.Var(&cfgpath, "cfgpath", "path to config file and included files (can be repated)")
var errorExit = flag.Bool("ee", false, "exit with error if offenders are present")
@@ -109,9 +110,9 @@ func main() {
os.Exit(1)
}

// if no alternative filetree directory is given use the directory "config filepath"
if *tree == "" {
*tree = path.Dir(*cfg)
// if no alternative extra data directory is given use the directory "config filepath"
if *extra == "" {
*extra = path.Dir(*cfg)
}

analyzer := analyzer.NewFromConfig(*in, string(cfgdata))
@@ -124,11 +125,12 @@ func main() {

analyzer.AddAnalyzerPlugin(globalfilechecks.New(string(cfgdata), analyzer))
analyzer.AddAnalyzerPlugin(filecontent.New(string(cfgdata), analyzer, *invertMatch))
analyzer.AddAnalyzerPlugin(filecmp.New(string(cfgdata), analyzer, *extra))
analyzer.AddAnalyzerPlugin(dataextract.New(string(cfgdata), analyzer))
analyzer.AddAnalyzerPlugin(dircontent.New(string(cfgdata), analyzer))
analyzer.AddAnalyzerPlugin(filestatcheck.New(string(cfgdata), analyzer))
analyzer.AddAnalyzerPlugin(filepathowner.New(string(cfgdata), analyzer))
analyzer.AddAnalyzerPlugin(filetree.New(string(cfgdata), analyzer, *tree))
analyzer.AddAnalyzerPlugin(filetree.New(string(cfgdata), analyzer, *extra))

analyzer.RunPlugins()

@@ -0,0 +1,193 @@
/*
Copyright 2019 GM Cruise LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package filecmp

import (
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path"
"syscall"

"github.com/BurntSushi/toml"

"github.com/cruise-automation/fwanalyzer/pkg/analyzer"
"github.com/cruise-automation/fwanalyzer/pkg/fsparser"
)

type cmpType struct {
File string // filename
OldFilePath string
Script string
ScriptOptions string
InformationalOnly bool // put result into Informational (not Offenders)
name string // name of this check (need to be unique)

}

type fileCmpType struct {
files map[string][]cmpType
a analyzer.AnalyzerType
}

func New(config string, a analyzer.AnalyzerType, fileDirectory string) *fileCmpType {
type fileCmpListType struct {
FileCmp map[string]cmpType
}
cfg := fileCmpType{a: a, files: make(map[string][]cmpType)}

var fcc fileCmpListType
_, err := toml.Decode(config, &fcc)
if err != nil {
panic("can't read config data: " + err.Error())
}

// convert text name based map to filename based map with an array of checks
for name, item := range fcc.FileCmp {
// make sure required options are set
if item.OldFilePath == "" || item.Script == "" {
continue
}
var items []cmpType
if _, ok := cfg.files[item.File]; ok {
items = cfg.files[item.File]
}

if fileDirectory != "" {
item.OldFilePath = path.Join(fileDirectory, item.OldFilePath)
}

item.name = name
item.File = path.Clean(item.File)
items = append(items, item)
cfg.files[item.File] = items
}

return &cfg
}

func (state *fileCmpType) Start() {}

func (state *fileCmpType) Finalize() string {
return ""
}

func (state *fileCmpType) Name() string {
return "FileCmp"
}

func fileExists(filePath string) error {
var fileState syscall.Stat_t
return syscall.Lstat(filePath, &fileState)
}

func copyFile(out string, in string) error {
src, err := os.Open(in)
if err != nil {
return err
}
defer src.Close()
dst, err := os.Create(out)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, src)
return err
}

func makeTmpFromOld(filePath string) (string, error) {
tmpfile, err := ioutil.TempFile("", "")
if err != nil {
return "", err
}
defer tmpfile.Close()
src, err := os.Open(filePath)
if err != nil {
return "", err
}
defer src.Close()
_, err = io.Copy(tmpfile, src)
return tmpfile.Name(), err
}

func (state *fileCmpType) CheckFile(fi *fsparser.FileInfo, filepath string) error {
if !fi.IsFile() {
return nil
}

fn := path.Join(filepath, fi.Name)
if _, ok := state.files[fn]; !ok {
return nil
}

for _, item := range state.files[fn] {
tmpfn, err := state.a.FileGet(fn)
if err != nil {
state.a.AddOffender(fn, fmt.Sprintf("FileCmp: error getting file: %s", err))
continue
}

// we don't have a saved file so save it now and skip this check
if fileExists(item.OldFilePath) != nil {
err := copyFile(item.OldFilePath+".new", tmpfn)
if err != nil {
state.a.AddOffender(fn, fmt.Sprintf("FileCmp: error saving file: %s", err))
continue
}
state.a.AddInformational(fn, "FileCmp: saved file for next run")
continue
}

oldTmp, err := makeTmpFromOld(item.OldFilePath)
if err != nil {
state.a.AddOffender(fn, fmt.Sprintf("FileCmp: error getting old file: %s", err))
continue
}
args := []string{fi.Name, oldTmp, tmpfn}
if len(item.ScriptOptions) > 0 {
args = append(args, "--")
args = append(args, item.ScriptOptions)
}

out, err := exec.Command(item.Script, args...).CombinedOutput()
if err != nil {
state.a.AddOffender(path.Join(filepath, fi.Name), fmt.Sprintf("script(%s) error=%s", item.Script, err))
}

err = state.a.RemoveFile(tmpfn)
if err != nil {
panic("removeFile failed")
}
err = state.a.RemoveFile(oldTmp)
if err != nil {
panic("removeFile failed")
}

if len(out) > 0 {
if item.InformationalOnly {
state.a.AddInformational(path.Join(filepath, fi.Name), string(out))
} else {
state.a.AddOffender(path.Join(filepath, fi.Name), string(out))
}
}
}

return nil
}

0 comments on commit 996312e

Please sign in to comment.
You can’t perform that action at this time.