✨ Support user-defined fuzz functions (GoLang) in fuzzing check (#1979)

* temp save 05262022 * finished golang fuzz func check, getLang interface to be done next week * temp save 05/31/2022 * temp save 06/01/2022 * temp save-2 06/01/2022 * temp save-1 06032022 * temp save-2 06022022 * temp save * temp save 06032022 * temp save 06032022 (2) * update err def * temp save 3 * update docs for fuzzing * update docs for fuzzing * update checks.yaml to gen docs * temp save 0606 * temp save-2 0606 * temp save-3 0606 * temp save-4 0606 * fix linter errors * fix linter errs-2 * fix e2e errors * 0608 * 0608-2 Co-authored-by: Aiden Wang <aidenwang@google.com>
ossf · Jun 9, 2022 · 64cd053 · 64cd053
1 parent 3b7c46f
commit 64cd053
Show file tree

Hide file tree

Showing 19 changed files with 593 additions and 62 deletions.
diff --git a/checker/raw_result.go b/checker/raw_result.go
@@ -174,10 +174,10 @@ type BranchProtectionsData struct {
 
 // Tool represents a tool.
 type Tool struct {
-	URL  *string
-	Desc *string
-	File *File
-	Name string
+	URL   *string
+	Desc  *string
+	Files []File
+	Name  string
 	// Runs of the tool.
 	Runs []Run
 	// Issues created by the tool.

diff --git a/checks/evaluation/dependency_update_tool.go b/checks/evaluation/dependency_update_tool.go
@@ -49,19 +49,20 @@ func DependencyUpdateTool(name string, dl checker.DetailLogger,
 		return checker.CreateRuntimeErrorResult(name, e)
 	}
 
-	if r.Tools[0].File == nil {
-		e := sce.WithMessage(sce.ErrScorecardInternal, "File is nil")
+	if r.Tools[0].Files == nil {
+		e := sce.WithMessage(sce.ErrScorecardInternal, "Files are nil")
 		return checker.CreateRuntimeErrorResult(name, e)
 	}
 
-	// Note: only one file per tool is present,
-	// so we do not iterate thru all entries.
-	dl.Info(&checker.LogMessage{
-		Path:   r.Tools[0].File.Path,
-		Type:   r.Tools[0].File.Type,
-		Offset: r.Tools[0].File.Offset,
-		Text:   fmt.Sprintf("%s detected", r.Tools[0].Name),
-	})
+	// Iterate over all the files, since a Tool can contain multiple files.
+	for _, file := range r.Tools[0].Files {
+		dl.Info(&checker.LogMessage{
+			Path:   file.Path,
+			Type:   file.Type,
+			Offset: file.Offset,
+			Text:   fmt.Sprintf("%s detected", r.Tools[0].Name),
+		})
+	}
 
 	// High score result.
 	return checker.CreateMaxScoreResult(name, "update tool detected")

diff --git a/checks/evaluation/dependency_update_tool_test.go b/checks/evaluation/dependency_update_tool_test.go
@@ -88,14 +88,15 @@ func TestDependencyUpdateTool(t *testing.T) {
 					Tools: []checker.Tool{
 						{
 							Name: "DependencyUpdateTool",
-							File: &checker.File{
-								Path: "/etc/dependency-update-tool.conf",
-								Snippet: `
+							Files: []checker.File{
+								{
+									Path: "/etc/dependency-update-tool.conf",
+									Snippet: `
 										[dependency-update-tool]
 										enabled = true
 										`,
-								Offset: 0,
-								Type:   0,
+									Type: checker.FileTypeSource,
+								},
 							},
 						},
 					},

diff --git a/checks/evaluation/fuzzing.go b/checks/evaluation/fuzzing.go
@@ -30,11 +30,25 @@ func Fuzzing(name string, dl checker.DetailLogger,
 		return checker.CreateRuntimeErrorResult(name, e)
 	}
 
+	if len(r.Fuzzers) == 0 {
+		return checker.CreateMinScoreResult(name, "project is not fuzzed")
+	}
+	fuzzers := []string{}
 	for i := range r.Fuzzers {
 		fuzzer := r.Fuzzers[i]
-		return checker.CreateMaxScoreResult(name,
-			fmt.Sprintf("project is fuzzed with %s", fuzzer.Name))
+		for _, f := range fuzzer.Files {
+			msg := checker.LogMessage{
+				Path:   f.Path,
+				Type:   f.Type,
+				Offset: f.Offset,
+			}
+			if f.Snippet != "" {
+				msg.Text = f.Snippet
+			}
+			dl.Info(&msg)
+		}
+		fuzzers = append(fuzzers, fuzzer.Name)
 	}
-
-	return checker.CreateMinScoreResult(name, "project is not fuzzed")
+	return checker.CreateMaxScoreResult(name,
+		fmt.Sprintf("project is fuzzed with %v", fuzzers))
 }
diff --git a/checks/fuzzing_test.go b/checks/fuzzing_test.go
@@ -34,6 +34,7 @@ func TestFuzzing(t *testing.T) {
 	tests := []struct {
 		name        string
 		want        checker.CheckResult
+		langs       map[clients.Language]int
 		response    clients.SearchResponse
 		wantErr     bool
 		wantFuzzErr bool
@@ -44,13 +45,20 @@ func TestFuzzing(t *testing.T) {
 		{
 			name:     "empty response",
 			response: clients.SearchResponse{},
-			wantErr:  false,
+			langs: map[clients.Language]int{
+				clients.Go: 300,
+			},
+			wantErr: false,
 		},
 		{
 			name: "hits 1",
 			response: clients.SearchResponse{
 				Hits: 1,
 			},
+			langs: map[clients.Language]int{
+				clients.Go:   100,
+				clients.Java: 70,
+			},
 			wantErr: false,
 			want:    checker.CheckResult{Score: 10},
 			expected: scut.TestReturn{
@@ -61,7 +69,10 @@ func TestFuzzing(t *testing.T) {
 			},
 		},
 		{
-			name:    "nil response",
+			name: "nil response",
+			langs: map[clients.Language]int{
+				clients.Python: 256,
+			},
 			wantErr: true,
 			want:    checker.CheckResult{Score: -1},
 			expected: scut.TestReturn{
@@ -73,7 +84,15 @@ func TestFuzzing(t *testing.T) {
 			},
 		},
 		{
-			name:        " error",
+			name: "min score since lang not supported",
+			langs: map[clients.Language]int{
+				clients.Language("not_supported_lang"): 1490,
+			},
+			wantFuzzErr: false,
+			want:        checker.CheckResult{Score: 0},
+		},
+		{
+			name:        "error",
 			wantFuzzErr: true,
 			want:        checker.CheckResult{},
 		},
@@ -94,7 +113,7 @@ func TestFuzzing(t *testing.T) {
 					}
 					return tt.response, nil
 				}).AnyTimes()
-
+			mockFuzz.EXPECT().ListProgrammingLanguages().Return(tt.langs, nil).AnyTimes()
 			mockFuzz.EXPECT().ListFiles(gomock.Any()).Return(tt.fileName, nil).AnyTimes()
 			mockFuzz.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) (string, error) {
 				if tt.wantErr {

diff --git a/checks/raw/dependency_update_tool.go b/checks/raw/dependency_update_tool.go
@@ -51,10 +51,12 @@ var checkDependencyFileExists fileparser.DoWhileTrueOnFilename = func(name strin
 			Name: "Dependabot",
 			URL:  asPointer("https://github.com/dependabot"),
 			Desc: asPointer("Automated dependency updates built into GitHub"),
-			File: &checker.File{
-				Path:   name,
-				Type:   checker.FileTypeSource,
-				Offset: checker.OffsetDefault,
+			Files: []checker.File{
+				{
+					Path:   name,
+					Type:   checker.FileTypeSource,
+					Offset: checker.OffsetDefault,
+				},
 			},
 		})
 
@@ -65,10 +67,12 @@ var checkDependencyFileExists fileparser.DoWhileTrueOnFilename = func(name strin
 			Name: "Renovabot",
 			URL:  asPointer("https://github.com/renovatebot/renovate"),
 			Desc: asPointer("Automated dependency updates. Multi-platform and multi-language."),
-			File: &checker.File{
-				Path:   name,
-				Type:   checker.FileTypeSource,
-				Offset: checker.OffsetDefault,
+			Files: []checker.File{
+				{
+					Path:   name,
+					Type:   checker.FileTypeSource,
+					Offset: checker.OffsetDefault,
+				},
 			},
 		})
 	default:

diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go
@@ -15,14 +15,51 @@
 package raw
 
 import (
+	"bytes"
 	"fmt"
+	"regexp"
+	"strings"
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/checks/fileparser"
 	"github.com/ossf/scorecard/v4/clients"
 	sce "github.com/ossf/scorecard/v4/errors"
 )
 
+const (
+	fuzzerOSSFuzz         = "OSSFuzz"
+	fuzzerClusterFuzzLite = "ClusterFuzzLite"
+	fuzzerBuiltInGo       = "GoBuiltInFuzzer"
+	// TODO: add more fuzzing check supports.
+)
+
+type filesWithPatternStr struct {
+	pattern string
+	files   []checker.File
+}
+
+// Configurations for language-specified fuzzers.
+type languageFuzzConfig struct {
+	URL, Desc                      *string
+	filePattern, funcPattern, Name string
+	//TODO: add more language fuzzing-related fields.
+}
+
+// Contains fuzzing speficications for programming languages.
+// Please use the type Language defined in clients/languages.go rather than a raw string.
+var languageFuzzSpecs = map[clients.Language]languageFuzzConfig{
+	// Default fuzz patterns for Go.
+	clients.Go: {
+		filePattern: "*_test.go",
+		funcPattern: `func\s+Fuzz\w+\s*\(\w+\s+\*testing.F\)`,
+		Name:        fuzzerBuiltInGo,
+		URL:         asPointer("https://go.dev/doc/fuzz/"),
+		Desc: asPointer(
+			"Go fuzzing intelligently walks through the source code to report failures and find vulnerabilities."),
+	},
+	// TODO: add more language-specific fuzz patterns & configs.
+}
+
 // Fuzzing runs Fuzzing check.
 func Fuzzing(c *checker.CheckRequest) (checker.FuzzingData, error) {
 	var fuzzers []checker.Tool
@@ -33,7 +70,7 @@ func Fuzzing(c *checker.CheckRequest) (checker.FuzzingData, error) {
 	if usingCFLite {
 		fuzzers = append(fuzzers,
 			checker.Tool{
-				Name: "ClusterFuzzLite",
+				Name: fuzzerClusterFuzzLite,
 				URL:  asPointer("https://github.com/google/clusterfuzzlite"),
 				Desc: asPointer("continuous fuzzing solution that runs as part of Continuous Integration (CI) workflows"),
 				// TODO: File.
@@ -48,14 +85,36 @@ func Fuzzing(c *checker.CheckRequest) (checker.FuzzingData, error) {
 	if usingOSSFuzz {
 		fuzzers = append(fuzzers,
 			checker.Tool{
-				Name: "OSS-Fuzz",
+				Name: fuzzerOSSFuzz,
 				URL:  asPointer("https://github.com/google/oss-fuzz"),
 				Desc: asPointer("Continuous Fuzzing for Open Source Software"),
 				// TODO: File.
 			},
 		)
 	}
 
+	langMap, err := c.RepoClient.ListProgrammingLanguages()
+	if err != nil {
+		return checker.FuzzingData{}, fmt.Errorf("cannot get langs of repo: %w", err)
+	}
+	prominentLangs := getProminentLanguages(langMap)
+
+	for _, lang := range prominentLangs {
+		usingFuzzFunc, files, e := checkFuzzFunc(c, lang)
+		if e != nil {
+			return checker.FuzzingData{}, fmt.Errorf("%w", e)
+		}
+		if usingFuzzFunc {
+			fuzzers = append(fuzzers,
+				checker.Tool{
+					Name:  languageFuzzSpecs[lang].Name,
+					URL:   languageFuzzSpecs[lang].URL,
+					Desc:  languageFuzzSpecs[lang].Desc,
+					Files: files,
+				},
+			)
+		}
+	}
 	return checker.FuzzingData{Fuzzers: fuzzers}, nil
 }
 
@@ -91,3 +150,93 @@ func checkOSSFuzz(c *checker.CheckRequest) (bool, error) {
 	}
 	return result.Hits > 0, nil
 }
+
+func checkFuzzFunc(c *checker.CheckRequest, lang clients.Language) (bool, []checker.File, error) {
+	if c.RepoClient == nil {
+		return false, nil, nil
+	}
+	data := filesWithPatternStr{
+		files: make([]checker.File, 0),
+	}
+	// Search language-specified fuzz func patterns in the hashmap.
+	pattern, found := languageFuzzSpecs[lang]
+	if !found {
+		// If the fuzz patterns for the current language not supported yet,
+		// we return it as false (not found), nil (no files), and nil (no errors).
+		return false, nil, nil
+	}
+	// Get patterns for file and func.
+	// We use the file pattern in the matcher to match the test files,
+	// and put the func pattern in var data to match file contents (func names).
+	filePattern, funcPattern := pattern.filePattern, pattern.funcPattern
+	matcher := fileparser.PathMatcher{
+		Pattern:       filePattern,
+		CaseSensitive: false,
+	}
+	data.pattern = funcPattern
+	err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data)
+	if err != nil {
+		return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err)
+	}
+
+	if len(data.files) == 0 {
+		// This means no fuzz funcs matched for this language.
+		return false, nil, nil
+	}
+	return true, data.files, nil
+}
+
+// This is the callback func for interface OnMatchingFileContentDo
+// used for matching fuzz functions in the file content,
+// and return a list of files (or nil for not found).
+var getFuzzFunc fileparser.DoWhileTrueOnFileContent = func(
+	path string, content []byte, args ...interface{}) (bool, error) {
+	if len(args) != 1 {
+		return false, fmt.Errorf("getFuzzFunc requires exactly one argument: %w", errInvalidArgLength)
+	}
+	pdata, ok := args[0].(*filesWithPatternStr)
+	if !ok {
+		return false, errInvalidArgType
+	}
+	r := regexp.MustCompile(pdata.pattern)
+	lines := bytes.Split(content, []byte("\n"))
+	for i, line := range lines {
+		found := r.FindString(string(line))
+		if found != "" {
+			// If fuzz func is found in the file, add it to the file array,
+			// with its file path as Path, func name as Snippet,
+			// FileTypeFuzz as Type, and # of lines as Offset.
+			pdata.files = append(pdata.files, checker.File{
+				Path:    path,
+				Type:    checker.FileTypeSource,
+				Snippet: found,
+				Offset:  uint(i + 1), // Since the # of lines starts from zero.
+			})
+		}
+	}
+	return true, nil
+}
+
+func getProminentLanguages(langs map[clients.Language]int) []clients.Language {
+	numLangs := len(langs)
+	if numLangs == 0 {
+		return nil
+	}
+	totalLoC := 0
+	for _, LoC := range langs {
+		totalLoC += LoC
+	}
+	// Var avgLoC calculates the average lines of code in the current repo,
+	// and it can stay as an int, no need for a float value.
+	avgLoC := totalLoC / numLangs
+
+	// Languages that have lines of code above average will be considered prominent.
+	ret := []clients.Language{}
+	for lang, LoC := range langs {
+		if LoC >= avgLoC {
+			lang = clients.Language(strings.ToLower(string(lang)))
+			ret = append(ret, lang)
+		}
+	}
+	return ret
+}