From 0ba4c126172ade890d00ffa9e17891c433bb2568 Mon Sep 17 00:00:00 2001 From: Nicolas DUBIEN Date: Fri, 26 May 2023 12:08:40 +0000 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Detect=20fast-check=20PBT=20library?= =?UTF-8?q?=20for=20fuzz=20section?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As suggested at https://github.com/ossf/scorecard/issues/2792#issuecomment-1562007596, we add support for the detection of fast-check as a possible fuzzing solution. I also adapted the documentation related to fuzzing accordingly. Signed-off-by: Nicolas DUBIEN --- checks/raw/fuzzing.go | 36 ++++++++++++++++--- checks/raw/fuzzing_test.go | 74 ++++++++++++++++++++++++++++++++++++++ docs/checks.md | 4 ++- 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 8f01595881b4..670a3c21f1de 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -28,11 +28,13 @@ import ( ) const ( - fuzzerOSSFuzz = "OSSFuzz" - fuzzerClusterFuzzLite = "ClusterFuzzLite" - oneFuzz = "OneFuzz" - fuzzerBuiltInGo = "GoBuiltInFuzzer" - fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting" + fuzzerOSSFuzz = "OSSFuzz" + fuzzerClusterFuzzLite = "ClusterFuzzLite" + oneFuzz = "OneFuzz" + fuzzerBuiltInGo = "GoBuiltInFuzzer" + fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting" + fuzzerPropertyBasedJavaScript = "JavaScriptPropertyBasedTesting" + fuzzerPropertyBasedTypeScript = "TypeScriptPropertyBasedTesting" // TODO: add more fuzzing check supports. ) @@ -87,6 +89,30 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ "Property-based testing in Haskell generates test instances randomly or exhaustively " + "and test that specific properties are satisfied."), }, + // Fuzz patterns for JavaScript and TypeScript based on property-based testing. + // + // Based on the import of one of these packages: + // * https://fast-check.dev/ + // + // This is not an exhaustive list. + clients.JavaScript: { + filePattern: "*.js", + // Look for direct imports of fast-check. + funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`, + Name: fuzzerPropertyBasedJavaScript, + Desc: asPointer( + "Property-based testing in JavaScript generates test instances randomly or exhaustively " + + "and test that specific properties are satisfied."), + }, + clients.TypeScript: { + filePattern: "*.ts", + // Look for direct imports of fast-check. + funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`, + Name: fuzzerPropertyBasedTypeScript, + Desc: asPointer( + "Property-based testing in JavaScript generates test instances randomly or exhaustively " + + "and test that specific properties are satisfied."), + }, // TODO: add more language-specific fuzz patterns & configs. } diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index 5804a30c62a3..c0a06edfe729 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -413,6 +413,80 @@ func Test_checkFuzzFunc(t *testing.T) { }, fileContent: "import Test.Hspec", }, + { + name: "JavaScript fast-check via require", + want: true, + fileName: []string{"main.spec.js"}, + langs: []clients.Language{ + { + Name: clients.JavaScript, + NumLines: 50, + }, + }, + fileContent: "const fc = require('fast-check');", + }, + { + name: "JavaScript fast-check via import", + want: true, + fileName: []string{"main.spec.js"}, + langs: []clients.Language{ + { + Name: clients.JavaScript, + NumLines: 50, + }, + }, + fileContent: "import fc from \"fast-check\";", + }, + { + name: "JavaScript with no property-based testing", + want: false, + fileName: []string{"main.spec.js"}, + wantErr: true, + langs: []clients.Language{ + { + Name: clients.JavaScript, + NumLines: 50, + }, + }, + fileContent: "const fc = require('fast-other');", + }, + { + name: "TypeScript fast-check via require", + want: true, + fileName: []string{"main.spec.ts"}, + langs: []clients.Language{ + { + Name: clients.TypeScript, + NumLines: 50, + }, + }, + fileContent: "const fc = require('fast-check');", + }, + { + name: "TypeScript fast-check via import", + want: true, + fileName: []string{"main.spec.ts"}, + langs: []clients.Language{ + { + Name: clients.TypeScript, + NumLines: 50, + }, + }, + fileContent: "import fc from \"fast-check\";", + }, + { + name: "TypeScript with no property-based testing", + want: false, + fileName: []string{"main.spec.ts"}, + wantErr: true, + langs: []clients.Language{ + { + Name: clients.TypeScript, + NumLines: 50, + }, + }, + fileContent: "const fc = require('fast-other');", + }, } for _, tt := range tests { tt := tt diff --git a/docs/checks.md b/docs/checks.md index 975eb407877b..95f2ce291cf2 100644 --- a/docs/checks.md +++ b/docs/checks.md @@ -335,7 +335,9 @@ This check tries to determine if the project uses 1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list; 2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository; 3. if there are user-defined language-specified fuzzing functions in the repository. - - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell. + - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) + - a limited set of property-based testing libraries for Haskell including [QuickCheck](https://hackage.haskell.org/package/QuickCheck), [Hedgehog](https://hedgehog.qa/), [validity](https://hackage.haskell.org/package/validity) or [SmallCheck](https://hackage.haskell.org/package/smallcheck) + - a limited set of property-based testing libraries for JavaScript and TypeScript including [fast-check](https://fast-check.dev/) 4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz); Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data