From 667da8a02b0d63f19c6f33799af511e9ed6a6102 Mon Sep 17 00:00:00 2001 From: Nicolas DUBIEN Date: Fri, 26 May 2023 12:08:40 +0000 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20Detect=20fast-check=20PBT=20lib?= =?UTF-8?q?rary=20for=20fuzz=20section?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As suggested at https://github.com/ossf/scorecard/issues/2792#issuecomment-1562007596, we add support for the detection of fast-check as a possible fuzzing solution. I also adapted the documentation related to fuzzing accordingly. Signed-off-by: Nicolas DUBIEN --- checks/raw/fuzzing.go | 36 +++++++++++++--- checks/raw/fuzzing_test.go | 74 ++++++++++++++++++++++++++++++++ docs/checks/internal/checks.yaml | 4 +- 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 8f01595881b..670a3c21f1d 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -28,11 +28,13 @@ import ( ) const ( - fuzzerOSSFuzz = "OSSFuzz" - fuzzerClusterFuzzLite = "ClusterFuzzLite" - oneFuzz = "OneFuzz" - fuzzerBuiltInGo = "GoBuiltInFuzzer" - fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting" + fuzzerOSSFuzz = "OSSFuzz" + fuzzerClusterFuzzLite = "ClusterFuzzLite" + oneFuzz = "OneFuzz" + fuzzerBuiltInGo = "GoBuiltInFuzzer" + fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting" + fuzzerPropertyBasedJavaScript = "JavaScriptPropertyBasedTesting" + fuzzerPropertyBasedTypeScript = "TypeScriptPropertyBasedTesting" // TODO: add more fuzzing check supports. ) @@ -87,6 +89,30 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ "Property-based testing in Haskell generates test instances randomly or exhaustively " + "and test that specific properties are satisfied."), }, + // Fuzz patterns for JavaScript and TypeScript based on property-based testing. + // + // Based on the import of one of these packages: + // * https://fast-check.dev/ + // + // This is not an exhaustive list. + clients.JavaScript: { + filePattern: "*.js", + // Look for direct imports of fast-check. + funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`, + Name: fuzzerPropertyBasedJavaScript, + Desc: asPointer( + "Property-based testing in JavaScript generates test instances randomly or exhaustively " + + "and test that specific properties are satisfied."), + }, + clients.TypeScript: { + filePattern: "*.ts", + // Look for direct imports of fast-check. + funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`, + Name: fuzzerPropertyBasedTypeScript, + Desc: asPointer( + "Property-based testing in JavaScript generates test instances randomly or exhaustively " + + "and test that specific properties are satisfied."), + }, // TODO: add more language-specific fuzz patterns & configs. } diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index 5804a30c62a..c0a06edfe72 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -413,6 +413,80 @@ func Test_checkFuzzFunc(t *testing.T) { }, fileContent: "import Test.Hspec", }, + { + name: "JavaScript fast-check via require", + want: true, + fileName: []string{"main.spec.js"}, + langs: []clients.Language{ + { + Name: clients.JavaScript, + NumLines: 50, + }, + }, + fileContent: "const fc = require('fast-check');", + }, + { + name: "JavaScript fast-check via import", + want: true, + fileName: []string{"main.spec.js"}, + langs: []clients.Language{ + { + Name: clients.JavaScript, + NumLines: 50, + }, + }, + fileContent: "import fc from \"fast-check\";", + }, + { + name: "JavaScript with no property-based testing", + want: false, + fileName: []string{"main.spec.js"}, + wantErr: true, + langs: []clients.Language{ + { + Name: clients.JavaScript, + NumLines: 50, + }, + }, + fileContent: "const fc = require('fast-other');", + }, + { + name: "TypeScript fast-check via require", + want: true, + fileName: []string{"main.spec.ts"}, + langs: []clients.Language{ + { + Name: clients.TypeScript, + NumLines: 50, + }, + }, + fileContent: "const fc = require('fast-check');", + }, + { + name: "TypeScript fast-check via import", + want: true, + fileName: []string{"main.spec.ts"}, + langs: []clients.Language{ + { + Name: clients.TypeScript, + NumLines: 50, + }, + }, + fileContent: "import fc from \"fast-check\";", + }, + { + name: "TypeScript with no property-based testing", + want: false, + fileName: []string{"main.spec.ts"}, + wantErr: true, + langs: []clients.Language{ + { + Name: clients.TypeScript, + NumLines: 50, + }, + }, + fileContent: "const fc = require('fast-other');", + }, } for _, tt := range tests { tt := tt diff --git a/docs/checks/internal/checks.yaml b/docs/checks/internal/checks.yaml index 7b8ad4eca13..406d9871b60 100644 --- a/docs/checks/internal/checks.yaml +++ b/docs/checks/internal/checks.yaml @@ -398,7 +398,9 @@ checks: 1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list; 2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository; 3. if there are user-defined language-specified fuzzing functions in the repository. - - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell. + - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/), + - a limited set of property-based testing libraries for Haskell including [QuickCheck](https://hackage.haskell.org/package/QuickCheck), [Hedgehog](https://hedgehog.qa/), [validity](https://hackage.haskell.org/package/validity) or [SmallCheck](https://hackage.haskell.org/package/smallcheck), + - a limited set of property-based testing libraries for JavaScript and TypeScript including [fast-check](https://fast-check.dev/). 4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz); Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data From 0d7bf2e65ae0931bfa6b34ff3adf7766e05fc392 Mon Sep 17 00:00:00 2001 From: Nicolas DUBIEN Date: Fri, 26 May 2023 12:25:16 +0000 Subject: [PATCH 2/3] Typo Signed-off-by: Nicolas DUBIEN --- checks/raw/fuzzing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 670a3c21f1d..a8a161d9847 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -110,7 +110,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`, Name: fuzzerPropertyBasedTypeScript, Desc: asPointer( - "Property-based testing in JavaScript generates test instances randomly or exhaustively " + + "Property-based testing in TypeScript generates test instances randomly or exhaustively " + "and test that specific properties are satisfied."), }, // TODO: add more language-specific fuzz patterns & configs. From f9081517fca73ca374eac714c5a40f942725a917 Mon Sep 17 00:00:00 2001 From: Nicolas DUBIEN Date: Mon, 29 May 2023 10:04:30 +0000 Subject: [PATCH 3/3] Update missing md files Signed-off-by: Nicolas DUBIEN --- README.md | 2 +- docs/checks.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a36a9763342..2055a4751f7 100644 --- a/README.md +++ b/README.md @@ -457,7 +457,7 @@ Name | Description | Risk Level | Token Req [Contributors](docs/checks.md#contributors) | Does the project have contributors from at least two different organizations? | Low | PAT, GITHUB_TOKEN | Validating | [Dangerous-Workflow](docs/checks.md#dangerous-workflow) | Does the project avoid dangerous coding patterns in GitHub Action workflows? | Critical | PAT, GITHUB_TOKEN | Unsupported | [Dependency-Update-Tool](docs/checks.md#dependency-update-tool) | Does the project use tools to help update its dependencies? | High | PAT, GITHUB_TOKEN | Unsupported | -[Fuzzing](docs/checks.md#fuzzing) | Does the project use fuzzing tools, e.g. [OSS-Fuzz](https://github.com/google/oss-fuzz)? | Medium | PAT, GITHUB_TOKEN | Validating +[Fuzzing](docs/checks.md#fuzzing) | Does the project use fuzzing tools, e.g. [OSS-Fuzz](https://github.com/google/oss-fuzz), [QuickCheck](https://hackage.haskell.org/package/QuickCheck) or [fast-check](https://fast-check.dev/)? | Medium | PAT, GITHUB_TOKEN | Validating [License](docs/checks.md#license) | Does the project declare a license? | Low | PAT, GITHUB_TOKEN | Validating | [Maintained](docs/checks.md#maintained) | Is the project at least 90 days old, and maintained? | High | PAT, GITHUB_TOKEN | Validating | [Pinned-Dependencies](docs/checks.md#pinned-dependencies) | Does the project declare and pin [dependencies](https://docs.github.com/en/free-pro-team@latest/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems)? | Medium | PAT, GITHUB_TOKEN | Validating | diff --git a/docs/checks.md b/docs/checks.md index 7db3d80baf7..406bb213f1b 100644 --- a/docs/checks.md +++ b/docs/checks.md @@ -338,7 +338,9 @@ This check tries to determine if the project uses 1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list; 2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository; 3. if there are user-defined language-specified fuzzing functions in the repository. - - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell. + - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/), + - a limited set of property-based testing libraries for Haskell including [QuickCheck](https://hackage.haskell.org/package/QuickCheck), [Hedgehog](https://hedgehog.qa/), [validity](https://hackage.haskell.org/package/validity) or [SmallCheck](https://hackage.haskell.org/package/smallcheck), + - a limited set of property-based testing libraries for JavaScript and TypeScript including [fast-check](https://fast-check.dev/). 4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz); Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data