Merge branch 'main' into unpinned-line-number-uses

.github/workflows/goreleaser.yaml

@@ -40,7 +40,7 @@ jobs:
       -
         name: Import GPG key
         id: import_gpg
-        uses: crazy-max/ghaction-import-gpg@b0793c0060c97f4ef0efbac949d476c6499b7775 # v3.1.0
+        uses: crazy-max/ghaction-import-gpg@1c6a9e9d3594f2d743f1b1dd7669ab0dfdffa922 # v3.1.0
         with:
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.PASSPHRASE }}

.github/workflows/integration.yml

@@ -116,7 +116,7 @@ jobs:
             Integration tests ${{ job.status }} for [${{ github.event.client_payload.slash_command.args.named.sha || github.event.pull_request.head.sha }}](https://github.com/ossf/scorecard/actions/runs/${{ github.run_id }})
 
       - name: set fork job status
-        uses: actions/github-script@a3e7071a34d7e1f219a8a4de9a5e0a34d1ee1293 # v4.0.2
+        uses: actions/github-script@f891eff65186019cbb3f7190c4590bc0a1b76fbc # v4.0.2
         if: ${{ always() }}
         id: update-check-run
         env:

.github/workflows/main.yml

@@ -64,9 +64,8 @@ jobs:
           go-version: '1.16'
       - name: Check license headers
         run: |
-          echo "license check disabled. Will be reenabled soon."
           go env -w GOFLAGS=-mod=mod
           make all
           set -e
-          addlicense -ignore "**/script-empty.sh"  -l apache -c 'Security Scorecard Authors' -v *
+          addlicense -ignore "**/script-empty.sh" -ignore "pkg/testdata/*"  -l apache -c 'Security Scorecard Authors' -v *
           git diff --exit-code

.gitignore

@@ -6,6 +6,7 @@ cron/data/validate/validate
 cron/data/update/projects-update
 cron/controller/controller
 cron/worker/worker
+cron/webhook/webhook
 cron/bq/data-transfer
 
 # Binaries for programs and plugins.

.goreleaser.yml

@@ -31,7 +31,7 @@ checksum:
 changelog:
   # Set it to true if you wish to skip the changelog generation.
   # This may result in an empty release notes on GitHub/GitLab/Gitea.
-  skip: true
+  skip: false
 signs:
   - artifacts: checksum
     args: ["--batch", "-u", "{{ .Env.GPG_FINGERPRINT }}", "--output", "${signature}", "--detach-sign", "${artifact}"]

Dockerfile

@@ -12,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-FROM  golang:1.16.7@sha256:5cdc91c9e67e7b95ef5a1c9156af24aaccbb4e339799efd441fd7e961b3e2e75 AS base
+FROM golang@sha256:3c4de86eec9cbc619cdd72424abd88326ffcf5d813a8338a7743c55e5898734f AS base
 WORKDIR /src
 ENV CGO_ENABLED=0
 COPY go.* ./

Makefile

@@ -1,7 +1,7 @@
 SHELL := /bin/bash
 GOPATH := $(go env GOPATH)
 GINKGO := ginkgo
-GIT_HASH := $(git rev-parse HEAD)
+GIT_HASH := $(shell git rev-parse HEAD)
 GOLANGGCI_LINT := golangci-lint
 PROTOC_GEN_GO := protoc-gen-go
 PROTOC := $(shell which protoc)
@@ -114,6 +114,10 @@ build-bq-transfer: ./cron/bq/*.go
 	# Run go build on the Copier cron job
 	cd cron/bq && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "static"' -o data-transfer
 
+build-webhook: ## Runs go build on the cron webhook
+	# Run go build on the cron webhook
+	cd cron/webhook && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "static"' -o webhook
+
 build-add-script: ## Runs go build on the add script
 build-add-script: cron/data/add/add
 cron/data/add/add: cron/data/add/*.go cron/data/*.go cron/data/projects.csv
@@ -140,6 +144,7 @@ dockerbuild: ## Runs docker build
 			--build-arg=COMMIT_SHA=$(GIT_HASH) --tag $(IMAGE_NAME)-batch-controller
 	DOCKER_BUILDKIT=1 docker build . --file cron/worker/Dockerfile  --tag $(IMAGE_NAME)-batch-worker
 	DOCKER_BUILDKIT=1 docker build . --file cron/bq/Dockerfile --tag $(IMAGE_NAME)-bq-transfer
+	DOCKER_BUILDKIT=1 docker build . --file cron/webhook/Dockerfile --tag ${IMAGE_NAME}-webhook
 ###############################################################################
 
 ################################# make test ###################################

README.md

@@ -11,11 +11,11 @@
 *   [Goals](#goals)
 *   [Scorecard Checks](#scorecard-checks)
 *   [Usage](#usage)
+    *   [Authentication](#authentication)
     *   [Docker](#docker)
     *   [Using repository URL](#using-repository-url)
     *   [Using a Package manager](#using-a-package-manager)
     *   [Running specific checks](#running-specific-checks)
-    *   [Authentication](#authentication)
     *   [Understanding Scorecard results](#understanding-scorecard-results)
     *   [Formatting Results](#formatting-results)
 *   [Public Data](#public-data)
@@ -59,7 +59,6 @@ Packaging                   | Does the project build and publish official packag
 SAST                        | Does the project use static code analysis tools, e.g. [CodeQL](https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/enabling-code-scanning-for-a-repository#enabling-code-scanning-using-actions), [SonarCloud](https://sonarcloud.io)?
 Security-Policy             | Does the project contain a [security policy](https://docs.github.com/en/free-pro-team@latest/github/managing-security-vulnerabilities/adding-a-security-policy-to-your-repository)?
 Signed-Releases             | Does the project cryptographically [sign releases](https://wiki.debian.org/Creating%20signed%20GitHub%20releases)?
-Signed-Tags                 | Does the project cryptographically sign release tags?
 Token-Permissions           | Does the project declare GitHub workflow tokens as [read only](https://docs.github.com/en/actions/reference/authentication-in-a-workflow)?
 Vulnerabilities             | Does the project have unfixed vulnerabilities? Uses the [OSV service](https://osv.dev).
 
@@ -68,14 +67,50 @@ the [checks documentation page](docs/checks.md).
 
 ## Usage
 
+### Authentication
+
+Before running Scorecard, you need to, either:
+
+-   [create a GitHub access token](https://docs.github.com/en/free-pro-team@latest/developers/apps/about-apps#personal-access-tokens)
+    and set it in an environment variable called `GITHUB_AUTH_TOKEN`,
+    `GITHUB_TOKEN`, `GH_AUTH_TOKEN` or `GH_TOKEN`. This helps to avoid the
+    GitHub's [api rate limits](https://developer.github.com/v3/#rate-limiting)
+    with unauthenticated requests.
+
+```shell
+# For posix platforms, e.g. linux, mac:
+export GITHUB_AUTH_TOKEN=<your access token>
+# Multiple tokens can be provided separated by comma to be utilized
+# in a round robin fashion.
+export GITHUB_AUTH_TOKEN=<your access token1>,<your access token2>
+
+# For windows:
+set GITHUB_AUTH_TOKEN=<your access token>
+set GITHUB_AUTH_TOKEN=<your access token1>,<your access token2>
+```
+
+-   create a GitHub App Installations for higher rate-limit quotas. If you have
+    an installed GitHub App and key file, you can use these three environment
+    variables, following the commands shown above for your platform.
+
+```
+GITHUB_APP_KEY_PATH=<path to the key file on disk>
+GITHUB_APP_INSTALLATION_ID=<installation id>
+GITHUB_APP_ID=<app id>
+```
+
+These can be obtained from the GitHub
+[developer settings](https://github.com/settings/apps) page.
+
+
 ### Docker
 
 `scorecard` is available as a Docker container:
 
 The `GITHUB_AUTH_TOKEN` has to be set to a valid [token](#Authentication)
 
 ```shell
-docker run -e GITHUB_AUTH_TOKEN=token gcr.io/openssf/scorecard:latest --show-details --repo=https://github.com/ossf/scorecard
+docker run -e GITHUB_AUTH_TOKEN=token gcr.io/openssf/scorecard:stable --show-details --repo=https://github.com/ossf/scorecard
 ```
 
 ### Using repository URL
@@ -85,7 +120,6 @@ The program can run using just one argument, the URL of the repo:
 ```shell
 $ go get github.com/ossf/scorecard/v2
 $ scorecard --repo=github.com/kubernetes/kubernetes
-Starting [Signed-Tags]
 Starting [Automatic-Dependency-Update]
 Starting [Frozen-Deps]
 Starting [Fuzzing]
@@ -116,7 +150,6 @@ Finished [Automatic-Dependency-Update]
 Finished [Frozen-Deps]
 Finished [Fuzzing]
 Finished [Pull-Requests]
-Finished [Signed-Tags]
 Finished [Branch-Protection]
 Finished [Code-Review]
 Finished [SAST]
@@ -139,7 +172,6 @@ Pull-Requests: Pass 10
 SAST: Fail 10
 Security-Policy: Fail 5
 Signed-Releases: Fail 10
-Signed-Tags: Fail 10
 Token-Permissions: Pass 10
 ```
 
@@ -186,7 +218,6 @@ Starting [Pull-Requests]
 Starting [SAST]
 Starting [Security-Policy]
 Starting [Signed-Releases]
-Starting [Signed-Tags]
 Finished [Signed-Releases]
 Finished [Fuzzing]
 Finished [CII-Best-Practices]
@@ -197,7 +228,6 @@ Finished [SAST]
 Finished [Code-Review]
 Finished [Branch-Protection]
 Finished [Frozen-Deps]
-Finished [Signed-Tags]
 Finished [Active]
 Finished [Pull-Requests]
 Finished [Contributors]
@@ -217,7 +247,6 @@ Pull-Requests: Fail 9
 SAST: Fail 10
 Security-Policy: Pass 10
 Signed-Releases: Fail 0
-Signed-Tags: Fail 10
 ```
 
 ### Running specific checks
@@ -227,41 +256,6 @@ names.
 
 For example, `--checks=CI-Tests,Code-Review`.
 
-### Authentication
-
-Before running Scorecard, you need to, either:
-
--   [create a GitHub access token](https://docs.github.com/en/free-pro-team@latest/developers/apps/about-apps#personal-access-tokens)
-    and set it in an environment variable called `GITHUB_AUTH_TOKEN`,
-    `GITHUB_TOKEN`, `GH_AUTH_TOKEN` or `GH_TOKEN`. This helps to avoid the
-    GitHub's [api rate limits](https://developer.github.com/v3/#rate-limiting)
-    with unauthenticated requests.
-
-```shell
-# For posix platforms, e.g. linux, mac:
-export GITHUB_AUTH_TOKEN=<your access token>
-# Multiple tokens can be provided separated by comma to be utilized
-# in a round robin fashion.
-export GITHUB_AUTH_TOKEN=<your access token1>,<your access token2>
-
-# For windows:
-set GITHUB_AUTH_TOKEN=<your access token>
-set GITHUB_AUTH_TOKEN=<your access token1>,<your access token2>
-```
-
--   create a GitHub App Installations for higher rate-limit quotas. If you have
-    an installed GitHub App and key file, you can use these three environment
-    variables, following the commands shown above for your platform.
-
-```
-GITHUB_APP_KEY_PATH=<path to the key file on disk>
-GITHUB_APP_INSTALLATION_ID=<installation id>
-GITHUB_APP_ID=<app id>
-```
-
-These can be obtained from the GitHub
-[developer settings](https://github.com/settings/apps) page.
-
 ### Understanding Scorecard results
 
 Each check returns a **Pass / Fail** decision, as well as a confidence score
@@ -292,7 +286,8 @@ the [`bq`](https://cloud.google.com/bigquery/docs/bq-command-line-tool) tool:
 ```
 # Get the latest PARTITION_ID
 bq query --nouse_legacy_sql 'SELECT partition_id FROM
-openssf.scorecardcron.INFORMATION_SCHEMA.PARTITIONS ORDER BY partition_id DESC
+openssf.scorecardcron.INFORMATION_SCHEMA.PARTITIONS WHERE table_name="scorecard"
+ORDER BY partition_id DESC
 LIMIT 1'
 
 # Extract to GCS

checker/check_request.go

@@ -18,7 +18,7 @@ import (
 	"context"
 	"net/http"
 
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v38/github"
 	"github.com/shurcooL/githubv4"
 
 	"github.com/ossf/scorecard/v2/clients"

checker/check_result.go

@@ -42,18 +42,52 @@ const (
 	DetailDebug
 )
 
+// FileType is the type of a file.
+type FileType int
+
+const (
+	// FileTypeNone is a default, not defined.
+	FileTypeNone FileType = iota
+	// FileTypeSource is for source code files.
+	FileTypeSource
+	// FileTypeBinary is for binary files.
+	FileTypeBinary
+	// FileTypeText is for text files.
+	FileTypeText
+	// FileTypeURL for URLs.
+	FileTypeURL
+)
+
+// LogMessage is a structure that encapsulates detail's information.
+// This allows updating the definition easily.
+//nolint
+type LogMessage struct {
+	Text    string   // A short string explaining why the detail was recorded/logged.
+	Path    string   // Fullpath to the file.
+	Type    FileType // Type of file.
+	Offset  int      // Offset in the file of Path (line for source/text files).
+	Snippet string   // Snippet of code
+	// UPGRADEv3: to remove.
+	Version int // `3` to indicate the detail was logged using new structure.
+}
+
 // CheckDetail contains information for each detail.
-//nolint:govet
 type CheckDetail struct {
+	Msg  LogMessage
 	Type DetailType // Any of DetailWarn, DetailInfo, DetailDebug.
-	Msg  string     // A short string explaining why the details was recorded/logged..
 }
 
-// DetailLogger logs map to CheckDetail struct.
+// DetailLogger logs a CheckDetail struct.
 type DetailLogger interface {
 	Info(desc string, args ...interface{})
 	Warn(desc string, args ...interface{})
 	Debug(desc string, args ...interface{})
+
+	// Functions to use for moving to SARIF format.
+	// UPGRADEv3: to rename.
+	Info3(msg *LogMessage)
+	Warn3(msg *LogMessage)
+	Debug3(msg *LogMessage)
 }
 
 //nolint

checker/check_runner.go

@@ -49,17 +49,45 @@ type logger struct {
 }
 
 func (l *logger) Info(desc string, args ...interface{}) {
-	cd := CheckDetail{Type: DetailInfo, Msg: fmt.Sprintf(desc, args...)}
+	cd := CheckDetail{Type: DetailInfo, Msg: LogMessage{Text: fmt.Sprintf(desc, args...)}}
 	l.messages2 = append(l.messages2, cd)
 }
 
 func (l *logger) Warn(desc string, args ...interface{}) {
-	cd := CheckDetail{Type: DetailWarn, Msg: fmt.Sprintf(desc, args...)}
+	cd := CheckDetail{Type: DetailWarn, Msg: LogMessage{Text: fmt.Sprintf(desc, args...)}}
 	l.messages2 = append(l.messages2, cd)
 }
 
 func (l *logger) Debug(desc string, args ...interface{}) {
-	cd := CheckDetail{Type: DetailDebug, Msg: fmt.Sprintf(desc, args...)}
+	cd := CheckDetail{Type: DetailDebug, Msg: LogMessage{Text: fmt.Sprintf(desc, args...)}}
+	l.messages2 = append(l.messages2, cd)
+}
+
+// UPGRADEv3: to rename.
+func (l *logger) Info3(msg *LogMessage) {
+	cd := CheckDetail{
+		Type: DetailInfo,
+		Msg:  *msg,
+	}
+	cd.Msg.Version = 3
+	l.messages2 = append(l.messages2, cd)
+}
+
+func (l *logger) Warn3(msg *LogMessage) {
+	cd := CheckDetail{
+		Type: DetailWarn,
+		Msg:  *msg,
+	}
+	cd.Msg.Version = 3
+	l.messages2 = append(l.messages2, cd)
+}
+
+func (l *logger) Debug3(msg *LogMessage) {
+	cd := CheckDetail{
+		Type: DetailDebug,
+		Msg:  *msg,
+	}
+	cd.Msg.Version = 3
 	l.messages2 = append(l.messages2, cd)
 }
 
@@ -100,8 +128,11 @@ func (r *Runner) Run(ctx context.Context, f CheckFn) CheckResult {
 		}
 		break
 	}
-	res.Details2 = l.messages2
 
+	res.Details2 = l.messages2
+	for _, d := range l.messages2 {
+		res.Details = append(res.Details, d.Msg.Text)
+	}
 	if err := logStats(ctx, startTime, &res); err != nil {
 		panic(err)
 	}