5348 sonar qube pr problem

cmd/kosli/attestSonar.go

@@ -33,19 +33,25 @@ const attestSonarShortDesc = `Report a SonarQube attestation to an artifact or a
 const attestSonarLongDesc = attestSonarShortDesc + `
 Retrieves results for the specified scan from SonarQube Cloud or SonarQube Server and attests them to Kosli.
 The results are parsed to find the status of the project's quality gate which is used to determine the attestation's compliance status.
+Both branch scans and pull request scans are supported.
 
-The scan to be retrieved can be specified in two ways:
+The scan to be retrieved can be specified in three ways:
 1. (Default) Using metadata created by the Sonar scanner. By default this is located within a temporary ^.scannerwork^ folder in the repo base directory.
 If you have overriden the location of this folder by passing parameters to the Sonar scanner, or are running Kosli's CLI locally outside the repo's base directory,
-you can provide the correct path using the ^--sonar-working-dir^ flag. This metadata is generated by a specific scan, allowing Kosli to retrieve the results of that scan.  
-If there are delays in the scan processing (either because the scanned project is very large, or because SonarQube is experiencing processing delays), it may happen that 
-the scan results are not available by the time the attest sonar command is executed. In this case you can use the ^--max-wait^ flag to retry the command while waiting for the scan to be processed. 
-This flag takes the maximum number of seconds to wait for the results to be available. The Kosli CLI will then attempt to retrieve the scan results until the maximum wait time is reached, with 
+you can provide the correct path using the ^--sonar-working-dir^ flag. This metadata is generated by a specific scan, allowing Kosli to retrieve the results of that scan.
+If there are delays in the scan processing (either because the scanned project is very large, or because SonarQube is experiencing processing delays), it may happen that
+the scan results are not available by the time the attest sonar command is executed. In this case you can use the ^--max-wait^ flag to retry the command while waiting for the scan to be processed.
+This flag takes the maximum number of seconds to wait for the results to be available. The Kosli CLI will then attempt to retrieve the scan results until the maximum wait time is reached, with
 exponential backoff between retries. Once the results are available they are attested to Kosli as usual.
 
 2. Providing the Sonar project key and the revision of the scan (plus the SonarQube server URL if relevant). If running the Kosli CLI in some CI/CD pipeline, the revision
 is defaulted to the commit SHA. If you are running the command locally, or have overriden the revision in SonarQube via parameters to the Sonar scanner, you can
 provide the correct revision using the ^--sonar-revision^ flag. Kosli then finds the scan results for the specified project key and revision.
+Note that this method does not support pull request scans; use method 1 or 3 instead.
+
+3. Providing the CE task URL directly via ^--sonar-ce-task-url^. The CE task URL can be found in the ^report-task.txt^ file
+generated by the Sonar scanner (the ^ceTaskUrl^ field). This is useful in CI/CD environments where the Sonar scanner and the Kosli CLI
+run in different containers that do not share a filesystem, making the ^report-task.txt^ file inaccessible to the CLI.
 
 Note that if your project is very large and you are using SonarQube Cloud's automatic analysis, it is possible for the attest sonar command to run before the SonarQube Cloud scan is completed.
 In this case, we recommend using Kosli's Sonar webhook integration ( https://docs.kosli.com/integrations/sonar/ ) rather than the CLI to attest the scan results.
@@ -107,6 +113,16 @@ kosli attest sonar \
 	--api-token yourAPIToken \
 	--org yourOrgName \
 	--max-wait 300
+
+# report a SonarQube attestation using the CE task URL directly (useful when report-task.txt is not accessible):
+kosli attest sonar \
+	--name yourAttestationName \
+	--flow yourFlowName \
+	--trail yourTrailName \
+	--sonar-api-token yourSonarAPIToken \
+	--sonar-ce-task-url yourCETaskURL \
+	--api-token yourAPIToken \
+	--org yourOrgName
 `
 
 func newAttestSonarCmd(out io.Writer) *cobra.Command {
@@ -163,6 +179,7 @@ func newAttestSonarCmd(out io.Writer) *cobra.Command {
 	cmd.Flags().StringVar(&o.projectKey, "sonar-project-key", "", sonarProjectKeyFlag)
 	cmd.Flags().StringVar(&o.serverURL, "sonar-server-url", "https://sonarcloud.io", sonarServerURLFlag)
 	cmd.Flags().StringVar(&o.revision, "sonar-revision", o.commitSHA, sonarRevisionFlag)
+	cmd.Flags().StringVar(&o.ceTaskURL, "sonar-ce-task-url", "", sonarCETaskURLFlag)
 	cmd.Flags().IntVar(&o.maxWait, "max-wait", 30, sonarMaxWaitFlag)
 
 	err := RequireFlags(cmd, []string{"flow", "trail", "name", "sonar-api-token"})

cmd/kosli/attestSonar_test.go

@@ -155,6 +155,16 @@ func (suite *AttestSonarCommandTestSuite) TestAttestSonarCmd() {
 			cmd:    fmt.Sprintf("attest sonar --name cli.foo --commit HEAD --origin-url http://www.example.com --sonar-working-dir testdata/sonar/sonarcloud/.scannerwork --sonar-project-key anyKey --sonar-revision anyRevision --sonar-server-url http://example.com %s", suite.defaultKosliArguments),
 			golden: "sonar attestation 'foo' is reported to trail: test-123\n",
 		},
+		{
+			name:   "can attest sonar for a pull request scan using report-task.txt",
+			cmd:    fmt.Sprintf("attest sonar --name cli.foo --commit HEAD --origin-url http://www.example.com --sonar-working-dir testdata/sonar/sonarcloud/.scannerwork-pr %s", suite.defaultKosliArguments),
+			golden: "sonar attestation 'foo' is reported to trail: test-123\n",
+		},
+		{
+			name:   "can attest sonar using --sonar-ce-task-url without report-task.txt",
+			cmd:    fmt.Sprintf("attest sonar --name cli.foo --commit HEAD --origin-url http://www.example.com --sonar-ce-task-url https://sonarcloud.io/api/ce/task?id=AZrs5eywBfkZKeU0sde9 %s", suite.defaultKosliArguments),
+			golden: "sonar attestation 'foo' is reported to trail: test-123\n",
+		},
 		{
 			wantError: true,
 			name:      "if outdated task given (i.e. we try to get results for an older scan that SonarCloud has deleted), we get an error",

cmd/kosli/root.go

@@ -241,6 +241,7 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file,
 	sonarServerURLFlag              = "[conditional] The URL of your SonarQube server. Only required if you are using SonarQube and not using SonarQube's metadata file to get scan results."
 	sonarRevisionFlag               = "[conditional] The revision of the SonarCloud/SonarQube project. Only required if you want to use the project key/revision to get the scan results rather than using Sonar's metadata file and you have overridden the default revision, or you aren't using a CI. Defaults to the value of the git commit flag."
 	sonarMaxWaitFlag                = "[optional] Allow the command to wait and retry fetching the scan results from SonarQube, up to the maximum number of seconds provided, with exponential backoff. Useful when using SonarQube's metadata file to retrieve and attest scans that take a long time to process . Defaults to 30 seconds."
+	sonarCETaskURLFlag              = "[conditional] The URL of the SonarQube CE task. Can be used instead of --sonar-working-dir when the report-task.txt file is not accessible, e.g. due to container isolation in CI/CD pipelines."
 	logicalEnvFlag                  = "[required] The logical environment."
 	physicalEnvFlag                 = "[required] The physical environment."
 	attestationTypeDescriptionFlag  = "[optional] The attestation type description."

cmd/kosli/testdata/sonar/sonarcloud/.scannerwork-pr/report-task.txt

@@ -0,0 +1,7 @@
+organization=cyber-dojo
+projectKey=cyber-dojo_differ
+serverUrl=https://sonarcloud.io
+serverVersion=8.0.0.56246
+dashboardUrl=https://sonarcloud.io/dashboard?id=cyber-dojo_differ&pullRequest=359
+ceTaskId=AZ2Qge89T7Y829rQbv87
+ceTaskUrl=https://sonarcloud.io/api/ce/task?id=AZ2Qge89T7Y829rQbv87

cmd/kosli/testdata/sonar/update-sonarqube-test-data.txt

@@ -37,6 +37,23 @@ After 5 years all snapshots are deleted, including snapshots marked by version e
 7. In report-task.txt, replace the value of ceTaskId with the ID you just copied.
    - Also replace the id at the end of the ceTaskUrl with that ID.
 
+## Instructions for updating PR scan test data in .scannerwork-pr/report-task.txt
+
+The PR scan fixture works the same way as the main branch fixture, but the CE task must be from a pull request scan
+(i.e. the CE task API response will have a `pullRequest` field instead of `branch`).
+
+1. Find a recent pull request analysis for the project. You can use the project_pull_requests/list API:
+   - The endpoint is https://sonarcloud.io/api/project_pull_requests/list?project=cyber-dojo_differ
+   - This returns a JSON object with an array of pull requests, each with a key, analysisDate, and commit SHA.
+2. Choose a pull request that has a recent analysis.
+3. Find the CE task for that PR analysis using the CE activity API:
+   - The endpoint is https://sonarcloud.io/api/ce/activity?component=cyber-dojo_differ
+   - Look for a task with a `pullRequest` field matching the PR key from step 2.
+4. Copy the task ID and update .scannerwork-pr/report-task.txt:
+   - Replace the ceTaskId value with the new task ID.
+   - Replace the id at the end of the ceTaskUrl with the same ID.
+   - Update the pullRequest parameter in the dashboardUrl if the PR number has changed.
+
 ## Instructions for updating test data passed as CLI command arguments
 
 Instead of using the report-task.txt file, some of our sonar tests take the key of the SonarQube project and a git commit as arguments to the CLI command, which are then used to find the relevant analysis in SonarQube. This should rarely need to be updated, since once an analysis snapshot is 4 weeks old in SonarQube, it will remain in the database until at least 2 years.

internal/sonar/sonar.go

@@ -33,6 +33,7 @@ type SonarResults struct {
 	Revision    string       `json:"revision"`
 	Project     Project      `json:"project"`
 	Branch      *Branch      `json:"branch,omitempty"`
+	PullRequest *PullRequest `json:"pullRequest,omitempty"`
 	QualityGate *QualityGate `json:"qualityGate,omitempty"`
 }
 
@@ -47,6 +48,10 @@ type Branch struct {
 	Type string `json:"type,omitempty"`
 }
 
+type PullRequest struct {
+	Key string `json:"key"`
+}
+
 type QualityGate struct {
 	Status     string      `json:"status"`
 	Conditions []Condition `json:"conditions"`
@@ -92,6 +97,7 @@ type Task struct {
 	Status        string `json:"status"`
 	Branch        string `json:"branch"`
 	BranchType    string `json:"branchType"`
+	PullRequest   string `json:"pullRequest"`
 }
 
 type ActivityResponse struct {
@@ -110,6 +116,22 @@ type Analysis struct {
 	Revision string `json:"revision"`
 }
 
+// These are the structs for the response from the project_pull_requests/list API
+type PullRequestsResponse struct {
+	PullRequests []PullRequestInfo `json:"pullRequests"`
+	Errors       []Error           `json:"errors,omitempty"`
+}
+
+type PullRequestInfo struct {
+	Key          string   `json:"key"`
+	AnalysisDate string   `json:"analysisDate"`
+	Commit       PRCommit `json:"commit"`
+}
+
+type PRCommit struct {
+	SHA string `json:"sha"`
+}
+
 // Struct for error messages from sonar APIs
 type Error struct {
 	Msg string `json:"msg"`
@@ -152,10 +174,18 @@ func (sc *SonarConfig) GetSonarResults(logger *log.Logger) (*SonarResults, error
 		return nil, fmt.Errorf("API token must be given to retrieve data from SonarQube")
 	}
 
-	// Read the report-task.txt file (if it exists) to get the project key, server URL, dashboard URL and ceTaskURL
+	// Read the report-task.txt file (if it exists) to get the server URL, dashboard URL and ceTaskURL
 	err = sc.readFile(project, sonarResults, logger)
 	if err != nil {
-		if sc.projectKey == "" || sc.revision == "" {
+		if sc.CETaskUrl != "" {
+			// If the CE task URL is provided directly (e.g. via --sonar-ce-task-url), we can skip the report-task.txt
+			// and use the CE task URL to get the data. Extract the server URL from the CE task URL.
+			parsedURL, parseErr := url.Parse(sc.CETaskUrl)
+			if parseErr != nil {
+				return nil, fmt.Errorf("failed to parse CE task URL: %s", parseErr)
+			}
+			sonarResults.ServerUrl = fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host)
+		} else if sc.projectKey == "" || sc.revision == "" {
 			return nil, fmt.Errorf("%s. Alternatively provide the project key and revision for the scan to attest", err)
 			// If the report-task.txt does not exist, but we've been given the project key and revision, we can still get the data
 		} else {
@@ -184,10 +214,19 @@ func (sc *SonarConfig) GetSonarResults(logger *log.Logger) (*SonarResults, error
 			return nil, err
 		}
 
-		//Get project revision and scan date/time from the projectAnalyses API
-		err = GetProjectAnalysisFromAnalysisID(httpClient, sonarResults, project, analysisID, tokenHeader)
-		if err != nil {
-			return nil, err
+		if sonarResults.PullRequest != nil {
+			// For PR scans, project_analyses/search does not return PR analyses on SonarCloud.
+			// Use the project_pull_requests/list API to get the revision and analysis date instead.
+			err = GetPRAnalysisData(httpClient, sonarResults, project, sonarResults.PullRequest.Key, tokenHeader)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			//Get project revision and scan date/time from the projectAnalyses API
+			err = GetProjectAnalysisFromAnalysisID(httpClient, sonarResults, project, analysisID, tokenHeader)
+			if err != nil {
+				return nil, err
+			}
 		}
 	}
 
@@ -237,10 +276,10 @@ func (sc *SonarConfig) readFile(project *Project, results *SonarResults, logger
 
 func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *SonarResults, ceTaskURL, tokenHeader string, maxWait int, logger *log.Logger) (string, error) {
 	taskRequest, err := http.NewRequest("GET", ceTaskURL, nil)
-	taskRequest.Header.Add("Authorization", tokenHeader)
 	if err != nil {
 		return "", err
 	}
+	taskRequest.Header.Add("Authorization", tokenHeader)
 
 	wait := 1    // start wait period
 	retries := 0 // number of retries so far
@@ -312,7 +351,10 @@ func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *Sona
 		}
 	}
 
-	if taskResponseData.Task.Branch != "" {
+	if taskResponseData.Task.PullRequest != "" {
+		sonarResults.PullRequest = &PullRequest{Key: taskResponseData.Task.PullRequest}
+		sonarResults.Branch = nil
+	} else if taskResponseData.Task.Branch != "" {
 		sonarResults.Branch = &Branch{}
 		sonarResults.Branch.Name = taskResponseData.Task.Branch
 		sonarResults.Branch.Type = taskResponseData.Task.BranchType
@@ -331,10 +373,10 @@ func GetProjectAnalysisFromRevision(httpClient *http.Client, sonarResults *Sonar
 		return "", err
 	}
 	projectAnalysesRequest, err := http.NewRequest("GET", projectAnalysesURL, nil)
-	projectAnalysesRequest.Header.Add("Authorization", tokenHeader)
 	if err != nil {
 		return "", err
 	}
+	projectAnalysesRequest.Header.Add("Authorization", tokenHeader)
 
 	projectAnalysesResponse, err := httpClient.Do(projectAnalysesRequest)
 	if err != nil {
@@ -376,22 +418,27 @@ func GetProjectAnalysisFromAnalysisID(httpClient *http.Client, sonarResults *Son
 		return err
 	}
 	projectAnalysesRequest, err := http.NewRequest("GET", projectAnalysesURL, nil)
-	projectAnalysesRequest.Header.Add("Authorization", tokenHeader)
 	if err != nil {
 		return err
 	}
+	projectAnalysesRequest.Header.Add("Authorization", tokenHeader)
 
 	projectAnalysesResponse, err := httpClient.Do(projectAnalysesRequest)
 	if err != nil {
 		return err
 	}
+	defer func() { _ = projectAnalysesResponse.Body.Close() }()
 
 	projectAnalysesData := &ProjectAnalyses{}
 	err = json.NewDecoder(projectAnalysesResponse.Body).Decode(projectAnalysesData)
 	if err != nil {
 		return fmt.Errorf("please check your API token is correct and you have the correct permissions in SonarQube")
 	}
 
+	if projectAnalysesData.Errors != nil {
+		return fmt.Errorf("SonarQube error: %s", projectAnalysesData.Errors[0].Msg)
+	}
+
 	for analysis := range projectAnalysesData.Analyses {
 		if projectAnalysesData.Analyses[analysis].Key == analysisID {
 			sonarResults.AnalaysedAt = projectAnalysesData.Analyses[analysis].Date
@@ -400,27 +447,64 @@ func GetProjectAnalysisFromAnalysisID(httpClient *http.Client, sonarResults *Son
 		}
 	}
 
-	if projectAnalysesData.Errors != nil {
-		return fmt.Errorf("SonarQube error: %s", projectAnalysesData.Errors[0].Msg)
-	}
-
 	if sonarResults.AnalaysedAt == "" {
 		return fmt.Errorf("analysis with ID %s not found on %s. Snapshot may have been deleted by SonarQube", analysisID, sonarResults.ServerUrl)
 	}
 
 	return nil
 }
 
+// GetPRAnalysisData retrieves the revision and analysis date for a pull request scan
+// from the project_pull_requests/list API. This is needed because the project_analyses/search
+// API does not return PR analyses on SonarCloud.
+func GetPRAnalysisData(httpClient *http.Client, sonarResults *SonarResults, project *Project, prKey, tokenHeader string) error {
+	prURL, err := sonarURL(sonarResults.ServerUrl, "api/project_pull_requests/list", url.Values{"project": {project.Key}})
+	if err != nil {
+		return err
+	}
+	prRequest, err := http.NewRequest("GET", prURL, nil)
+	if err != nil {
+		return err
+	}
+	prRequest.Header.Add("Authorization", tokenHeader)
+
+	prResponse, err := httpClient.Do(prRequest)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = prResponse.Body.Close() }()
+
+	prData := &PullRequestsResponse{}
+	err = json.NewDecoder(prResponse.Body).Decode(prData)
+	if err != nil {
+		return fmt.Errorf("please check your API token is correct and you have the correct permissions in SonarQube")
+	}
+
+	if prData.Errors != nil {
+		return fmt.Errorf("SonarQube error: %s", prData.Errors[0].Msg)
+	}
+
+	for _, pr := range prData.PullRequests {
+		if pr.Key == prKey {
+			sonarResults.AnalaysedAt = pr.AnalysisDate
+			sonarResults.Revision = pr.Commit.SHA
+			return nil
+		}
+	}
+
+	return fmt.Errorf("pull request %s not found for project %s on %s", prKey, project.Key, sonarResults.ServerUrl)
+}
+
 func GetQualityGate(httpClient *http.Client, sonarResults *SonarResults, qualityGate *QualityGate, analysisID, tokenHeader string) (*QualityGate, error) {
 	qualityGateURL, err := sonarURL(sonarResults.ServerUrl, "api/qualitygates/project_status", url.Values{"analysisId": {analysisID}})
 	if err != nil {
 		return nil, err
 	}
 	qualityGateRequest, err := http.NewRequest("GET", qualityGateURL, nil)
-	qualityGateRequest.Header.Add("Authorization", tokenHeader)
 	if err != nil {
 		return nil, err
 	}
+	qualityGateRequest.Header.Add("Authorization", tokenHeader)
 
 	qualityGateResponse, err := httpClient.Do(qualityGateRequest)
 	if err != nil {
@@ -460,10 +544,10 @@ func GetTaskID(httpClient *http.Client, sonarResults *SonarResults, project *Pro
 		return err
 	}
 	CEActivityRequest, err := http.NewRequest("GET", CEActivityURL, nil)
-	CEActivityRequest.Header.Add("Authorization", tokenHeader)
 	if err != nil {
 		return err
 	}
+	CEActivityRequest.Header.Add("Authorization", tokenHeader)
 
 	CEActivityResponse, err := httpClient.Do(CEActivityRequest)
 	if err != nil {