Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add operations for container scan to status.json #960

Merged
merged 3 commits into from
Oct 5, 2023
Merged

Add operations for container scan to status.json #960

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2800c2e
fix: Report container scan image and metadata uploads as Detect opera…
niravrsynopsys Oct 3, 2023
ff7bd53
Return empty optional on exception, not optional of scan ID
niravrsynopsys Oct 3, 2023
38fa162
Report bdio header upload operation
niravrsynopsys Oct 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 69 additions & 59 deletions src/main/java/com/synopsys/integration/detect/lifecycle/run/operation/OperationRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,8 @@ public UUID initiateStatelessBdbaScan(BlackDuckRunData blackDuckRunData) throws
);
}

return uploadBdioHeaderToInitiateScan(blackDuckRunData, bdioHeader);
String uploadHeaderOperationName = "Upload BDIO Header to Initiate Scan";
return uploadBdioHeaderToInitiateScan(blackDuckRunData, bdioHeader, uploadHeaderOperationName);
});
}

Expand Down Expand Up @@ -390,18 +391,20 @@ public File downloadContainerImage(Gson gson, File downloadDirectory, String con
return artifactResolver.downloadArtifact(new File(targetPathName), containerImageUri);
}

public File getContainerScanImage(Gson gson, File downloadDirectory) throws IntegrationException, IOException, DetectUserFriendlyException {
Optional<String> containerImageFilePath = getContainerScanFilePath();
File containerImageFile = null;
if (containerImageFilePath.isPresent()) {
String containerImageUri = containerImageFilePath.get();
if (containerImageUri.startsWith("http")) {
containerImageFile = downloadContainerImage(gson, downloadDirectory, containerImageUri);
} else {
containerImageFile = new File(containerImageUri);
public File getContainerScanImage(Gson gson, File downloadDirectory) throws OperationException {
return auditLog.namedPublic("Retrieve Container Scan Image File", () -> {
Optional<String> containerImageFilePath = getContainerScanFilePath();
File containerImageFile = null;
if (containerImageFilePath.isPresent()) {
niravrsynopsys marked this conversation as resolved.
Show resolved Hide resolved
String containerImageUri = containerImageFilePath.get();
if (containerImageUri.startsWith("http")) {
containerImageFile = downloadContainerImage(gson, downloadDirectory, containerImageUri);
} else {
containerImageFile = new File(containerImageUri);
}
}
}
return containerImageFile;
return containerImageFile;
});
}

public JsonObject createContainerScanImageMetadata(UUID scanId, NameVersion projectNameVersion) {
Expand All @@ -419,66 +422,73 @@ public JsonObject createContainerScanImageMetadata(UUID scanId, NameVersion proj
return imageMetadataObject;
}

public Response uploadFileToStorageService(BlackDuckRunData blackDuckRunData, String storageServiceEndpoint, File payloadFile, String postContentType)
throws IntegrationException, IOException {
BlackDuckServicesFactory blackDuckServicesFactory = blackDuckRunData.getBlackDuckServicesFactory();
BlackDuckApiClient blackDuckApiClient = blackDuckServicesFactory.getBlackDuckApiClient();
// Generic method to POST a file to /api/storage/containers endpoint of storage service
public Response uploadFileToStorageService(BlackDuckRunData blackDuckRunData, String storageServiceEndpoint, File payloadFile, String postContentType, String operationName)
throws OperationException {
return auditLog.namedPublic(operationName, () -> {
BlackDuckServicesFactory blackDuckServicesFactory = blackDuckRunData.getBlackDuckServicesFactory();
BlackDuckApiClient blackDuckApiClient = blackDuckServicesFactory.getBlackDuckApiClient();

HttpUrl postUrl = blackDuckRunData.getBlackDuckServerConfig().getBlackDuckUrl().appendRelativeUrl(storageServiceEndpoint);
BlackDuckResponseRequest buildBlackDuckResponseRequest = new BlackDuckRequestBuilder()
.postFile(payloadFile, ContentType.create(postContentType))
.buildBlackDuckResponseRequest(postUrl);
HttpUrl postUrl = blackDuckRunData.getBlackDuckServerConfig().getBlackDuckUrl().appendRelativeUrl(storageServiceEndpoint);
BlackDuckResponseRequest buildBlackDuckResponseRequest = new BlackDuckRequestBuilder()
.postFile(payloadFile, ContentType.create(postContentType))
.buildBlackDuckResponseRequest(postUrl);

try (Response response = blackDuckApiClient.execute(buildBlackDuckResponseRequest)) {
return response;
} catch (IntegrationException e) {
logger.trace("Could not execute file upload request to storage service.");
throw new IntegrationException("Could not execute file upload request to storage service.", e);
} catch (IOException e) {
logger.trace("I/O error occurred during file upload request.");
throw new IOException("I/O error occurred during file upload request to storage service.", e);
}
try (Response response = blackDuckApiClient.execute(buildBlackDuckResponseRequest)) {
return response;
} catch (IntegrationException e) {
logger.trace("Could not execute file upload request to storage service.");
throw new IntegrationException("Could not execute file upload request to storage service.", e);
} catch (IOException e) {
logger.trace("I/O error occurred during file upload request.");
throw new IOException("I/O error occurred during file upload request to storage service.", e);
}
});
}

public Response uploadJsonToStorageService(BlackDuckRunData blackDuckRunData, String storageServiceEndpoint, String jsonPayload, String postContentType)
throws IntegrationException, IOException {
BlackDuckServicesFactory blackDuckServicesFactory = blackDuckRunData.getBlackDuckServicesFactory();
BlackDuckApiClient blackDuckApiClient = blackDuckServicesFactory.getBlackDuckApiClient();
public Response uploadJsonToStorageService(BlackDuckRunData blackDuckRunData, String storageServiceEndpoint, String jsonPayload, String postContentType, String operationName)
throws OperationException {

HttpUrl postUrl = blackDuckRunData.getBlackDuckServerConfig().getBlackDuckUrl().appendRelativeUrl(storageServiceEndpoint);
return auditLog.namedInternal(operationName, () -> {
BlackDuckServicesFactory blackDuckServicesFactory = blackDuckRunData.getBlackDuckServicesFactory();
BlackDuckApiClient blackDuckApiClient = blackDuckServicesFactory.getBlackDuckApiClient();

BlackDuckResponseRequest buildBlackDuckResponseRequest = new BlackDuckRequestBuilder()
.postString(jsonPayload, ContentType.create(postContentType))
.buildBlackDuckResponseRequest(postUrl);
HttpUrl postUrl = blackDuckRunData.getBlackDuckServerConfig().getBlackDuckUrl().appendRelativeUrl(storageServiceEndpoint);

try (Response response = blackDuckApiClient.execute(buildBlackDuckResponseRequest)) {
return response;
} catch (IntegrationException e) {
logger.trace("Could not execute JSON upload request to storage service.");
throw new IntegrationException("Could not execute JSON upload request to storage service.", e);
} catch (IOException e) {
logger.trace("I/O error occurred during JSON upload request.");
throw new IOException("I/O error occurred during JSON upload request to storage service.", e);
}
}
BlackDuckResponseRequest buildBlackDuckResponseRequest = new BlackDuckRequestBuilder()
.postString(jsonPayload, ContentType.create(postContentType))
.buildBlackDuckResponseRequest(postUrl);

try (Response response = blackDuckApiClient.execute(buildBlackDuckResponseRequest)) {
return response;
} catch (IntegrationException e) {
logger.trace("Could not execute JSON upload request to storage service.");
throw new IntegrationException("Could not execute JSON upload request to storage service.", e);
} catch (IOException e) {
logger.trace("I/O error occurred during JSON upload request.");
throw new IOException("I/O error occurred during JSON upload request to storage service.", e);
}
});
}

public UUID uploadBdioHeaderToInitiateScan(BlackDuckRunData blackDuckRunData, File bdioHeaderFile) throws IntegrationException {
BlackDuckServicesFactory blackDuckServicesFactory = blackDuckRunData.getBlackDuckServicesFactory();
BlackDuckApiClient blackDuckApiClient = blackDuckServicesFactory.getBlackDuckApiClient();
public UUID uploadBdioHeaderToInitiateScan(BlackDuckRunData blackDuckRunData, File bdioHeaderFile, String operationName) throws OperationException {
return auditLog.namedInternal(operationName, () -> {
BlackDuckServicesFactory blackDuckServicesFactory = blackDuckRunData.getBlackDuckServicesFactory();
BlackDuckApiClient blackDuckApiClient = blackDuckServicesFactory.getBlackDuckApiClient();

String scanServicePostEndpoint = getScanServicePostEndpoint();
HttpUrl postUrl = blackDuckRunData.getBlackDuckServerConfig().getBlackDuckUrl().appendRelativeUrl(scanServicePostEndpoint);
String scanServicePostEndpoint = getScanServicePostEndpoint();
HttpUrl postUrl = blackDuckRunData.getBlackDuckServerConfig().getBlackDuckUrl().appendRelativeUrl(scanServicePostEndpoint);

String scanServicePostContentType = getScanServicePostContentType();
BlackDuckResponseRequest buildBlackDuckResponseRequest = new BlackDuckRequestBuilder()
.postFile(bdioHeaderFile, ContentType.create(scanServicePostContentType))
.buildBlackDuckResponseRequest(postUrl);
String scanServicePostContentType = getScanServicePostContentType();
BlackDuckResponseRequest buildBlackDuckResponseRequest = new BlackDuckRequestBuilder()
.postFile(bdioHeaderFile, ContentType.create(scanServicePostContentType))
.buildBlackDuckResponseRequest(postUrl);

HttpUrl responseUrl = blackDuckApiClient.executePostRequestAndRetrieveURL(buildBlackDuckResponseRequest);
String path = responseUrl.uri().getPath();
HttpUrl responseUrl = blackDuckApiClient.executePostRequestAndRetrieveURL(buildBlackDuckResponseRequest);
String path = responseUrl.uri().getPath();

return UUID.fromString(path.substring(path.lastIndexOf('/') + 1));
return UUID.fromString(path.substring(path.lastIndexOf('/') + 1));
});
}

public void uploadBdioEntries(BlackDuckRunData blackDuckRunData, UUID bdScanId) throws IntegrationException, IOException {
Expand Down
23 changes: 15 additions & 8 deletions ...main/java/com/synopsys/integration/detect/lifecycle/run/step/ContainerScanStepRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import com.google.gson.Gson;
import com.synopsys.integration.blackduck.version.BlackDuckVersion;
import com.synopsys.integration.detect.configuration.DetectUserFriendlyException;
import com.synopsys.integration.detect.lifecycle.OperationException;
import com.synopsys.integration.detect.lifecycle.run.data.BlackDuckRunData;
import com.synopsys.integration.detect.lifecycle.run.operation.OperationRunner;
import com.synopsys.integration.detect.util.bdio.protobuf.DetectProtobufBdioHeaderUtil;
Expand All @@ -37,7 +38,7 @@ public class ContainerScanStepRunner {
private static final String STORAGE_IMAGE_METADATA_CONTENT_TYPE = "application/vnd.blackducksoftware.container-scan-message-1+json";

public ContainerScanStepRunner(OperationRunner operationRunner, NameVersion projectNameVersion, BlackDuckRunData blackDuckRunData, Gson gson)
throws IntegrationException, DetectUserFriendlyException, IOException {
throws IntegrationException, OperationException {
this.operationRunner = operationRunner;
this.projectNameVersion = projectNameVersion;
this.blackDuckRunData = blackDuckRunData;
Expand Down Expand Up @@ -74,8 +75,9 @@ public Optional<UUID> invokeContainerScanningWorkflow() {
uploadImageMetadataToStorageService();
operationRunner.publishContainerSuccess();
logger.info("Container scan image uploaded successfully.");
} catch (IntegrationException | IOException e) {
} catch (IntegrationException | IOException | OperationException e) {
operationRunner.publishContainerFailure(e);
return Optional.empty();
}
return Optional.ofNullable(scanId);
}
Expand All @@ -98,15 +100,16 @@ private String getContainerScanCodeLocationName() {
return codeLocationNameManager.createContainerScanCodeLocationName(containerImage, projectNameVersion.getName(), projectNameVersion.getVersion());
}

public void initiateScan() throws IOException, IntegrationException {
public void initiateScan() throws IOException, IntegrationException, OperationException {
DetectProtobufBdioHeaderUtil detectProtobufBdioHeaderUtil = new DetectProtobufBdioHeaderUtil(
UUID.randomUUID().toString(),
"CONTAINER",
projectNameVersion,
projectGroupName,
getContainerScanCodeLocationName());
File bdioHeaderFile = detectProtobufBdioHeaderUtil.createProtobufBdioHeader(binaryRunDirectory);
scanId = operationRunner.uploadBdioHeaderToInitiateScan(blackDuckRunData, bdioHeaderFile);
String operationName = "Upload Container Scan BDIO Header to Initiate Scan";
scanId = operationRunner.uploadBdioHeaderToInitiateScan(blackDuckRunData, bdioHeaderFile, operationName);
if (scanId == null) {
logger.warn("Scan ID was not found in the response from the server.");
throw new IntegrationException("Scan ID was not found in the response from the server.");
Expand All @@ -115,15 +118,17 @@ public void initiateScan() throws IOException, IntegrationException {
logger.debug("Scan initiated with scan service. Scan ID received: {}", scanIdString);
}

public void uploadImageToStorageService() throws IntegrationException, IOException {
public void uploadImageToStorageService() throws IOException, IntegrationException, OperationException {
String storageServiceEndpoint = String.join("", STORAGE_CONTAINERS_ENDPOINT, scanId.toString());
String operationName = "Upload Container Scan Image";
logger.debug("Uploading container image artifact to storage endpoint: {}", storageServiceEndpoint);

try (Response response = operationRunner.uploadFileToStorageService(
blackDuckRunData,
storageServiceEndpoint,
containerImage,
STORAGE_IMAGE_CONTENT_TYPE
STORAGE_IMAGE_CONTENT_TYPE,
operationName
)
) {
if (response.isStatusCodeSuccess()) {
Expand All @@ -135,8 +140,9 @@ public void uploadImageToStorageService() throws IntegrationException, IOExcepti
}
}

public void uploadImageMetadataToStorageService() throws IntegrationException, IOException {
public void uploadImageMetadataToStorageService() throws IntegrationException, IOException, OperationException {
String storageServiceEndpoint = String.join("", STORAGE_CONTAINERS_ENDPOINT, scanId.toString(), "/message");
String operationName = "Upload Container Scan Image Metadata JSON";
logger.debug("Uploading container image metadata to storage endpoint: {}", storageServiceEndpoint);

JsonObject imageMetadataObject = operationRunner.createContainerScanImageMetadata(scanId, projectNameVersion);
Expand All @@ -145,7 +151,8 @@ public void uploadImageMetadataToStorageService() throws IntegrationException, I
blackDuckRunData,
storageServiceEndpoint,
imageMetadataObject.toString(),
STORAGE_IMAGE_METADATA_CONTENT_TYPE
STORAGE_IMAGE_METADATA_CONTENT_TYPE,
operationName
)
) {
if (response.isStatusCodeSuccess()) {
Expand Down
8 changes: 5 additions & 3 deletions ...synopsys/integration/detect/lifecycle/run/operation/OperationRunnerContainerScanTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.google.gson.JsonObject;
import com.synopsys.integration.detect.configuration.DetectUserFriendlyException;
import com.synopsys.integration.detect.configuration.enumeration.BlackduckScanMode;
import com.synopsys.integration.detect.lifecycle.OperationException;
import com.synopsys.integration.detect.testutils.ContainerScanTestUtils;
import com.synopsys.integration.exception.IntegrationException;
import com.synopsys.integration.util.NameVersion;
Expand All @@ -21,15 +22,16 @@ public class OperationRunnerContainerScanTest {
private static final Gson gson = new Gson();
private static final ContainerScanTestUtils containerScanTestUtils = new ContainerScanTestUtils();

private File updateDetectConfigAndGetContainerImage(BlackduckScanMode blackduckScanMode, String imageFilePath) throws IntegrationException, IOException, DetectUserFriendlyException {
private File updateDetectConfigAndGetContainerImage(BlackduckScanMode blackduckScanMode, String imageFilePath)
throws IntegrationException, IOException, DetectUserFriendlyException, OperationException {
OperationRunner operationRunner = containerScanTestUtils.setUpDetectConfig(blackduckScanMode, imageFilePath);
OperationRunner operationRunnerSpy = Mockito.spy(operationRunner);
Mockito.doReturn(ContainerScanTestUtils.TEST_IMAGE_DOWNLOADED_FILE).when(operationRunnerSpy).downloadContainerImage(gson, ContainerScanTestUtils.TEST_DOWNLOAD_DIRECTORY, imageFilePath);
return operationRunnerSpy.getContainerScanImage(gson, ContainerScanTestUtils.TEST_DOWNLOAD_DIRECTORY);
}

@Test
public void testGetContainerScanImageForLocalFilePath() throws DetectUserFriendlyException, IntegrationException, IOException {
public void testGetContainerScanImageForLocalFilePath() throws DetectUserFriendlyException, IntegrationException, IOException, OperationException {
// Intelligent
File containerImageRetrieved = updateDetectConfigAndGetContainerImage(BlackduckScanMode.INTELLIGENT, ContainerScanTestUtils.TEST_IMAGE_LOCAL_FILE_PATH);
Assertions.assertTrue(containerImageRetrieved != null && containerImageRetrieved.exists());
Expand All @@ -44,7 +46,7 @@ public void testGetContainerScanImageForLocalFilePath() throws DetectUserFriendl
}

@Test
public void testGetContainerScanImageForImageUrl() throws DetectUserFriendlyException, IntegrationException, IOException {
public void testGetContainerScanImageForImageUrl() throws DetectUserFriendlyException, IntegrationException, IOException, OperationException {
// Intelligent
File containerImageRetrieved = updateDetectConfigAndGetContainerImage(BlackduckScanMode.INTELLIGENT, ContainerScanTestUtils.TEST_IMAGE_URL);
Assertions.assertTrue(containerImageRetrieved != null && containerImageRetrieved.exists());
Expand Down