Skip to content

Commit

Permalink
Implemented resilience inside sechub PDS adapter #2280
Browse files Browse the repository at this point in the history
  • Loading branch information
@de-jcup
de-jcup committed Jun 2, 2023
1 parent 81c5a7e commit a634b6c
Show file tree
Hide file tree
Showing 17 changed files with 300 additions and 48 deletions.
3 changes: 3 additions & 0 deletions ...b-adapter-pds/src/main/java/com/mercedesbenz/sechub/adapter/pds/PDSAdapterConfigData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,7 @@ public interface PDSAdapterConfigData {

Long getSourceCodeZipFileSizeInBytesOrNull();

int getResilienceMaxRetries();

long getResilienceTimeToWaitBeforeRetryInMilliseconds();
}
4 changes: 4 additions & 0 deletions ...adapter-pds/src/main/java/com/mercedesbenz/sechub/adapter/pds/PDSAdapterConfigurator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,8 @@ public default void configure() {

void setPDSScriptTrustsAllCertificates(boolean trustAllCertificates);

void setResilienceMaxRetries(int maxRetries);

void setResilienceTimeToWaitBeforeRetryInMilliseconds(long milliseconds);

}
24 changes: 24 additions & 0 deletions ...ter-pds/src/main/java/com/mercedesbenz/sechub/adapter/pds/PDSAdapterDataConfigurator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ public class PDSAdapterDataConfigurator implements PDSAdapterConfigData, PDSAdap

private Long sourceCodeZipFileSizeInBytes;

private int resilienceMaxRetries;

private long resilienceTimeToWaitBeforeRetryInMilliseconds;

public void setTargetType(String targetType) {
if (targetType == null) {
this.targetType = EMPTY_TARGET_TYPE;
Expand Down Expand Up @@ -226,4 +230,24 @@ public Long getSourceCodeZipFileSizeInBytesOrNull() {
return sourceCodeZipFileSizeInBytes;
}

@Override
public void setResilienceMaxRetries(int maxRetries) {
this.resilienceMaxRetries = maxRetries;
}

@Override
public void setResilienceTimeToWaitBeforeRetryInMilliseconds(long milliseconds) {
this.resilienceTimeToWaitBeforeRetryInMilliseconds = milliseconds;
}

@Override
public int getResilienceMaxRetries() {
return resilienceMaxRetries;
}

@Override
public long getResilienceTimeToWaitBeforeRetryInMilliseconds() {
return resilienceTimeToWaitBeforeRetryInMilliseconds;
}

}
103 changes: 75 additions & 28 deletions sechub-adapter-pds/src/main/java/com/mercedesbenz/sechub/adapter/pds/PDSAdapterV1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,35 +63,74 @@ public int getAdapterVersion() {

@Override
protected AdapterExecutionResult execute(PDSAdapterConfig config, AdapterRuntimeContext runtimeContext) throws AdapterException {
AdapterExecutionResult result = null;

assertThreadNotInterrupted();

PDSContext context = contextFactory.create(config, this, runtimeContext);
PDSContext pdsContext = contextFactory.create(config, this, runtimeContext);
handleResilienceConfiguration(pdsContext);

try {
result = execute(pdsContext);
} catch (Exception e) {
if (e instanceof AdapterException) {
throw (AdapterException) e;
}
if (e instanceof RuntimeException) {
RuntimeException re = (RuntimeException) e;
throw re;
}
throw asAdapterException("Unexpected error: " + e.getMessage(), config);
}

return result;

}

private void handleResilienceConfiguration(PDSContext pdsContext) {
PDSAdapterConfig config = pdsContext.getConfig();

PDSAdapterConfigData data = config.getPDSAdapterConfigData();
assertConfigDataNotNull(data);

int maxRetries = data.getResilienceMaxRetries();
if (maxRetries >= 0) {
LOG.info("Change resilience max retries to: {}", maxRetries);
pdsContext.getSocketExceptionConsultant().setMaxRetries(maxRetries);
}

long maxWaitInMillis = data.getResilienceTimeToWaitBeforeRetryInMilliseconds();
if (maxWaitInMillis > 0) {
LOG.info("Change resilience max wait time to: {} milliseconds", maxRetries);
pdsContext.getSocketExceptionConsultant().setRetryTimeToWaitInMilliseconds(maxWaitInMillis);
}
}

private AdapterExecutionResult execute(PDSContext pdsContext) throws Exception {

AdapterExecutionResult alreadyKnownResult = handleExecutionType(context, runtimeContext);
AdapterExecutionResult alreadyKnownResult = handleExecutionType(pdsContext);
if (alreadyKnownResult != null) {
return alreadyKnownResult;
}

assertThreadNotInterrupted();

uploadJobDataIfNecessary(context, runtimeContext);
uploadJobDataIfNecessary(pdsContext);
assertThreadNotInterrupted();

markJobAsReadyIfNecessary(context, runtimeContext);
markJobAsReadyIfNecessary(pdsContext);
assertThreadNotInterrupted();

waitForJobDone(context);
waitForJobDone(pdsContext);
assertThreadNotInterrupted();

return collectAdapterExecutionResult(context);

return collectAdapterExecutionResult(pdsContext);
}

private AdapterExecutionResult collectAdapterExecutionResult(PDSContext context) throws AdapterException {
return new AdapterExecutionResult(fetchReport(context), fetchMessages(context));
}

private void waitForJobDone(PDSContext context) throws AdapterException {
private void waitForJobDone(PDSContext context) throws Exception {
PDSAdapterConfig config = context.getConfig();
PDSAdapterConfigData data = config.getPDSAdapterConfigData();

Expand Down Expand Up @@ -130,7 +169,7 @@ private void waitForJobDone(PDSContext context) throws AdapterException {
}

/* see PDSJobStatusState.java */
jobstatus = getJobStatus(context);
jobstatus = context.getResilientJobStatusResultExecutor().executeResilient(() -> getJobStatus(context));

PDSJobStatusState state = jobstatus.state;
switch (state) {
Expand Down Expand Up @@ -196,7 +235,8 @@ private String fetchReport(PDSContext context) throws AdapterException {
UUID pdsJobUUID = context.getPdsJobUUID();
String url = context.getUrlBuilder().buildGetJobResult(pdsJobUUID);

ResponseEntity<String> response = context.getRestOperations().getForEntity(url, String.class);
ResponseEntity<String> response = null;
response = context.getRestOperations().getForEntity(url, String.class);

return response.getBody();
}
Expand Down Expand Up @@ -236,8 +276,9 @@ private PDSJobStatus getJobStatus(PDSContext context, UUID pdsJobUUID) {
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++ */
/* + ................Mark JOB as ready............... + */
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++ */
private void markJobAsReadyIfNecessary(PDSContext context, AdapterRuntimeContext runtimeContext) {
private void markJobAsReadyIfNecessary(PDSContext context) throws Exception {

AdapterRuntimeContext runtimeContext = context.getRuntimeContext();
AdapterMetaData metaData = runtimeContext.getMetaData();
if (metaData.getValueAsBoolean(PDS_JOB_MARKED_AS_READY)) {
LOG.info("Mark job ready skipped for pds job: {}, becausse already marked as ready");
Expand All @@ -247,7 +288,7 @@ private void markJobAsReadyIfNecessary(PDSContext context, AdapterRuntimeContext

UUID uuid = context.getPdsJobUUID();
String url = context.getUrlBuilder().buildMarkJobReadyToStart(uuid);
context.getRestSupport().put(url);
context.getResilientExecutor().executeResilient(() -> context.getRestSupport().put(url));

metaData.setValue(PDS_JOB_MARKED_AS_READY, true);

Expand All @@ -257,7 +298,7 @@ private void markJobAsReadyIfNecessary(PDSContext context, AdapterRuntimeContext
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++ */
/* + ................Upload.......................... + */
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++ */
private void uploadJobDataIfNecessary(PDSContext context, AdapterRuntimeContext runtimeContext) throws AdapterException {
private void uploadJobDataIfNecessary(PDSContext context) throws Exception {
PDSAdapterConfig config = context.getConfig();
PDSAdapterConfigData data = config.getPDSAdapterConfigData();

Expand All @@ -267,13 +308,13 @@ private void uploadJobDataIfNecessary(PDSContext context, AdapterRuntimeContext
}

/* PDS has other storage - we must upload content */
AdapterRuntimeContext runtimeContext = context.getRuntimeContext();
AdapterMetaData metaData = runtimeContext.getMetaData();
if (metaData.getValueAsBoolean(PDS_JOB_UPLOAD_DONE)) {
LOG.info("Upload skipped for pds job: {}, becausse already uploaded");
/* already uploaded */
return;
}

handleUploadWhenRequired(context, SecHubDataConfigurationType.SOURCE);
handleUploadWhenRequired(context, SecHubDataConfigurationType.BINARY);

Expand All @@ -282,7 +323,7 @@ private void uploadJobDataIfNecessary(PDSContext context, AdapterRuntimeContext
runtimeContext.getCallback().persist(metaData);
}

private void handleUploadWhenRequired(PDSContext context, SecHubDataConfigurationType type) throws AdapterException {
private void handleUploadWhenRequired(PDSContext context, SecHubDataConfigurationType type) throws Exception {
PDSAdapterConfig config = context.getConfig();
PDSAdapterConfigData data = config.getPDSAdapterConfigData();

Expand Down Expand Up @@ -313,8 +354,9 @@ private void handleUploadWhenRequired(PDSContext context, SecHubDataConfiguratio
if (fileSize != null) {
fileSizeAsString = "" + fileSize;
}
final String finalFileSizeAsString = fileSizeAsString;

uploadSupport.upload(type, context, data, checksum, fileSizeAsString);
context.getResilientExecutor().executeResilient(() -> uploadSupport.upload(type, context, data, checksum, finalFileSizeAsString));

/* after this - mark file upload done - at least for debugging */
metaData.setValue(sourceUploadMetaDataKey, true);
Expand Down Expand Up @@ -376,7 +418,8 @@ private String createUploadMetaDataKey(UUID pdsJobUUID, SecHubDataConfigurationT
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++ */
/* + ................Execution type handling ........ + */
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++ */
private AdapterExecutionResult handleExecutionType(PDSContext context, AdapterRuntimeContext runtimeContext) throws AdapterException {
private AdapterExecutionResult handleExecutionType(PDSContext context) throws Exception {
AdapterRuntimeContext runtimeContext = context.getRuntimeContext();

ExecutionType executionType = runtimeContext.getType();

Expand All @@ -392,7 +435,7 @@ private AdapterExecutionResult handleExecutionType(PDSContext context, AdapterRu
}
}

private AdapterExecutionResult handleExecutionTypeInitial(PDSContext context, AdapterRuntimeContext runtimeContext) throws AdapterException {
private AdapterExecutionResult handleExecutionTypeInitial(PDSContext context, AdapterRuntimeContext runtimeContext) throws Exception {
LOG.debug("Initial pds job creation necessary");

String pdsJobUUID = createNewPDSJobAndRememberInAdapterMetaData(context, runtimeContext);
Expand All @@ -401,7 +444,7 @@ private AdapterExecutionResult handleExecutionTypeInitial(PDSContext context, Ad
return null;
}

private AdapterExecutionResult handleExecutionTypeRestart(PDSContext context, AdapterRuntimeContext runtimeContext) throws AdapterException {
private AdapterExecutionResult handleExecutionTypeRestart(PDSContext context, AdapterRuntimeContext runtimeContext) throws Exception {
AdapterMetaData metaData = runtimeContext.getMetaData();
String pdsJobUUID = metaData.getValueAsStringOrNull(PDS_JOB_UUID);

Expand Down Expand Up @@ -463,7 +506,7 @@ private AdapterExecutionResult handleExecutionTypeRestart(PDSContext context, Ad
return NO_EXISTING_ADAPTER_EXECUTION_RESULT;
}

private AdapterExecutionResult handleExecutionTypeCancel(PDSContext context, AdapterRuntimeContext runtimeContext) throws AdapterException {
private AdapterExecutionResult handleExecutionTypeCancel(PDSContext context, AdapterRuntimeContext runtimeContext) throws Exception {
AdapterMetaData metaData = runtimeContext.getMetaData();
String pdsJobUUID = metaData.getValueAsStringOrNull(PDS_JOB_UUID);

Expand All @@ -480,11 +523,11 @@ private AdapterExecutionResult handleExecutionTypeCancel(PDSContext context, Ada
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++ */
/* + ................New Job......................... + */
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++ */
private String createNewPDSJobAndRememberInAdapterMetaData(PDSContext context, AdapterRuntimeContext runtimeContext) throws AdapterException {
private String createNewPDSJobAndRememberInAdapterMetaData(PDSContext context, AdapterRuntimeContext runtimeContext) throws Exception {
String json = createJobDataJSON(context);
String url = context.getUrlBuilder().buildCreateJob();

String jsonResult = context.getRestSupport().postJSON(url, json);
String jsonResult = context.getResilientStringResultExecutor().executeResilient(() -> context.getRestSupport().postJSON(url, json));
PDSJobCreateResult result = context.getJsonSupport().fromJSON(PDSJobCreateResult.class, jsonResult);
String pdsJobUUID = result.jobUUID;

Expand All @@ -502,11 +545,11 @@ private String createNewPDSJobAndRememberInAdapterMetaData(PDSContext context, A
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++ */
/* + ................Cancel Job ....................... + */
/* ++++++++++++++++++++++++++++++++++++++++++++++++++++ */
private void cancelJob(PDSContext context) {
private void cancelJob(PDSContext context) throws Exception {
UUID pdsJobUUID = context.getPdsJobUUID();

String url = context.getUrlBuilder().buildCancelJob(pdsJobUUID);
context.getRestSupport().put(url);
context.getResilientExecutor().executeResilient(() -> context.getRestSupport().put(url));

LOG.info("PDS job canceled: {}", pdsJobUUID);

Expand All @@ -522,9 +565,7 @@ private String createJobDataJSON(PDSContext context) throws AdapterException {
private PDSJobData createJobData(PDSContext context) {
PDSAdapterConfig config = context.getConfig();
PDSAdapterConfigData data = config.getPDSAdapterConfigData();
if (data == null) {
throw new IllegalStateException("Adapter config data may not be null!");
}
assertConfigDataNotNull(data);
Map<String, String> parameters = data.getJobParameters();

PDSJobData jobData = new PDSJobData();
Expand All @@ -543,6 +584,12 @@ private PDSJobData createJobData(PDSContext context) {
return jobData;
}

private void assertConfigDataNotNull(PDSAdapterConfigData data) {
if (data == null) {
throw new IllegalStateException("Adapter config data may not be null!");
}
}

@Override
protected String getAPIPrefix() {
return "/api/";
Expand Down
36 changes: 36 additions & 0 deletions sechub-adapter-pds/src/main/java/com/mercedesbenz/sechub/adapter/pds/PDSContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
import com.mercedesbenz.sechub.adapter.AdapterRuntimeContext;
import com.mercedesbenz.sechub.adapter.support.JSONAdapterSupport;
import com.mercedesbenz.sechub.adapter.support.RestOperationsSupport;
import com.mercedesbenz.sechub.commons.core.resilience.ResilientActionExecutor;
import com.mercedesbenz.sechub.commons.core.resilience.ResilientRunnableExecutor;
import com.mercedesbenz.sechub.commons.pds.data.PDSJobStatus;

/**
* Context for PDS execution.
Expand All @@ -23,11 +26,44 @@ public class PDSContext extends AbstractSpringRestAdapterContext<PDSAdapterConfi
private RestOperationsSupport restSupport;
private UUID pdsJobUUID;

private PDSSocketExceptionResilienceConsultant socketExceptionConsultant;
private ResilientRunnableExecutor resilientExecutor;
private ResilientActionExecutor<PDSJobStatus> resilientJobStatusResultExecutor;
private ResilientActionExecutor<String> resilientStringResultExecutor;

public PDSContext(PDSAdapterConfig config, PDSAdapter adapter, AdapterRuntimeContext runtimeContext) {
super(config, adapter, runtimeContext);
urlBuilder = new PDSUrlBuilder(config.getProductBaseURL());
jsonSupport = new JSONAdapterSupport(adapter, config);
restSupport = new RestOperationsSupport(getRestOperations());

socketExceptionConsultant = new PDSSocketExceptionResilienceConsultant();

resilientExecutor = new ResilientRunnableExecutor();
resilientExecutor.add(socketExceptionConsultant);

resilientJobStatusResultExecutor = new ResilientActionExecutor<>();
resilientJobStatusResultExecutor.add(socketExceptionConsultant);

resilientStringResultExecutor = new ResilientActionExecutor<>();
resilientStringResultExecutor.add(socketExceptionConsultant);

}

public PDSSocketExceptionResilienceConsultant getSocketExceptionConsultant() {
return socketExceptionConsultant;
}

public ResilientRunnableExecutor getResilientExecutor() {
return resilientExecutor;
}

public ResilientActionExecutor<PDSJobStatus> getResilientJobStatusResultExecutor() {
return resilientJobStatusResultExecutor;
}

public ResilientActionExecutor<String> getResilientStringResultExecutor() {
return resilientStringResultExecutor;
}

public RestOperationsSupport getRestSupport() {
Expand Down
Loading

0 comments on commit a634b6c

Please sign in to comment.