Skip to content

Commit

Permalink
getting service state
Browse files Browse the repository at this point in the history
  • Loading branch information
@sjeropkipruto
sjeropkipruto committed Jan 16, 2019
1 parent 7429663 commit 03675b2
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 31 deletions.
17 changes: 8 additions & 9 deletions ...ularityBase/src/main/java/com/hubspot/singularity/SingularityCheckingUpstreamsUpdate.java
View file
Expand Up @@ -3,17 +3,20 @@
import com.google.common.base.Optional; import com.google.common.base.Optional;
import com.hubspot.baragon.models.BaragonRequestState; import com.hubspot.baragon.models.BaragonRequestState;
import com.hubspot.baragon.models.BaragonServiceState; import com.hubspot.baragon.models.BaragonServiceState;
import com.hubspot.singularity.LoadBalancerRequestType.LoadBalancerRequestId;


public class SingularityCheckingUpstreamsUpdate { public class SingularityCheckingUpstreamsUpdate {
private final BaragonRequestState baragonRequestState; private final BaragonRequestState baragonRequestState;
private final Optional<BaragonServiceState> baragonServiceState; private final Optional<BaragonServiceState> baragonServiceState;
private final LoadBalancerRequestId loadBalancerRequestId; private final String singularityRequestId;


public SingularityCheckingUpstreamsUpdate (BaragonRequestState baragonRequestState,Optional<BaragonServiceState> baragonServiceState, LoadBalancerRequestId loadBalancerRequestId) { public SingularityCheckingUpstreamsUpdate (BaragonRequestState baragonRequestState,Optional<BaragonServiceState> baragonServiceState, String singularityRequest) {
this.baragonRequestState = baragonRequestState; this.baragonRequestState = baragonRequestState;
this.baragonServiceState = baragonServiceState; this.baragonServiceState = baragonServiceState;
this.loadBalancerRequestId = loadBalancerRequestId; this.singularityRequestId = singularityRequest;
}

public String getSingularityRequestId() {
return singularityRequestId;
} }


public BaragonRequestState getBaragonRequestState() { public BaragonRequestState getBaragonRequestState() {
Expand All @@ -24,16 +27,12 @@ public Optional<BaragonServiceState> getBaragonServiceState() {
return baragonServiceState; return baragonServiceState;
} }


public LoadBalancerRequestId getLoadBalancerRequestId() {
return loadBalancerRequestId;
}

@Override @Override
public String toString() { public String toString() {
return "SingularityCheckingUpstreamsUpdate{" + return "SingularityCheckingUpstreamsUpdate{" +
"baragonRequestState=" + baragonRequestState + "baragonRequestState=" + baragonRequestState +
", baragonServiceState=" + baragonServiceState + ", baragonServiceState=" + baragonServiceState +
", loadBalancerRequestId=" + loadBalancerRequestId + ", singularityRequestId=" + singularityRequestId +
'}'; '}';
} }
} }
2 changes: 1 addition & 1 deletion SingularityService/src/main/java/com/hubspot/singularity/hooks/LoadBalancerClient.java
View file
Expand Up @@ -25,7 +25,7 @@ public interface LoadBalancerClient {


SingularityLoadBalancerUpdate delete(LoadBalancerRequestId loadBalancerRequestId, String requestId, Set<String> loadBalancerGroups, String serviceBasePath); SingularityLoadBalancerUpdate delete(LoadBalancerRequestId loadBalancerRequestId, String requestId, Set<String> loadBalancerGroups, String serviceBasePath);


SingularityCheckingUpstreamsUpdate getLoadBalancerServiceStateForLoadBalancerRequest(LoadBalancerRequestId loadBalancerRequestId) throws IOException, InterruptedException, ExecutionException, TimeoutException; SingularityCheckingUpstreamsUpdate getLoadBalancerServiceStateForRequest(String singularityRequestId) throws IOException, InterruptedException, ExecutionException, TimeoutException;


List<UpstreamInfo> getUpstreamsForTasks(List<SingularityTask> tasks, String requestId, Optional<String> loadBalancerUpstreamGroup); List<UpstreamInfo> getUpstreamsForTasks(List<SingularityTask> tasks, String requestId, Optional<String> loadBalancerUpstreamGroup);


Expand Down
16 changes: 8 additions & 8 deletions SingularityService/src/main/java/com/hubspot/singularity/hooks/LoadBalancerClientImpl.java
View file
Expand Up @@ -75,22 +75,22 @@ private String getStateUriFromRequestUri(){
return loadBalancerUri.replace("request", "state"); return loadBalancerUri.replace("request", "state");
} }


private String getLoadBalancerStateUri(LoadBalancerRequestId loadBalancerRequestId){ private String getLoadBalancerStateUri(String singularityRequestId){
return String.format(OPERATION_URI, getStateUriFromRequestUri(), loadBalancerRequestId); return String.format(OPERATION_URI, getStateUriFromRequestUri(), singularityRequestId);
} }



public SingularityCheckingUpstreamsUpdate getLoadBalancerServiceStateForRequest(String singularityRequestId) throws IOException, InterruptedException, ExecutionException, TimeoutException {
public SingularityCheckingUpstreamsUpdate getLoadBalancerServiceStateForLoadBalancerRequest(LoadBalancerRequestId loadBalancerRequestId) throws IOException, InterruptedException, ExecutionException, TimeoutException { final String loadBalancerStateUri = getLoadBalancerStateUri(singularityRequestId);
final String loadBalancerStateUri = getLoadBalancerStateUri(loadBalancerRequestId);
final BoundRequestBuilder requestBuilder = httpClient.prepareGet(loadBalancerStateUri); final BoundRequestBuilder requestBuilder = httpClient.prepareGet(loadBalancerStateUri);
final Request request = requestBuilder.build(); final Request request = requestBuilder.build();
LOG.trace("Sending LB {} request for {} to {}", request.getMethod(), loadBalancerRequestId, request.getUrl()); LOG.trace("Sending LB {} request for {} to {}", request.getMethod(), singularityRequestId, request.getUrl());
ListenableFuture<Response> future = httpClient.executeRequest(request); ListenableFuture<Response> future = httpClient.executeRequest(request);
Response response = future.get(loadBalancerTimeoutMillis, TimeUnit.MILLISECONDS); Response response = future.get(loadBalancerTimeoutMillis, TimeUnit.MILLISECONDS);
LOG.trace("LB {} request {} returned with code {}", request.getMethod(), loadBalancerRequestId, response.getStatusCode()); LOG.trace("LB {} request {} returned with code {}", request.getMethod(), singularityRequestId, response.getStatusCode());
BaragonResponse lbResponse = readResponse(response); BaragonResponse lbResponse = readResponse(response);
Optional<BaragonServiceState> maybeBaragonServiceState = objectMapper.readValue(response.getResponseBodyAsBytes(), new TypeReference<BaragonServiceState>() {}); Optional<BaragonServiceState> maybeBaragonServiceState = objectMapper.readValue(response.getResponseBodyAsBytes(), new TypeReference<BaragonServiceState>() {});
return new SingularityCheckingUpstreamsUpdate(lbResponse.getLoadBalancerState(), maybeBaragonServiceState, loadBalancerRequestId);
return new SingularityCheckingUpstreamsUpdate(lbResponse.getLoadBalancerState(), maybeBaragonServiceState, singularityRequestId);
} }




Expand Down
18 changes: 8 additions & 10 deletions ...tyService/src/main/java/com/hubspot/singularity/scheduler/SingularityUpstreamChecker.java
View file
Expand Up @@ -106,31 +106,29 @@ private List<UpstreamInfo> getExtraUpstreams(Collection<UpstreamInfo> upstreamsI


public Collection<UpstreamInfo> getLoadBalancerUpstreamsForLoadBalancerRequest(SingularityCheckingUpstreamsUpdate singularityCheckingUpstreamsUpdate) throws InterruptedException, ExecutionException, TimeoutException, IOException { public Collection<UpstreamInfo> getLoadBalancerUpstreamsForLoadBalancerRequest(SingularityCheckingUpstreamsUpdate singularityCheckingUpstreamsUpdate) throws InterruptedException, ExecutionException, TimeoutException, IOException {
if (singularityCheckingUpstreamsUpdate.getBaragonServiceState().isPresent()){ if (singularityCheckingUpstreamsUpdate.getBaragonServiceState().isPresent()){
LOG.info("Baragon service state for LB request {} was present", singularityCheckingUpstreamsUpdate.getLoadBalancerRequestId()); LOG.info("Baragon service state for service {} was present", singularityCheckingUpstreamsUpdate.getSingularityRequestId());
BaragonServiceState baragonServiceState = singularityCheckingUpstreamsUpdate.getBaragonServiceState().get(); BaragonServiceState baragonServiceState = singularityCheckingUpstreamsUpdate.getBaragonServiceState().get();
return baragonServiceState.getUpstreams(); return baragonServiceState.getUpstreams();
} }
LOG.info("Baragon service state for LB request {} was absent", singularityCheckingUpstreamsUpdate.getLoadBalancerRequestId()); LOG.info("Baragon service state for service {} was absent", singularityCheckingUpstreamsUpdate.getSingularityRequestId());
return Collections.emptyList(); return Collections.emptyList();
} }


private Collection<UpstreamInfo> getUpstreamsInLoadBalancer (SingularityRequest singularityRequest, SingularityDeploy deploy) { private Collection<UpstreamInfo> getUpstreamsInLoadBalancer (String singularityRequestId) {
final LoadBalancerRequestId checkUpstreamsId = new LoadBalancerRequestId(String.format("%s-%s-%s", singularityRequest.getId(), deploy.getId(), System.currentTimeMillis()), LoadBalancerRequestType.REMOVE, Optional.absent()); LOG.info("Sent request to fetch upstream for service ", singularityRequestId);
SingularityLoadBalancerUpdate singularityLoadBalancerUpdate = lbClient.enqueue(checkUpstreamsId, singularityRequest, deploy, Collections.emptyList(), Collections.emptyList());
LOG.info("Sent LB request {} to fetch upstreams", singularityLoadBalancerUpdate.getLoadBalancerRequestId());
try { try {
Retryer<SingularityCheckingUpstreamsUpdate> getLoadBalancerUpstreamsRetryer = RetryerBuilder.<SingularityCheckingUpstreamsUpdate>newBuilder() Retryer<SingularityCheckingUpstreamsUpdate> getLoadBalancerUpstreamsRetryer = RetryerBuilder.<SingularityCheckingUpstreamsUpdate>newBuilder()
.retryIfException() .retryIfException()
.withWaitStrategy(WaitStrategies.fixedWait(1, TimeUnit.SECONDS)) .withWaitStrategy(WaitStrategies.fixedWait(1, TimeUnit.SECONDS))
.retryIfResult(CHECKING_IS_WAITING_STATE) .retryIfResult(CHECKING_IS_WAITING_STATE)
.build(); .build();
SingularityCheckingUpstreamsUpdate checkUpstreamsState = getLoadBalancerUpstreamsRetryer.call(() -> lbClient.getLoadBalancerServiceStateForLoadBalancerRequest(checkUpstreamsId)); SingularityCheckingUpstreamsUpdate checkUpstreamsState = getLoadBalancerUpstreamsRetryer.call(() -> lbClient.getLoadBalancerServiceStateForRequest(singularityRequestId));
LOG.info("Getting LB upstreams for singularity request {} through LB request {} is: {}.", singularityRequest.getId(), checkUpstreamsId, checkUpstreamsState.toString()); LOG.info("Getting LB upstreams for singularity request {} is {}.", singularityRequestId, checkUpstreamsState.toString());
if (checkUpstreamsState.getBaragonRequestState() == BaragonRequestState.SUCCESS){ if (checkUpstreamsState.getBaragonRequestState() == BaragonRequestState.SUCCESS){
return getLoadBalancerUpstreamsForLoadBalancerRequest(checkUpstreamsState); return getLoadBalancerUpstreamsForLoadBalancerRequest(checkUpstreamsState);
} }
} catch (Exception e) { } catch (Exception e) {
LOG.error("Could not get LB upstreams for singularity request {} through LB request {}. ", singularityRequest.getId(), checkUpstreamsId, e); LOG.error("Could not get LB upstreams for singularity request {}. ", singularityRequestId, e);
} }
return Collections.emptyList(); //TODO: confirm return Collections.emptyList(); //TODO: confirm
} }
Expand All @@ -139,7 +137,7 @@ private SingularityLoadBalancerUpdate syncUpstreamsForService(SingularityRequest
final LoadBalancerRequestId loadBalancerRequestId = new LoadBalancerRequestId(String.format("%s-%s-%s", singularityRequest.getId(), deploy.getId(), System.currentTimeMillis()), LoadBalancerRequestType.REMOVE, Optional.absent()); final LoadBalancerRequestId loadBalancerRequestId = new LoadBalancerRequestId(String.format("%s-%s-%s", singularityRequest.getId(), deploy.getId(), System.currentTimeMillis()), LoadBalancerRequestType.REMOVE, Optional.absent());
final long start = System.currentTimeMillis(); final long start = System.currentTimeMillis();
try { try {
Collection<UpstreamInfo> upstreamsInLoadBalancerForRequest = getUpstreamsInLoadBalancer(singularityRequest, deploy); Collection<UpstreamInfo> upstreamsInLoadBalancerForRequest = getUpstreamsInLoadBalancer(singularityRequest.getId());
LOG.info("Upstreams in load balancer for service {} are {}.", singularityRequest.getId(), upstreamsInLoadBalancerForRequest); LOG.info("Upstreams in load balancer for service {} are {}.", singularityRequest.getId(), upstreamsInLoadBalancerForRequest);
Collection<UpstreamInfo> upstreamsInSingularityForRequest = getUpstreamsFromActiveTasksForRequest(singularityRequest.getId(), loadBalancerUpstreamGroup); Collection<UpstreamInfo> upstreamsInSingularityForRequest = getUpstreamsFromActiveTasksForRequest(singularityRequest.getId(), loadBalancerUpstreamGroup);
LOG.info("Upstreams in singularity for service {} are {}.", singularityRequest.getId(), upstreamsInSingularityForRequest); LOG.info("Upstreams in singularity for service {} are {}.", singularityRequest.getId(), upstreamsInSingularityForRequest);
Expand Down
5 changes: 2 additions & 3 deletions ...ityService/src/test/java/com/hubspot/singularity/scheduler/TestingLoadBalancerClient.java
View file
Expand Up @@ -56,11 +56,10 @@ public SingularityLoadBalancerUpdate delete(LoadBalancerRequestId loadBalancerRe
} }


@Override @Override
public SingularityCheckingUpstreamsUpdate getLoadBalancerServiceStateForLoadBalancerRequest(LoadBalancerRequestId loadBalancerRequestId) throws IOException, InterruptedException, ExecutionException, TimeoutException { public SingularityCheckingUpstreamsUpdate getLoadBalancerServiceStateForRequest(String singularityRequestId) throws IOException, InterruptedException, ExecutionException, TimeoutException {
return new SingularityCheckingUpstreamsUpdate(requestState, Optional.absent(), loadBalancerRequestId); return new SingularityCheckingUpstreamsUpdate(requestState, Optional.absent(), singularityRequestId);
} }



@Override @Override
public List<UpstreamInfo> getUpstreamsForTasks(List<SingularityTask> tasks, String requestId, Optional<String> loadBalancerUpstreamGroup) { public List<UpstreamInfo> getUpstreamsForTasks(List<SingularityTask> tasks, String requestId, Optional<String> loadBalancerUpstreamGroup) {
return Collections.emptyList(); return Collections.emptyList();
Expand Down

0 comments on commit 03675b2

Please sign in to comment.