Skip to content

Commit

Permalink
Adds a factory for building ResponseProcessor.
Browse files Browse the repository at this point in the history
-- We would like to do a little bit of Druid error analysis to extract
some more illuminating error codes out of some of Druid's 500's. The
error callback is controlled by the `ResponseProcessor`.

-- Unfortunately, the `ResponseProcessor` can't be injected directly
because it depends on the `DataApiRequest`, which is built directly in
each request. So instead we wrap the creation of `ResponseProcessor` in
a factory that can be injected.
  • Loading branch information
Andrew Cholewa committed Apr 6, 2018
1 parent 98612ca commit f4a6054
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 8 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ Current

### Changed:

- [ResponseProcessor is now injectable.](https://github.com/yahoo/fili/pull/663)
* To add a custom `ResponseProcessor`, implement `ResponseProcessorFactory`, override
`AbstractBinderFactory::buildResponseProcessorFactory` to return your custom `ResponseProcessorFactory.class`.

- [Add config to ignore partial/volatile intervals and cache everything in cache V2](https://github.com/yahoo/fili/pull/645)
* In cache V2, user should be able to decide whether partial data or volatile data should be cached or not. This PR
adds a config that allows the user to do this.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@
import com.yahoo.bard.webservice.druid.util.SketchFieldConverter;
import com.yahoo.bard.webservice.metadata.DataSourceMetadataLoadTask;
import com.yahoo.bard.webservice.metadata.DataSourceMetadataService;
import com.yahoo.bard.webservice.metadata.RegisteredLookupMetadataLoadTask;
import com.yahoo.bard.webservice.metadata.QuerySigningService;
import com.yahoo.bard.webservice.metadata.RegisteredLookupMetadataLoadTask;
import com.yahoo.bard.webservice.metadata.RequestedIntervalsFunction;
import com.yahoo.bard.webservice.metadata.SegmentIntervalsHashIdGenerator;
import com.yahoo.bard.webservice.table.LogicalTableDictionary;
Expand All @@ -85,9 +85,6 @@
import com.yahoo.bard.webservice.util.SimplifiedIntervalList;
import com.yahoo.bard.webservice.web.CsvResponseWriter;
import com.yahoo.bard.webservice.web.DataApiRequest;
import com.yahoo.bard.webservice.web.apirequest.DataApiRequestFactory;
import com.yahoo.bard.webservice.web.apirequest.DefaultDataApiRequestFactory;
import com.yahoo.bard.webservice.web.ratelimit.DefaultRateLimiter;
import com.yahoo.bard.webservice.web.DefaultResponseFormatResolver;
import com.yahoo.bard.webservice.web.DimensionApiRequestMapper;
import com.yahoo.bard.webservice.web.DimensionsApiRequest;
Expand All @@ -106,11 +103,16 @@
import com.yahoo.bard.webservice.web.ResponseWriter;
import com.yahoo.bard.webservice.web.SlicesApiRequest;
import com.yahoo.bard.webservice.web.TablesApiRequest;
import com.yahoo.bard.webservice.web.apirequest.DataApiRequestFactory;
import com.yahoo.bard.webservice.web.apirequest.DefaultDataApiRequestFactory;
import com.yahoo.bard.webservice.web.apirequest.DefaultHavingApiGenerator;
import com.yahoo.bard.webservice.web.apirequest.HavingGenerator;
import com.yahoo.bard.webservice.web.apirequest.PerRequestDictionaryHavingGenerator;
import com.yahoo.bard.webservice.web.handlers.workflow.DruidWorkflow;
import com.yahoo.bard.webservice.web.handlers.workflow.RequestWorkflowProvider;
import com.yahoo.bard.webservice.web.ratelimit.DefaultRateLimiter;
import com.yahoo.bard.webservice.web.responseprocessors.ResponseProcessorFactory;
import com.yahoo.bard.webservice.web.responseprocessors.ResultSetResponseProcessorFactory;
import com.yahoo.bard.webservice.web.util.QueryWeightUtil;

import com.codahale.metrics.Gauge;
Expand Down Expand Up @@ -339,6 +341,8 @@ protected void configure() {

bind(buildResponseFormatResolver()).to(ResponseFormatResolver.class);

bind(buildResponseProcessorFactory()).to(ResponseProcessorFactory.class);

bind(buildRateLimiter()).to(RateLimiter.class);

if (DRUID_DIMENSIONS_LOADER.isOn()) {
Expand Down Expand Up @@ -1200,6 +1204,19 @@ protected ResponseFormatResolver buildResponseFormatResolver() {
return new DefaultResponseFormatResolver();
}

/**
* Returns the class to bind to {@link ResponseProcessorFactory}.
* <p>
* The ResponseProcessorFactory allows us to inject a custom {@link
* com.yahoo.bard.webservice.web.responseprocessors.ResponseProcessor} despite the fact that these processors depend
* on objects that are built uniquely for each request.
*
* @return A class that implements {@link ResponseProcessorFactory}.
*/
protected Class<? extends ResponseProcessorFactory> buildResponseProcessorFactory() {
return ResultSetResponseProcessorFactory.class;
}

/**
* Creates a new RateLimiter for the RateLimitFilter.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package com.yahoo.bard.webservice.web.endpoints;

import static com.yahoo.bard.webservice.web.handlers.workflow.DruidWorkflow.REQUEST_WORKFLOW_TIMER;

import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static javax.ws.rs.core.Response.Status.GATEWAY_TIMEOUT;
import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR;
Expand Down Expand Up @@ -52,7 +53,8 @@
import com.yahoo.bard.webservice.web.handlers.RequestContext;
import com.yahoo.bard.webservice.web.handlers.RequestHandlerUtils;
import com.yahoo.bard.webservice.web.handlers.workflow.RequestWorkflowProvider;
import com.yahoo.bard.webservice.web.responseprocessors.ResultSetResponseProcessor;
import com.yahoo.bard.webservice.web.responseprocessors.ResponseProcessor;
import com.yahoo.bard.webservice.web.responseprocessors.ResponseProcessorFactory;
import com.yahoo.bard.webservice.web.util.BardConfigResources;

import com.codahale.metrics.MetricRegistry;
Expand Down Expand Up @@ -122,6 +124,7 @@ public class DataServlet extends CORSPreflightServlet implements BardConfigResou
private final ObjectMappersSuite objectMappers;
private final HttpResponseMaker httpResponseMaker;
private final ResponseFormatResolver formatResolver;
private final ResponseProcessorFactory responseProcessorFactory;

private final DataApiRequestFactory dataApiRequestFactory;

Expand Down Expand Up @@ -153,6 +156,7 @@ public class DataServlet extends CORSPreflightServlet implements BardConfigResou
* @param formatResolver The formatResolver for determining correct response format
* @param dataApiRequestFactory A factory to build dataApiRequests
* {@link com.yahoo.bard.webservice.async.preresponses.stores.PreResponseStore}
* @param responseProcessorFactory Builds the object that performs post processing on a Druid response
*/
@Inject
public DataServlet(
Expand All @@ -172,7 +176,8 @@ public DataServlet(
BroadcastChannel<String> preResponseStoredNotifications,
HttpResponseMaker httpResponseMaker,
ResponseFormatResolver formatResolver,
DataApiRequestFactory dataApiRequestFactory
DataApiRequestFactory dataApiRequestFactory,
ResponseProcessorFactory responseProcessorFactory
) {
this.resourceDictionaries = resourceDictionaries;
this.druidQueryBuilder = druidQueryBuilder;
Expand All @@ -192,6 +197,7 @@ public DataServlet(
this.httpResponseMaker = httpResponseMaker;
this.formatResolver = formatResolver;
this.dataApiRequestFactory = dataApiRequestFactory;
this.responseProcessorFactory = responseProcessorFactory;

LOG.trace(
"Initialized with ResourceDictionaries: {} \n\n" +
Expand Down Expand Up @@ -426,7 +432,7 @@ public void getData(
httpResponseMaker
);

ResultSetResponseProcessor response = new ResultSetResponseProcessor(
ResponseProcessor responseProcessor = responseProcessorFactory.build(
apiRequest,
queryResultsEmitter,
druidResponseParser,
Expand All @@ -441,7 +447,7 @@ public void getData(

// Process the request
RequestLog.startTiming(REQUEST_WORKFLOW_TIMER);
boolean complete = dataRequestHandler.handleRequest(context, apiRequest, druidQuery, response);
boolean complete = dataRequestHandler.handleRequest(context, apiRequest, druidQuery, responseProcessor);
if (!complete) {
throw new IllegalStateException("No request handler accepted request.");
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright 2018 Yahoo Inc.
// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
package com.yahoo.bard.webservice.web.responseprocessors;

import com.yahoo.bard.webservice.application.ObjectMappersSuite;
import com.yahoo.bard.webservice.data.DruidResponseParser;
import com.yahoo.bard.webservice.data.HttpResponseMaker;
import com.yahoo.bard.webservice.web.DataApiRequest;
import com.yahoo.bard.webservice.web.PreResponse;

import rx.subjects.Subject;

/**
* A {@link ResponseProcessor} relies on things that are directly constructed at request time (i.e. the
* {@link DataApiRequest}). Therefore, we can't inject a `ResponseProcessor` directly. We can however inject a factory.
*/
public interface ResponseProcessorFactory {

/**
* Constructs a custom ResponseProcessor.
*
* @param apiRequest The current request
* @param responseEmitter Generates the response to be processed
* @param druidResponseParser Transforms a druid response into a {@link ResultSet}
* @param objectMappers Dictates how to format
* @param httpResponseMaker Crafts an HTTP response to be sent back to the user from a ResultSet or error message
*
* @return An object that handles parsing and post-processing of Druid requests
*/
ResponseProcessor build(
DataApiRequest apiRequest,
Subject<PreResponse, PreResponse> responseEmitter,
DruidResponseParser druidResponseParser,
ObjectMappersSuite objectMappers,
HttpResponseMaker httpResponseMaker
);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// Copyright 2018 Yahoo Inc.
// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
package com.yahoo.bard.webservice.web.responseprocessors;

import com.yahoo.bard.webservice.application.ObjectMappersSuite;
import com.yahoo.bard.webservice.data.DruidResponseParser;
import com.yahoo.bard.webservice.data.HttpResponseMaker;
import com.yahoo.bard.webservice.web.DataApiRequest;
import com.yahoo.bard.webservice.web.PreResponse;

import rx.subjects.Subject;

/**
* Builds the default Druid response processor: {@link ResultSetResponseProcessor}.
*/
public class ResultSetResponseProcessorFactory implements ResponseProcessorFactory {

@Override
public ResponseProcessor build(
DataApiRequest apiRequest,
Subject<PreResponse, PreResponse> responseEmitter,
DruidResponseParser druidResponseParser,
ObjectMappersSuite objectMappers,
HttpResponseMaker httpResponseMaker
) {
return new ResultSetResponseProcessor(
apiRequest,
responseEmitter,
druidResponseParser,
objectMappers,
httpResponseMaker
);
}
}

0 comments on commit f4a6054

Please sign in to comment.