Reorganizes the asynchronous classes.

CHANGELOG.md

@@ -2,8 +2,8 @@ Change Log
 ==========
 
 All notable changes to Fili will be documented here. Changes are accumulated as new paragraphs at the top of the current 
-major version. Each change has a link to the issue that triggered the change, and to the pull request that makes the
-change.
+major version. Each change has a link to the pull request that makes the change and to the issue that triggered the
+pull request if there was one.
 
 Current
 -------
@@ -17,15 +17,36 @@ Current
 ### Removed:
 
 
-#### Changed:
+### Changed:
+
+- [Reorganizes asynchronous package structure](https://github.com/yahoo/fili/pull/19)
+    * The `jobs` package is renamed to `async` and split into the following subpackages:
+        - `broadcastchannels` - Everything dealing with broadcast channels
+        - `jobs` - Everything related to `jobs`, broken into subpackages
+            * `jobrows` - Everything related to the content of the job metadata
+            * `payloads` - Everything related to building the version of the job metadata to send to the user
+            * `stores` - Everything related to the databases for job data
+        - `preresponses` - Everything related to `PreResponses`, broken into subpackages
+            * `stores` - Everything related to the the databases for PreResponse data
+        - `workflows` - Everything related to the asynchronous workflow
+
+-  [Removed timing component in JobsApiRequestSpec](https://github.com/yahoo/fili/pull/27)
+    * Rather than setting an async timeout, and then sleeping, 
+      `JobsApiReqeustSpec::handleBroadcastChannelNotification returns an empty Observable if a timeout occurs before the notification is received`
+      now verifies that the Observable returned terminates without sending any
+      messages.
 
 -  [Fix Dimension Serialization Problem with Nested Queries](https://github.com/yahoo/fili/pull/15)
     * Modified `DimensionToDefaultDimensionSpec` serializer to serialize dimension to apiName if it is not the inner most query
     * Added helper `hasInnerQuery` to `Util` in serializer package to determine if current query is the inner most query or not
     * Added tests for `DimensionToDefaultDimensionSpec`
 
+
 ### Fixed:
 
+- [`DruidDimensionsLoader` doesn't set the dimension's lastUpdated date](https://github.com/yahoo/fili/pull/24)
+  * `DruidDimensionsLoader` now properly sets the `lastUpdated` field after it finished processing the Druid response
+
 
 ### Known Issues:
 

docs/usage.sample

@@ -1,5 +1,5 @@
 <dependency>
     <groupId>com.yahoo.fili</groupId>
-    <artifactId>fili-core</artifactId>
+    <artifactId>fili</artifactId>
     <version>0.1</version>
 </dependency>

fili-core/pom.xml

@@ -16,7 +16,7 @@
     <parent>
         <groupId>com.yahoo.fili</groupId>
         <artifactId>fili-parent-pom</artifactId>
-        <version>0.1.3</version>
+        <version>0.1-SNAPSHOT</version>
     </parent>
 
     <properties>
@@ -29,7 +29,7 @@
         <dependency>
             <groupId>com.yahoo.fili</groupId>
             <artifactId>fili-system-config</artifactId>
-            <version>${fili.version}</version>
+            <version>0.1-SNAPSHOT</version>
         </dependency>
 
         <dependency>
@@ -261,16 +261,6 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-checkstyle-plugin</artifactId>
-                <version>2.16</version>
-                <configuration>
-                    <configLocation>${checkstyle.config.location}</configLocation>
-                    <suppressionsLocation>${checkstyle.suppressions.location}</suppressionsLocation>
-                    <suppressionsFileExpression>checkstyle.suppressions.file</suppressionsFileExpression>
-                    <includes>**\/*.java</includes>
-                    <resourceIncludes>**/*.properties,**/*.sample</resourceIncludes>
-                    <includeTestSourceDirectory>true</includeTestSourceDirectory>
-                    <consoleOutput>true</consoleOutput>
-                </configuration>
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -284,7 +274,6 @@
                     </execution>
                 </executions>
             </plugin>
-
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>

fili-core/src/main/java/com/yahoo/bard/webservice/application/AbstractBinderFactory.java

@@ -15,8 +15,8 @@
 import com.yahoo.bard.webservice.application.healthchecks.DruidDimensionsLoaderHealthCheck;
 import com.yahoo.bard.webservice.application.healthchecks.SegmentMetadataLoaderHealthCheck;
 import com.yahoo.bard.webservice.application.healthchecks.VersionHealthCheck;
-import com.yahoo.bard.webservice.jobs.AsynchronousWorkflowsBuilder;
-import com.yahoo.bard.webservice.jobs.DefaultAsynchronousWorkflowsBuilder;
+import com.yahoo.bard.webservice.async.workflows.AsynchronousWorkflowsBuilder;
+import com.yahoo.bard.webservice.async.workflows.DefaultAsynchronousWorkflowsBuilder;
 import com.yahoo.bard.webservice.config.BardFeatureFlag;
 import com.yahoo.bard.webservice.config.FeatureFlag;
 import com.yahoo.bard.webservice.config.FeatureFlagRegistry;
@@ -55,17 +55,17 @@
 import com.yahoo.bard.webservice.druid.util.FieldConverterSupplier;
 import com.yahoo.bard.webservice.druid.util.FieldConverters;
 import com.yahoo.bard.webservice.druid.util.SketchFieldConverter;
-import com.yahoo.bard.webservice.jobs.ApiJobStore;
-import com.yahoo.bard.webservice.jobs.BroadcastChannel;
-import com.yahoo.bard.webservice.jobs.DefaultJobField;
-import com.yahoo.bard.webservice.jobs.DefaultJobPayloadBuilder;
-import com.yahoo.bard.webservice.jobs.DefaultJobRowBuilder;
-import com.yahoo.bard.webservice.jobs.JobPayloadBuilder;
-import com.yahoo.bard.webservice.jobs.JobRowBuilder;
-import com.yahoo.bard.webservice.jobs.NoOpApiJobStore;
-import com.yahoo.bard.webservice.jobs.NoOpPreResponseStore;
-import com.yahoo.bard.webservice.jobs.PreResponseStore;
-import com.yahoo.bard.webservice.jobs.SimpleBroadcastChannel;
+import com.yahoo.bard.webservice.async.jobs.stores.ApiJobStore;
+import com.yahoo.bard.webservice.async.broadcastchannels.BroadcastChannel;
+import com.yahoo.bard.webservice.async.jobs.jobrows.DefaultJobField;
+import com.yahoo.bard.webservice.async.jobs.payloads.DefaultJobPayloadBuilder;
+import com.yahoo.bard.webservice.async.jobs.jobrows.DefaultJobRowBuilder;
+import com.yahoo.bard.webservice.async.jobs.payloads.JobPayloadBuilder;
+import com.yahoo.bard.webservice.async.jobs.jobrows.JobRowBuilder;
+import com.yahoo.bard.webservice.async.jobs.stores.NoOpApiJobStore;
+import com.yahoo.bard.webservice.async.preresponses.stores.NoOpPreResponseStore;
+import com.yahoo.bard.webservice.async.preresponses.stores.PreResponseStore;
+import com.yahoo.bard.webservice.async.broadcastchannels.SimpleBroadcastChannel;
 import com.yahoo.bard.webservice.metadata.DataSourceMetadataLoader;
 import com.yahoo.bard.webservice.metadata.DataSourceMetadataService;
 import com.yahoo.bard.webservice.metadata.QuerySigningService;

fili-core/src/main/java/com/yahoo/bard/webservice/application/DruidDimensionsLoader.java

@@ -169,6 +169,9 @@ public void invoke(JsonNode rootNode) {
                         }
                     }
                 }
+
+                // Tell the dimension it's been updated
+                dimension.setLastUpdated(DateTime.now());
             }
         };
     }

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/AsyncUtils.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/AsyncUtils.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async;
 
 import com.yahoo.bard.webservice.data.ResultSet;
 import com.yahoo.bard.webservice.druid.model.query.AllGranularity;

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/ErrorUtils.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/ErrorUtils.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async;
 
 import javax.validation.constraints.NotNull;
 import javax.ws.rs.ProcessingException;

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/MetadataHttpResponseChannel.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/MetadataHttpResponseChannel.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async;
 
 import static com.yahoo.bard.webservice.web.handlers.workflow.DruidWorkflow.RESPONSE_WORKFLOW_TIMER;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/ResponseException.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/ResponseException.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async;
 
 import com.yahoo.bard.webservice.druid.model.query.DruidAggregationQuery;
 import com.yahoo.bard.webservice.druid.model.query.DruidQuery;

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/BroadcastChannel.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/broadcastchannels/BroadcastChannel.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.broadcastchannels;
 
 import rx.Observable;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/RedisBroadcastChannel.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/broadcastchannels/RedisBroadcastChannel.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.broadcastchannels;
 
 import com.yahoo.bard.webservice.config.SystemConfig;
 import com.yahoo.bard.webservice.config.SystemConfigProvider;

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/SimpleBroadcastChannel.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/broadcastchannels/SimpleBroadcastChannel.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.broadcastchannels;
 
 import rx.Observable;
 import rx.subjects.Subject;

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/DefaultJobField.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/jobrows/DefaultJobField.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.jobrows;
 
 import com.yahoo.bard.webservice.util.EnumUtils;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/DefaultJobRowBuilder.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/jobrows/DefaultJobRowBuilder.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.jobrows;
 
 import org.joda.time.DateTime;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/DefaultJobStatus.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/jobrows/DefaultJobStatus.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.jobrows;
 
 import java.util.Locale;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/JobField.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/jobrows/JobField.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.jobrows;
 
 /**
  * Represents a column in the table of job metadata.

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/JobRow.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/jobrows/JobRow.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.jobrows;
 
 import com.yahoo.bard.webservice.util.StreamUtils;
 import com.yahoo.bard.webservice.web.ErrorMessageFormat;

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/JobRowBuilder.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/jobrows/JobRowBuilder.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.jobrows;
 
 import javax.ws.rs.container.ContainerRequestContext;
 import javax.ws.rs.core.UriInfo;

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/JobStatus.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/jobrows/JobStatus.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.jobrows;
 
 /**
  * A job status describes the current state of a job (i.e. 'pending', 'success', 'failure').

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/DefaultJobPayloadBuilder.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/payloads/DefaultJobPayloadBuilder.java

@@ -1,7 +1,9 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.payloads;
 
+import com.yahoo.bard.webservice.async.jobs.jobrows.DefaultJobField;
+import com.yahoo.bard.webservice.async.jobs.jobrows.JobRow;
 import com.yahoo.bard.webservice.web.ErrorMessageFormat;
 import com.yahoo.bard.webservice.web.JobRequestFailedException;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/JobPayloadBuilder.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/payloads/JobPayloadBuilder.java

@@ -1,7 +1,8 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.payloads;
 
+import com.yahoo.bard.webservice.async.jobs.jobrows.JobRow;
 import com.yahoo.bard.webservice.web.endpoints.JobsServlet;
 
 import java.util.Map;
@@ -44,7 +45,7 @@ static String getResultsUrl(String ticket, UriInfo uriInfo) {
     /**
      * Get the url for obtaining the job results synchronously.
      * <pre>
-     *     "http://localhost:9998/jobs/ticket1/results?async=never"
+     *     "http://localhost:9998/jobs/ticket1/results?asyncAfter=never"
      * </pre>
      *
      * @param ticket  The ticket that can uniquely identify a Job
@@ -60,11 +61,11 @@ static String getSyncResultsUrl(String ticket, UriInfo uriInfo) {
     }
 
     /**
-     * Get the UriBuilder for the /job/ticket/results endpoint.
+     * Get the UriBuilder for the /jobs/ticket/results endpoint.
      *
      * @param uriInfo  UriInfo of the request
      *
-     * @return the UriBuilder for the /job/ticket/results endpoint
+     * @return the UriBuilder for the /jobs/ticket/results endpoint
      */
     static UriBuilder getResultsBaseUrl(UriInfo uriInfo) {
         return uriInfo.getBaseUriBuilder()

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/ApiJobStore.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/stores/ApiJobStore.java

@@ -1,6 +1,8 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.stores;
+
+import com.yahoo.bard.webservice.async.jobs.jobrows.JobRow;
 
 import rx.Observable;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/ApiJobStoreException.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/stores/ApiJobStoreException.java

@@ -1,6 +1,8 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.stores;
+
+import com.yahoo.bard.webservice.async.jobs.jobrows.JobRow;
 
 /**
  * Class for exception thrown while saving a record in ApiJobStore.

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/HashJobStore.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/stores/HashJobStore.java

@@ -1,6 +1,8 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.stores;
+
+import com.yahoo.bard.webservice.async.jobs.jobrows.JobRow;
 
 import rx.Observable;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/NoOpApiJobStore.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/jobs/stores/NoOpApiJobStore.java

@@ -1,6 +1,8 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.jobs.stores;
+
+import com.yahoo.bard.webservice.async.jobs.jobrows.JobRow;
 
 import rx.Observable;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/HashPreResponseStore.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/preresponses/stores/HashPreResponseStore.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.preresponses.stores;
 
 import com.yahoo.bard.webservice.web.PreResponse;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/NoOpPreResponseStore.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/preresponses/stores/NoOpPreResponseStore.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.preresponses.stores;
 
 import com.yahoo.bard.webservice.web.PreResponse;
 

fili-core/src/main/java/com/yahoo/bard/webservice/jobs/PreResponseStore.java → fili-core/src/main/java/com/yahoo/bard/webservice/async/preresponses/stores/PreResponseStore.java

@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs;
+package com.yahoo.bard.webservice.async.preresponses.stores;
 
 import com.yahoo.bard.webservice.web.PreResponse;