Implement dimension metadata to indicate storage strategy

CHANGELOG.md

@@ -10,14 +10,18 @@ Current
 
 ### Added:
 
+- [Implement dimension metadata to indicate storage strategy](https://github.com/yahoo/fili/pull/558)
+    * In order to allow clients to be notified if a dimension's values are browsable and searchable, a storage strategy
+      metadata is added to dimension.
+
 - [Refactor ApiRequest](https://github.com/yahoo/fili/pull/538)
     * Add inteface layer to each type of API request class. The types of API request under the refactor are
         - `TablesApiRequest`
         - `DimensionApiRequest`
         - `SlicesApiRequest`
         - `MetricsApiRequest`
         - `JobsApiRequest`
-
+        
 - [Implement Query Split Logging](https://github.com/yahoo/fili/pull/537)
     * Include metrics in logging to allow for better evaluation of the impact of caching for split queries.
         - Currently there is only a binary flag (`BardQueryInfo.cached`) that is inconsistently set for split queries

fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/Dimension.java

@@ -2,6 +2,7 @@
 // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
 package com.yahoo.bard.webservice.data.dimension;
 
+import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy;
 import com.yahoo.bard.webservice.druid.serializers.DimensionToDefaultDimensionSpec;
 
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@@ -149,6 +150,15 @@ public interface Dimension {
      */
     String getLongName();
 
+    /**
+     * Returns the storage strategy of the dimension.
+     * <p>
+     * See {@link com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy}.
+     *
+     * @return the storage strategy of the dimension.
+     */
+    StorageStrategy getStorageStrategy();
+
     /**
      * Get the cardinality of the dimension.
      *

fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/impl/KeyValueStoreDimension.java

@@ -9,6 +9,7 @@
 import com.yahoo.bard.webservice.data.dimension.DimensionRow;
 import com.yahoo.bard.webservice.data.dimension.KeyValueStore;
 import com.yahoo.bard.webservice.data.dimension.SearchProvider;
+import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy;
 import com.yahoo.bard.webservice.util.DimensionStoreKeyUtils;
 
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -65,6 +66,7 @@ public class KeyValueStoreDimension implements Dimension {
     private final DimensionField key;
 
     private final boolean isAggregatable;
+    private final StorageStrategy storageStrategy;
 
     /**
      * Constructor.
@@ -78,6 +80,8 @@ public class KeyValueStoreDimension implements Dimension {
      * @param searchProvider  Search provider over the metadata for the dimension
      * @param defaultDimensionFields  Default fields for the dimension
      * @param isAggregatable  Whether the dimension is aggregatable
+     * @param storageStrategy  Strategy of how dimension is loaded. See
+     * {@link com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy}
      */
     public KeyValueStoreDimension(
             String dimensionName,
@@ -88,7 +92,8 @@ public KeyValueStoreDimension(
             @NotNull KeyValueStore keyValueStore,
             SearchProvider searchProvider,
             @NotNull LinkedHashSet<DimensionField> defaultDimensionFields,
-            boolean isAggregatable
+            boolean isAggregatable,
+            StorageStrategy storageStrategy
     ) {
         this.apiName = dimensionName;
         this.longName = longName;
@@ -112,6 +117,7 @@ public KeyValueStoreDimension(
         this.searchProvider.setKeyValueStore(keyValueStore);
 
         this.isAggregatable = isAggregatable;
+        this.storageStrategy = storageStrategy;
     }
 
     /**
@@ -142,7 +148,8 @@ public KeyValueStoreDimension(
                 keyValueStore,
                 searchProvider,
                 new LinkedHashSet<>(),
-                true
+                true,
+                StorageStrategy.LOADED
         );
     }
 
@@ -175,7 +182,8 @@ public KeyValueStoreDimension(
                 keyValueStore,
                 searchProvider,
                 new LinkedHashSet<>(),
-                isAggregatable
+                isAggregatable,
+                StorageStrategy.LOADED
         );
     }
 
@@ -243,7 +251,8 @@ public KeyValueStoreDimension(
                 keyValueStore,
                 searchProvider,
                 new LinkedHashSet<>(),
-                true
+                true,
+                StorageStrategy.LOADED
         );
         this.addAllDimensionRows(dimensionRows);
     }
@@ -263,7 +272,8 @@ public KeyValueStoreDimension(DimensionConfig dimensionConfig) {
                 dimensionConfig.getKeyValueStore(),
                 dimensionConfig.getSearchProvider(),
                 dimensionConfig.getDefaultDimensionFields(),
-                dimensionConfig.isAggregatable()
+                dimensionConfig.isAggregatable(),
+                StorageStrategy.LOADED
         );
     }
 
@@ -344,6 +354,11 @@ public String getLongName() {
         return longName;
     }
 
+    @Override
+    public StorageStrategy getStorageStrategy() {
+        return storageStrategy;
+    }
+
     @Override
     public int getCardinality() {
         return searchProvider.getDimensionCardinality();
@@ -541,6 +556,29 @@ public boolean isAggregatable() {
         return isAggregatable;
     }
 
+    /**
+     * Constructs a new KeyValueStoreDimension with specified
+     * {@link com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy}.
+     *
+     * @param storageStrategy  The specified StorageStrategy
+     *
+     * @return the new KeyValueStoreDimension with the specified StorageStrategy
+     */
+    public KeyValueStoreDimension withStorageStrategy(StorageStrategy storageStrategy) {
+        return new KeyValueStoreDimension(
+                apiName,
+                longName,
+                category,
+                description,
+                dimensionFields,
+                keyValueStore,
+                searchProvider,
+                defaultDimensionFields,
+                isAggregatable,
+                storageStrategy
+        );
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) { return true; }

fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/metadata/StorageStrategy.java

@@ -0,0 +1,28 @@
+// Copyright 2017 Yahoo Inc.
+// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
+package com.yahoo.bard.webservice.data.dimension.metadata;
+
+/**
+ * Allows clients to be notified if a dimension's values are browsable and searchable.
+ * <p>
+ * For the non-loaded dimensions(A "non-loaded dimension" is a fact based dimension, where we don't load any domain data
+ * for it, but simply send queries directly to druid), we need to surface metadata to the UI. If there aren't dimension
+ * values loaded, we can't validate when we build filters and you can't use the dimension values endpoint to browse
+ * values. UI needs to know that a dimension isn't going to be validated and searched. The way that UI knows about this
+ * is through this <tt>StorageStrategy</tt>
+ */
+public enum StorageStrategy {
+    /**
+     * Loaded dimension.
+     */
+    LOADED,
+    /**
+     * Non-loaded dimension.
+     */
+    NONE;
+
+    @Override
+    public String toString() {
+        return name();
+    }
+}

fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/DimensionsServlet.java

@@ -375,6 +375,7 @@ public static Map<String, Object> getDimensionSummaryView(Dimension dimension, f
         resultRow.put("longName", dimension.getLongName());
         resultRow.put("uri", getDimensionUrl(dimension, uriInfo));
         resultRow.put("cardinality", dimension.getCardinality());
+        resultRow.put("storageStrategy", dimension.getStorageStrategy());
         return resultRow;
     }
 
@@ -400,6 +401,7 @@ public static Map<String, Object> getDimensionFullView(
         resultRow.put("fields", dimension.getDimensionFields());
         resultRow.put("values", getDimensionValuesUrl(dimension, uriInfo));
         resultRow.put("cardinality", dimension.getCardinality());
+        resultRow.put("storageStrategy", dimension.getStorageStrategy());
         resultRow.put(
                 "tables",
                 TablesServlet.getLogicalTableListSummaryView(

fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/DimensionToNameSerializerSpec.groovy

@@ -7,6 +7,7 @@ import com.yahoo.bard.webservice.data.dimension.Dimension
 import com.yahoo.bard.webservice.data.dimension.DimensionField
 import com.yahoo.bard.webservice.data.dimension.DimensionRow
 import com.yahoo.bard.webservice.data.dimension.SearchProvider
+import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy
 
 import com.fasterxml.jackson.annotation.JsonValue
 import com.fasterxml.jackson.databind.ObjectMapper
@@ -30,100 +31,105 @@ class DimensionToNameSerializerSpec extends Specification {
     }
 
     @JsonSerialize
-    public static class DummyDimension implements Dimension {
+    static class DummyDimension implements Dimension {
         @JsonValue
-        public String example() {
-            return "woot";
+        String example() {
+            return "woot"
         }
 
         @Override
-        public void setLastUpdated(DateTime lastUpdated) {
+        void setLastUpdated(DateTime lastUpdated) {
 
         }
 
         @Override
-        public String getApiName() {
-            return "abc";
+        String getApiName() {
+            return "abc"
         }
 
         @Override
-        public String getDescription() {
-            return null;
+        String getDescription() {
+            return null
         }
 
         @Override
-        public DateTime getLastUpdated() {
-            return null;
+        DateTime getLastUpdated() {
+            return null
         }
 
         @Override
-        public LinkedHashSet<DimensionField> getDimensionFields() {
-            return null;
+        LinkedHashSet<DimensionField> getDimensionFields() {
+            return null
         }
 
         @Override
-        public LinkedHashSet<DimensionField> getDefaultDimensionFields() {
-            return null;
+        LinkedHashSet<DimensionField> getDefaultDimensionFields() {
+            return null
         }
 
         @Override
-        public DimensionField getFieldByName(String name) {
-            return null;
+        DimensionField getFieldByName(String name) {
+            return null
         }
 
         @Override
-        public SearchProvider getSearchProvider() {
-            return null;
+        SearchProvider getSearchProvider() {
+            return null
         }
 
         @Override
-        public void addDimensionRow(DimensionRow dimensionRow) {
+        void addDimensionRow(DimensionRow dimensionRow) {
 
         }
 
         @Override
-        public void addAllDimensionRows(Set<DimensionRow> dimensionRows) {
+        void addAllDimensionRows(Set<DimensionRow> dimensionRows) {
 
         }
 
         @Override
-        public DimensionRow findDimensionRowByKeyValue(String value) {
-            return null;
+        DimensionRow findDimensionRowByKeyValue(String value) {
+            return null
         }
 
         @Override
-        public DimensionField getKey() {
-            return null;
+        DimensionField getKey() {
+            return null
         }
 
         @Override
-        public DimensionRow parseDimensionRow(Map<String, String> fieldNameValueMap) {
-            return null;
+        DimensionRow parseDimensionRow(Map<String, String> fieldNameValueMap) {
+            return null
         }
 
         @Override
-        public DimensionRow createEmptyDimensionRow(String keyFieldValue) {
-            return null;
+        DimensionRow createEmptyDimensionRow(String keyFieldValue) {
+            return null
         }
 
         @Override
-        public String getCategory() {
-            return null;
+        String getCategory() {
+            return null
         }
 
         @Override
-        public String getLongName() {
-            return null;
+        String getLongName() {
+            return null
         }
 
         @Override
-        public int getCardinality() {
-            return 0;
+        int getCardinality() {
+            return 0
         }
 
         @Override
-        public boolean isAggregatable() {
-            return false;
+        boolean isAggregatable() {
+            return false
+        }
+
+        @Override
+        StorageStrategy getStorageStrategy() {
+            return null
         }
     }
 }

fili-core/src/test/groovy/com/yahoo/bard/webservice/web/AggregatabilityValidationSpec.groovy

@@ -11,6 +11,7 @@ import com.yahoo.bard.webservice.data.dimension.DimensionField
 import com.yahoo.bard.webservice.data.dimension.MapStoreManager
 import com.yahoo.bard.webservice.data.dimension.impl.KeyValueStoreDimension
 import com.yahoo.bard.webservice.data.dimension.impl.ScanSearchProviderManager
+import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy
 import com.yahoo.bard.webservice.data.metric.MetricDictionary
 import com.yahoo.bard.webservice.table.LogicalTable
 import com.yahoo.bard.webservice.table.TableGroup
@@ -62,7 +63,8 @@ class AggregatabilityValidationSpec extends Specification {
                     MapStoreManager.getInstance(name),
                     ScanSearchProviderManager.getInstance(name),
                     new LinkedHashSet<DimensionField>(),
-                    false
+                    false,
+                    StorageStrategy.LOADED
             )
             keyValueStoreDimension.setLastUpdated(new DateTime(10000))
             dimensionDict.add(keyValueStoreDimension)