TableExport Spring Controller (#1811)

* Export spring controller * review comments * review comments * review comments * review comments * review comments Co-authored-by: Aaron Klish <aklish@gmail.com>
yahoo · Feb 5, 2021 · 34fbcd2 · 34fbcd2
1 parent 7a16286
commit 34fbcd2
Show file tree

Hide file tree

Showing 28 changed files with 680 additions and 170 deletions.
diff --git a/elide-async/src/main/java/com/yahoo/elide/async/export/formatter/JSONExportFormatter.java b/elide-async/src/main/java/com/yahoo/elide/async/export/formatter/JSONExportFormatter.java
@@ -8,12 +8,20 @@
 import com.yahoo.elide.Elide;
 import com.yahoo.elide.async.models.TableExport;
 import com.yahoo.elide.core.PersistentResource;
+import com.yahoo.elide.core.request.Attribute;
 import com.yahoo.elide.core.request.EntityProjection;
+import com.yahoo.elide.jsonapi.models.Resource;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
 /**
  * JSON output format implementation.
  */
@@ -48,16 +56,31 @@ public static String resourceToJSON(ObjectMapper mapper, PersistentResource reso
         }
 
         StringBuilder str = new StringBuilder();
-
         try {
-            str.append(mapper.writeValueAsString(resource.getObject()));
+            Resource jsonResource = resource.toResource(getRelationships(resource), getAttributes(resource));
+
+            str.append(mapper.writeValueAsString(jsonResource.getAttributes()));
         } catch (JsonProcessingException e) {
             log.error("Exception when converting to JSON {}", e.getMessage());
             throw new IllegalStateException(e);
         }
         return str.toString();
     }
 
+    protected static Map<String, Object> getAttributes(PersistentResource resource) {
+        final Map<String, Object> attributes = new LinkedHashMap<>();
+        final Set<Attribute> attrFields = resource.getRequestScope().getEntityProjection().getAttributes();
+
+        for (Attribute field : attrFields) {
+            attributes.put(field.getName(), resource.getAttribute(field));
+        }
+        return attributes;
+    }
+
+    protected static Map<String, Object> getRelationships(PersistentResource resource) {
+        return Collections.emptyMap();
+    }
+
     @Override
     public String preFormat(EntityProjection projection, TableExport query) {
         return "[";

diff --git a/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java b/elide-async/src/main/java/com/yahoo/elide/async/operation/TableExportOperation.java
@@ -81,9 +81,9 @@ public AsyncAPIResult call() {
             exportResult.setUrl(new URL(generateDownloadURL(exportObj, (RequestScope) scope)));
             exportResult.setRecordCount(recordNumber);
         } catch (BadRequestException e) {
-            exportResult.setMessage("Download url generation failure.");
-        } catch (MalformedURLException e) {
             exportResult.setMessage("EntityProjection generation failure.");
+        } catch (MalformedURLException e) {
+            exportResult.setMessage("Download url generation failure.");
         }  catch (Exception e) {
             exportResult.setMessage(e.getMessage());
         } finally {
@@ -125,6 +125,7 @@ public Observable<PersistentResource> export(TableExport exportObj, RequestScope
             //TODO - Can we have projectionInfo as null?
             RequestScope exportRequestScope = getRequestScope(exportObj, prevScope.getUser(),
                     prevScope.getApiVersion(), tx);
+            exportRequestScope.setEntityProjection(projection);
 
             if (projection != null) {
                 results = PersistentResource.loadRecords(projection, Collections.emptyList(), exportRequestScope);

diff --git a/...nc/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java b/...nc/src/main/java/com/yahoo/elide/async/service/storageengine/FileResultStorageEngine.java
@@ -31,9 +31,6 @@
 public class FileResultStorageEngine implements ResultStorageEngine {
     @Setter private String basePath;
 
-    public FileResultStorageEngine() {
-    }
-
     /**
      * Constructor.
      * @param basePath basePath for storing the files. Can be absolute or relative.
@@ -55,14 +52,14 @@ public TableExport storeResults(TableExport tableExport, Observable<String> resu
                             writer.flush();
                         },
                         throwable -> {
-                            throw new IllegalStateException(throwable);
+                            throw new IllegalStateException(STORE_ERROR, throwable);
                         },
                         () -> {
                             writer.flush();
                         }
                 );
         } catch (IOException e) {
-            throw new IllegalStateException(e);
+            throw new IllegalStateException(STORE_ERROR, e);
         }
 
         return tableExport;
@@ -85,7 +82,7 @@ public boolean hasNext() {
                                     record = reader.readLine();
                                     return record != null;
                                 } catch (IOException e) {
-                                    throw new IllegalStateException(e);
+                                    throw new IllegalStateException(RETRIEVE_ERROR, e);
                                 }
                             }
 
@@ -106,7 +103,7 @@ private BufferedReader getReader(String asyncQueryID) {
             return Files.newBufferedReader(Paths.get(basePath + File.separator + asyncQueryID));
         } catch (IOException e) {
             log.debug(e.getMessage());
-            throw new IllegalStateException("Unable to retrieve results.");
+            throw new IllegalStateException(RETRIEVE_ERROR, e);
         }
     }
 
@@ -115,7 +112,7 @@ private BufferedWriter getWriter(String asyncQueryID) {
             return Files.newBufferedWriter(Paths.get(basePath + File.separator + asyncQueryID));
         } catch (IOException e) {
             log.debug(e.getMessage());
-            throw new IllegalStateException("Unable to store results.");
+            throw new IllegalStateException(STORE_ERROR, e);
         }
     }
 }
diff --git a/...-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java b/...-async/src/main/java/com/yahoo/elide/async/service/storageengine/ResultStorageEngine.java
@@ -13,6 +13,8 @@
  * Utility interface used for storing the results of AsyncQuery for downloads.
  */
 public interface ResultStorageEngine {
+    public static final String RETRIEVE_ERROR = "Unable to retrieve results.";
+    public static final String STORE_ERROR = "Unable to store results.";
 
     /**
      * Stores the result of the query.

diff --git a/elide-async/src/test/java/com/yahoo/elide/async/export/formatter/CSVExportFormatterTest.java b/elide-async/src/test/java/com/yahoo/elide/async/export/formatter/CSVExportFormatterTest.java
@@ -6,6 +6,7 @@
 package com.yahoo.elide.async.export.formatter;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -15,11 +16,13 @@
 import com.yahoo.elide.async.models.ResultType;
 import com.yahoo.elide.async.models.TableExport;
 import com.yahoo.elide.core.PersistentResource;
+import com.yahoo.elide.core.RequestScope;
 import com.yahoo.elide.core.datastore.inmemory.HashMapDataStore;
 import com.yahoo.elide.core.dictionary.EntityDictionary;
 import com.yahoo.elide.core.request.Attribute;
 import com.yahoo.elide.core.request.EntityProjection;
 import com.yahoo.elide.core.security.checks.Check;
+import com.yahoo.elide.jsonapi.models.Resource;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -28,6 +31,7 @@
 import java.nio.file.Path;
 import java.text.SimpleDateFormat;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
@@ -38,6 +42,7 @@ public class CSVExportFormatterTest {
     private static final SimpleDateFormat FORMATTER = new SimpleDateFormat(FORMAT);
     private HashMapDataStore dataStore;
     private Elide elide;
+    private RequestScope scope;
 
     @BeforeEach
     public void setupMocks(@TempDir Path tempDir) {
@@ -49,28 +54,44 @@ public void setupMocks(@TempDir Path tempDir) {
                         .withISO8601Dates("yyyy-MM-dd'T'HH:mm'Z'", TimeZone.getTimeZone("UTC"))
                         .build());
         FORMATTER.setTimeZone(TimeZone.getTimeZone("GMT"));
+        scope = mock(RequestScope.class);
     }
 
     @Test
     public void testResourceToCSV() {
         CSVExportFormatter formatter = new CSVExportFormatter(elide, false);
         TableExport queryObj = new TableExport();
-        String query = "{ tableExport { edges { node { id query queryType requestId principalName status createdOn updatedOn"
-                + " asyncAfterSeconds resultType result} } } }";
+        String query = "{ tableExport { edges { node { query queryType createdOn} } } }";
         String id = "edc4a871-dff2-4054-804e-d80075cf827d";
         queryObj.setId(id);
         queryObj.setQuery(query);
         queryObj.setQueryType(QueryType.GRAPHQL_V1_0);
         queryObj.setResultType(ResultType.CSV);
 
-        String row = "\"edc4a871-dff2-4054-804e-d80075cf827d\", \"{ tableExport { edges { node { id query queryType requestId"
-                + " principalName status createdOn updatedOn asyncAfterSeconds resultType result} } } }\", \"GRAPHQL_V1_0\""
-                + ", \"" + queryObj.getRequestId() + "\", " + "null" + ", \"" + queryObj.getStatus() + "\", \"" + FORMATTER.format(queryObj.getCreatedOn())
-                + "\", \"" + FORMATTER.format(queryObj.getUpdatedOn()) + "\", " + "10.0, \"CSV\", null";
+        String row = "\"{ tableExport { edges { node { query queryType createdOn} } } }\", \"GRAPHQL_V1_0\""
+                + ", \"" + FORMATTER.format(queryObj.getCreatedOn());
 
-        PersistentResource resource = mock(PersistentResource.class);
-        when(resource.getObject()).thenReturn(queryObj);
-        String output = formatter.format(resource, 1);
+        // Prepare EntityProjection
+        Set<Attribute> attributes = new LinkedHashSet<Attribute>();
+        attributes.add(Attribute.builder().type(TableExport.class).name("query").alias("query").build());
+        attributes.add(Attribute.builder().type(TableExport.class).name("queryType").build());
+        attributes.add(Attribute.builder().type(TableExport.class).name("createdOn").build());
+        EntityProjection projection = EntityProjection.builder().type(TableExport.class).attributes(attributes).build();
+
+        Map<String, Object> resourceAttributes = new LinkedHashMap<>();
+        resourceAttributes.put("query", query);
+        resourceAttributes.put("queryType", queryObj.getQueryType());
+        resourceAttributes.put("createdOn", queryObj.getCreatedOn());
+
+        Resource resource = new Resource("tableExport", "0", resourceAttributes, null, null, null);
+        //Resource(type=stats, id=0, attributes={dimension=Bar, measure=150}, relationships=null, links=null, meta=null)
+        PersistentResource persistentResource = mock(PersistentResource.class);
+        when(persistentResource.getObject()).thenReturn(queryObj);
+        when(persistentResource.getRequestScope()).thenReturn(scope);
+        when(persistentResource.toResource(any(), any())).thenReturn(resource);
+        when(scope.getEntityProjection()).thenReturn(projection);
+
+        String output = formatter.format(persistentResource, 1);
         assertEquals(true, output.contains(row));
     }
 

diff --git a/...e-async/src/test/java/com/yahoo/elide/async/export/formatter/JSONExportFormatterTest.java b/...e-async/src/test/java/com/yahoo/elide/async/export/formatter/JSONExportFormatterTest.java
@@ -6,6 +6,7 @@
 package com.yahoo.elide.async.export.formatter;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -15,9 +16,13 @@
 import com.yahoo.elide.async.models.ResultType;
 import com.yahoo.elide.async.models.TableExport;
 import com.yahoo.elide.core.PersistentResource;
+import com.yahoo.elide.core.RequestScope;
 import com.yahoo.elide.core.datastore.inmemory.HashMapDataStore;
 import com.yahoo.elide.core.dictionary.EntityDictionary;
+import com.yahoo.elide.core.request.Attribute;
+import com.yahoo.elide.core.request.EntityProjection;
 import com.yahoo.elide.core.security.checks.Check;
+import com.yahoo.elide.jsonapi.models.Resource;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -26,14 +31,18 @@
 import java.nio.file.Path;
 import java.text.SimpleDateFormat;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.TimeZone;
 
 public class JSONExportFormatterTest {
    public static final String FORMAT = "yyyy-MM-dd'T'HH:mm'Z'";
     private static final SimpleDateFormat FORMATTER = new SimpleDateFormat(FORMAT);
     private HashMapDataStore dataStore;
     private Elide elide;
+    private RequestScope scope;
 
    @BeforeEach
     public void setupMocks(@TempDir Path tempDir) {
@@ -45,28 +54,43 @@ public void setupMocks(@TempDir Path tempDir) {
                         .withISO8601Dates("yyyy-MM-dd'T'HH:mm'Z'", TimeZone.getTimeZone("UTC"))
                         .build());
         FORMATTER.setTimeZone(TimeZone.getTimeZone("GMT"));
+        scope = mock(RequestScope.class);
     }
 
     @Test
     public void testResourceToJSON() {
         JSONExportFormatter formatter = new JSONExportFormatter(elide);
         TableExport queryObj = new TableExport();
-        String query = "/tableExport";
+        String query = "{ tableExport { edges { node { query queryType createdOn} } } }";
         String id = "edc4a871-dff2-4054-804e-d80075cf827d";
         queryObj.setId(id);
         queryObj.setQuery(query);
         queryObj.setQueryType(QueryType.GRAPHQL_V1_0);
         queryObj.setResultType(ResultType.CSV);
 
-        String start = "{\"id\":\"edc4a871-dff2-4054-804e-d80075cf827d\",\"query\":\"/tableExport\","
-                + "\"queryType\":\"GRAPHQL_V1_0\",\"requestId\":\"" + queryObj.getRequestId() + "\",\"principalName\":null"
-                + ",\"status\":\""  + queryObj.getStatus() + "\",\"createdOn\":\"" + FORMATTER.format(queryObj.getCreatedOn())
-                + "\",\"updatedOn\":\"" + FORMATTER.format(queryObj.getUpdatedOn()) + "\",\"asyncAfterSeconds\":10,\"resultType\":\"CSV\","
-                + "\"result\":null}";
+        String start = "{\"query\":\"{ tableExport { edges { node { query queryType createdOn} } } }\","
+                + "\"queryType\":\"GRAPHQL_V1_0\",\"createdOn\":\"" + FORMATTER.format(queryObj.getCreatedOn()) + "\"}";
 
-        PersistentResource resource = mock(PersistentResource.class);
-        when(resource.getObject()).thenReturn(queryObj);
-        String output = formatter.format(resource, 1);
+        // Prepare EntityProjection
+        Set<Attribute> attributes = new LinkedHashSet<Attribute>();
+        attributes.add(Attribute.builder().type(TableExport.class).name("query").alias("query").build());
+        attributes.add(Attribute.builder().type(TableExport.class).name("queryType").build());
+        attributes.add(Attribute.builder().type(TableExport.class).name("createdOn").build());
+        EntityProjection projection = EntityProjection.builder().type(TableExport.class).attributes(attributes).build();
+
+        Map<String, Object> resourceAttributes = new LinkedHashMap<>();
+        resourceAttributes.put("query", query);
+        resourceAttributes.put("queryType", queryObj.getQueryType());
+        resourceAttributes.put("createdOn", queryObj.getCreatedOn());
+
+        Resource resource = new Resource("tableExport", "0", resourceAttributes, null, null, null);
+        PersistentResource persistentResource = mock(PersistentResource.class);
+        when(persistentResource.getObject()).thenReturn(queryObj);
+        when(persistentResource.getRequestScope()).thenReturn(scope);
+        when(persistentResource.toResource(any(), any())).thenReturn(resource);
+        when(scope.getEntityProjection()).thenReturn(projection);
+
+        String output = formatter.format(persistentResource, 1);
         assertEquals(true, output.contains(start));
     }
 }
diff --git a/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java b/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java
@@ -1342,12 +1342,23 @@ public Resource toResource(EntityProjection projection) {
      */
     private Resource toResource(final Supplier<Map<String, Relationship>> relationshipSupplier,
                                final Supplier<Map<String, Object>> attributeSupplier) {
+        return toResource(relationshipSupplier.get(), attributeSupplier.get());
+    }
+
+    /**
+     * Convert a persistent resource to a resource.
+     * @param relationships The relationships
+     * @param attributes The attributes
+     * @return The Resource
+     */
+    public Resource toResource(final Map<String, Relationship> relationships,
+                               final Map<String, Object> attributes) {
         final Resource resource = new Resource(typeName, (obj == null)
                 ? uuid.orElseThrow(
                 () -> new InvalidEntityBodyException("No id found on object"))
                 : dictionary.getId(obj));
-        resource.setRelationships(relationshipSupplier.get());
-        resource.setAttributes(attributeSupplier.get());
+        resource.setRelationships(relationships);
+        resource.setAttributes(attributes);
         if (requestScope.getElideSettings().isEnableJsonLinks()) {
             resource.setLinks(requestScope.getElideSettings().getJsonApiLinks().getResourceLevelLinks(this));
         }

diff --git a/elide-core/src/main/java/com/yahoo/elide/core/exceptions/HttpStatus.java b/elide-core/src/main/java/com/yahoo/elide/core/exceptions/HttpStatus.java
@@ -16,6 +16,7 @@ public class HttpStatus {
     public static final int SC_BAD_REQUEST = 400;
     public static final int SC_FORBIDDEN = 403;
     public static final int SC_NOT_FOUND = 404;
+    public static final int SC_METHOD_NOT_ALLOWED = 405;
     public static final int SC_TIMEOUT = 408;
     public static final int SC_LOCKED = 423;
     public static final int SC_INTERNAL_SERVER_ERROR = 500;

diff --git a/elide-core/src/main/java/com/yahoo/elide/core/utils/coerce/converters/URLSerde.java b/elide-core/src/main/java/com/yahoo/elide/core/utils/coerce/converters/URLSerde.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2021, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+package com.yahoo.elide.core.utils.coerce.converters;
+
+import com.yahoo.elide.core.exceptions.InvalidValueException;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+@ElideTypeConverter(type = URL.class, name = "URL")
+public class URLSerde implements Serde<String, URL> {
+
+    @Override
+    public URL deserialize(String val) {
+        URL url;
+        try {
+            url = new URL(val);
+        } catch (MalformedURLException e) {
+            throw new InvalidValueException("Invalid URL " + val);
+        }
+        return url;
+    }
+
+    @Override
+    public String serialize(URL val) {
+        return val.toString();
+    }
+}
diff --git a/elide-core/src/test/java/com/yahoo/elide/core/utils/ClassScannerTest.java b/elide-core/src/test/java/com/yahoo/elide/core/utils/ClassScannerTest.java
@@ -18,7 +18,7 @@ public class ClassScannerTest {
     @Test
     public void testGetAllClasses() {
         Set<Class<?>> classes = ClassScanner.getAllClasses("com.yahoo.elide.core.utils");
-        assertEquals(29, classes.size());
+        assertEquals(31, classes.size());
         assertTrue(classes.contains(ClassScannerTest.class));
     }