Merge pull request #728 from molgenis/feat/m10506-ui-link-file-view-edit

Feat: M10506 UI link file view
molgenis · May 2, 2024 · 6068c2e · 6068c2e
2 parents 307e212 + 0ab3976
commit 6068c2e
Show file tree

Hide file tree

Showing 7 changed files with 125 additions and 36 deletions.
diff --git a/armadillo/src/main/java/org/molgenis/armadillo/storage/LocalStorageService.java b/armadillo/src/main/java/org/molgenis/armadillo/storage/LocalStorageService.java
@@ -245,7 +245,19 @@ public List<Map<String, String>> preview(
       Objects.requireNonNull(objectName);
 
       Path objectPath = getPathIfObjectExists(bucketName, objectName);
-      return ParquetUtils.previewRecords(objectPath, rowLimit, columnLimit);
+      if (objectPath.toString().endsWith(PARQUET)) {
+        return ParquetUtils.previewRecords(objectPath, rowLimit, columnLimit, new String[0]);
+      } else if (objectPath.toString().endsWith(LINK_FILE)) {
+        ArmadilloLinkFile linkFile = getArmadilloLinkFileFromName(bucketName, objectName);
+        String srcProject = linkFile.getSourceProject();
+        String srcObject = linkFile.getSourceObject();
+        String[] variables = linkFile.getVariables().split(",");
+        Path srcObjectPath = getPathIfObjectExists(SHARED_PREFIX + srcProject, srcObject + PARQUET);
+        return ParquetUtils.previewRecords(srcObjectPath, rowLimit, columnLimit, variables);
+      } else {
+        throw new StorageException(
+            format("Preview not supported for: %s/%s", bucketName, objectName));
+      }
     } catch (Exception e) {
       throw new StorageException(e);
     }

diff --git a/armadillo/src/main/java/org/molgenis/armadillo/storage/ParquetUtils.java b/armadillo/src/main/java/org/molgenis/armadillo/storage/ParquetUtils.java
@@ -1,11 +1,10 @@
 package org.molgenis.armadillo.storage;
 
-import static java.lang.Math.min;
-
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.IntStream;
 import org.apache.parquet.column.page.PageReadStore;
 import org.apache.parquet.example.data.Group;
@@ -18,38 +17,67 @@
 import org.apache.parquet.schema.MessageType;
 
 public class ParquetUtils {
-  public static List<Map<String, String>> previewRecords(Path path, int rowLimit, int columnLimit)
-      throws IOException {
+  public static List<Map<String, String>> previewRecords(
+      Path path, int rowLimit, int columnLimit, String[] variables) throws IOException {
     List<Map<String, String>> result = new ArrayList<>();
     LocalInputFile file = new LocalInputFile(path);
     try (ParquetFileReader reader = ParquetFileReader.open(file)) {
-      MessageType schema = reader.getFooter().getFileMetaData().getSchema();
-      MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
-      PageReadStore store = reader.readNextRowGroup();
-      RecordReader<Group> recordReader =
-          columnIO.getRecordReader(store, new GroupRecordConverter(schema));
-      int fieldSize = schema.getFields().size();
+      MessageType schema = getSchemaFromReader(reader);
+      RecordReader<Group> recordReader = getRecordReader(schema, reader);
+      List<String> columns = getColumnsFromSchema(schema);
+
       for (int i = 0; i < rowLimit; i++) {
-        SimpleGroup group = (SimpleGroup) recordReader.read();
-        Map<String, String> row = new LinkedHashMap<>();
-        for (int fieldIndex = 0; fieldIndex < min(fieldSize, columnLimit); fieldIndex++) {
-          try {
-            row.put(schema.getFieldName(fieldIndex), group.getValueToString(fieldIndex, 0));
-          } catch (Exception e) {
-            row.put(schema.getFieldName(fieldIndex), "???");
-          }
-        }
-        result.add(row);
+        result.add(getPreviewRow(recordReader, schema, columns, columnLimit, variables));
       }
     }
     return result;
   }
 
+  private static Map<String, String> getPreviewRow(
+      RecordReader<Group> recordReader,
+      MessageType schema,
+      List<String> columns,
+      int columnLimit,
+      String[] variables) {
+    AtomicInteger colCount = new AtomicInteger();
+    SimpleGroup group = (SimpleGroup) recordReader.read();
+    Map<String, String> row = new LinkedHashMap<>();
+    columns.forEach(
+        column -> {
+          if (colCount.get() < columnLimit) {
+            if (variables.length == 0 || Arrays.stream(variables).toList().contains(column)) {
+              colCount.getAndIncrement();
+              try {
+                row.put(column, group.getValueToString(schema.getFieldIndex(column), 0));
+              } catch (Exception e) {
+                row.put(column, "NA");
+              }
+            }
+          }
+        });
+    return row;
+  }
+
+  private static List<String> getColumnsFromSchema(MessageType schema) {
+    return IntStream.range(0, schema.getFieldCount()).mapToObj(schema::getFieldName).toList();
+  }
+
+  private static MessageType getSchemaFromReader(ParquetFileReader reader) {
+    return reader.getFooter().getFileMetaData().getSchema();
+  }
+
+  private static RecordReader<Group> getRecordReader(MessageType schema, ParquetFileReader reader)
+      throws IOException {
+    MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
+    PageReadStore store = reader.readNextRowGroup();
+    return columnIO.getRecordReader(store, new GroupRecordConverter(schema));
+  }
+
   public static List<String> getColumns(Path path) throws IOException {
     LocalInputFile file = new LocalInputFile(path);
     try (ParquetFileReader reader = ParquetFileReader.open(file)) {
-      var schema = reader.getFooter().getFileMetaData().getSchema();
-      return IntStream.range(0, schema.getFieldCount()).mapToObj(schema::getFieldName).toList();
+      var schema = getSchemaFromReader(reader);
+      return getColumnsFromSchema(schema);
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
@@ -58,10 +86,10 @@ public static List<String> getColumns(Path path) throws IOException {
   public static Map<String, String> retrieveDimensions(Path path) throws FileNotFoundException {
     LocalInputFile file = new LocalInputFile(path);
     try (ParquetFileReader reader = ParquetFileReader.open(file)) {
-      MessageType schema = reader.getFooter().getFileMetaData().getSchema();
+      MessageType schema = getSchemaFromReader(reader);
       int numberOfColumns = schema.getFields().size();
       long numberOfRows = reader.getRecordCount();
-      Map<String, String> dimensions = new HashMap();
+      Map<String, String> dimensions = new HashMap<>();
       dimensions.put("rows", Long.toString(numberOfRows));
       dimensions.put("columns", Integer.toString(numberOfColumns));
       return dimensions;

diff --git a/armadillo/src/test/java/org/molgenis/armadillo/storage/LocalStorageServiceTest.java b/armadillo/src/test/java/org/molgenis/armadillo/storage/LocalStorageServiceTest.java
@@ -344,7 +344,7 @@ void testPreview() {
     MockedStatic<ParquetUtils> mockedParquetUtils = Mockito.mockStatic(ParquetUtils.class);
     localStorageService.preview(bucket, object, 10, 10);
     Path path = localStorageService.getObjectPathSafely(bucket, object);
-    mockedParquetUtils.verify(() -> ParquetUtils.previewRecords(path, 10, 10));
+    mockedParquetUtils.verify(() -> ParquetUtils.previewRecords(path, 10, 10, new String[0]));
     mockedParquetUtils.close();
   }
 }
diff --git a/armadillo/src/test/java/org/molgenis/armadillo/storage/ParquetUtilsTest.java b/armadillo/src/test/java/org/molgenis/armadillo/storage/ParquetUtilsTest.java
@@ -1,34 +1,57 @@
 package org.molgenis.armadillo.storage;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import org.junit.jupiter.api.Test;
 
 public class ParquetUtilsTest {
   @Test
   public void testParquetPreview() throws IOException, URISyntaxException {
-    Path path = Path.of(this.getClass().getClassLoader().getResource("patient.parquet").toURI());
-    List<Map<String, String>> preview = ParquetUtils.previewRecords(path, 10, 10);
+    Path path =
+        Path.of(
+            Objects.requireNonNull(this.getClass().getClassLoader().getResource("patient.parquet"))
+                .toURI());
+    List<Map<String, String>> preview = ParquetUtils.previewRecords(path, 10, 10, new String[0]);
     assertEquals("Patient1", preview.get(0).get("name"));
+    assertEquals(preview.size(), 10);
   }
 
   @Test
   void testRetrieveDimensions() throws URISyntaxException, FileNotFoundException {
-    Path path = Path.of(this.getClass().getClassLoader().getResource("patient.parquet").toURI());
+    Path path =
+        Path.of(
+            Objects.requireNonNull(this.getClass().getClassLoader().getResource("patient.parquet"))
+                .toURI());
     Map<String, String> dimensions = ParquetUtils.retrieveDimensions(path);
     assertEquals("3", dimensions.get("columns"));
     assertEquals("11", dimensions.get("rows"));
   }
 
   @Test
   void testGetColumns() throws URISyntaxException, IOException {
-    Path path = Path.of(this.getClass().getClassLoader().getResource("patient.parquet").toURI());
+    Path path =
+        Path.of(
+            Objects.requireNonNull(this.getClass().getClassLoader().getResource("patient.parquet"))
+                .toURI());
     assertEquals(List.of("id", "age", "name"), ParquetUtils.getColumns(path));
   }
+
+  @Test
+  void testPreviewWithVariables() throws URISyntaxException, IOException {
+    Path path =
+        Path.of(
+            Objects.requireNonNull(this.getClass().getClassLoader().getResource("patient.parquet"))
+                .toURI());
+    List<Map<String, String>> preview =
+        ParquetUtils.previewRecords(path, 10, 10, new String[] {"age"});
+    assertEquals("24", preview.get(0).get("age"));
+    assertNull(preview.get(0).get("name"));
+  }
 }
diff --git a/ui/src/helpers/utils.ts b/ui/src/helpers/utils.ts
@@ -167,6 +167,10 @@ export function isNonTableType(item: string): boolean {
   return !isTableType(item);
 }
 
+export function isLinkFileType(item: string): boolean {
+  return item.endsWith(".alf");
+}
+
 export function getRestructuredProject(
   projectContent: StringArray,
   projectId: string

diff --git a/ui/src/views/ProjectsExplorer.vue b/ui/src/views/ProjectsExplorer.vue
@@ -118,7 +118,11 @@
               v-if="loading_preview"
               class="pt-3 mt-3"
             ></LoadingSpinner>
-            <div v-if="isNonTableType(selectedFile)">
+            <div
+              v-if="
+                isNonTableType(selectedFile) && !isLinkFileType(selectedFile)
+              "
+            >
               <div class="fst-italic">
                 No preview available for: {{ selectedFile }} ({{ fileSize }})
               </div>
@@ -130,15 +134,15 @@
                   `${dataSizeRows}x${dataSizeColumns}`
                 }})
                 <button
-                  v-if="!createLinkFromSrc"
+                  v-if="!createLinkFromSrc && isTableType(selectedFile)"
                   @click="createLinkFromSrc = true"
                   type="button"
                   class="btn btn-primary btn-sm m-1"
                 >
                   <i class="bi bi-box-arrow-in-up-right"></i> Create view
                 </button>
                 <button
-                  v-else
+                  v-else-if="!isLinkFileType(selectedFile)"
                   @click="createLinkFromSrc = false"
                   type="button"
                   class="btn btn-danger btn-sm m-1"
@@ -189,6 +193,7 @@ import {
 } from "@/api/api";
 import {
   isEmptyObject,
+  isLinkFileType,
   isTableType,
   isNonTableType,
   getRestructuredProject,
@@ -271,7 +276,10 @@ export default defineComponent({
   watch: {
     selectedFile() {
       this.resetCreateLinkFile();
-      if (this.isTableType(this.selectedFile)) {
+      if (
+        this.isTableType(this.selectedFile) ||
+        this.isLinkFileType(this.selectedFile)
+      ) {
         this.loading_preview = true;
         previewObject(
           this.projectId,
@@ -315,6 +323,7 @@ export default defineComponent({
   },
   methods: {
     isTableType,
+    isLinkFileType,
     isNonTableType,
     askIfPreviewIsEmpty() {
       return isEmptyObject(this.filePreview[0]);

diff --git a/ui/tests/unit/helpers/utils.spec.ts b/ui/tests/unit/helpers/utils.spec.ts
@@ -14,7 +14,8 @@ import {
   isTableType,
   isNonTableType,
   getRestructuredProject,
-  getTablesFromListOfFiles
+  getTablesFromListOfFiles,
+  isLinkFileType
 } from "@/helpers/utils";
 import { StringObject } from "@/types/types";
 
@@ -227,6 +228,18 @@ describe("utils", () => {
       expect(actual).toBe(false);
     });
   });
+
+  describe("isLinkFileType", () => {
+    it("should return true if item has parquet extension", () => {
+      const actual = isLinkFileType("test.alf");
+      expect(actual).toBe(true);
+    });
+    it("should return false if item doesnt have parquet extension", () => {
+      const actual = isLinkFileType("test.somethingelse");
+      expect(actual).toBe(false);
+    });
+  }); 
+
   describe("isNonTableType", () => {
     it("should return false if item has parquet extension", () => {
       const actual = isNonTableType("test.parquet");