Added DataGroupReadTest in preparation for DM-14029.

- introduce test category - add howto section for Test in Developer Setup.txt doc.
Caltech-IPAC · Jun 1, 2018 · 06594d7 · 06594d7
1 parent 5afc0d9
commit 06594d7
Show file tree

Hide file tree

Showing 6 changed files with 173 additions and 17 deletions.
diff --git a/buildScript/gwt_webapp.gincl b/buildScript/gwt_webapp.gincl
@@ -52,32 +52,40 @@ test {
   description= 'Run Java and JavaScript unit test'
   group = DEV_GROUP
 
-  dependsOn jsTest
-  // set a system property for the test JVM(s)
-  systemProperty 'some.prop', 'value'
+  dependsOn loadConfig, jsTest
+  // should rerun this everytime.  properties could be modified from external files.
+  outputs.upToDateWhen { false }
 
   workingDir = rootDir
 
   // explicitly include or exclude tests
   include 'edu/**'
 
+  if (project.hasProperty("category")) {
+    if (project.category) {
+      logger.lifecycle("Running test for category: " + project.category)
+      useJUnit {
+        includeCategories project.category
+      }
+    }
+  } else {
+    useJUnit {
+      excludeCategories 'edu.caltech.ipac.TestCategory$Perf', 'edu.caltech.ipac.TestCategory$Integration'
+    }
+  }
+
   // show standard out and standard error of the test JVM(s) on the console
   testLogging.showStandardStreams = true
 
   // set heap size for the test JVM(s)
   minHeapSize = "128m"
-  maxHeapSize = "512m"
+  maxHeapSize = "2g"
 
   // listen to events in the test execution lifecycle
   beforeTest { descriptor ->
     logger.lifecycle("Running test: " + descriptor)
   }
 
-  // listen to standard out and standard error of the test JVM(s)
-  onOutput { descriptor, event ->
-    logger.lifecycle("Test: " + descriptor + " produced standard out/err: " + event.message )
-  }
-
   doFirst {
     // things needed before test run
     copy {

diff --git a/gradle.properties b/gradle.properties
@@ -1 +1 @@
-org.gradle.jvmargs=-Xms256m -Xmx2048m
+org.gradle.jvmargs=-Xms256m -Xmx8048m
diff --git a/src/firefly/test/edu/caltech/ipac/TestCategory.java b/src/firefly/test/edu/caltech/ipac/TestCategory.java
@@ -0,0 +1,9 @@
+package edu.caltech.ipac;
+
+/**
+ * category markers.  use for include/exclude tests
+ */
+public class TestCategory {
+    public interface Perf{}
+    public interface Integration{}
+}
diff --git a/src/firefly/test/edu/caltech/ipac/firefly/server/util/tables/DataGroupReadTest.java b/src/firefly/test/edu/caltech/ipac/firefly/server/util/tables/DataGroupReadTest.java
@@ -0,0 +1,129 @@
+/*
+ * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
+ */
+package edu.caltech.ipac.firefly.server.util.tables;
+
+import edu.caltech.ipac.astro.FITSTableReader;
+import edu.caltech.ipac.util.VoTableUtil;
+import edu.caltech.ipac.TestCategory;
+import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupReader;
+import edu.caltech.ipac.firefly.util.FileLoader;
+import edu.caltech.ipac.util.DataGroup;
+import edu.caltech.ipac.util.DataType;
+import edu.caltech.ipac.util.IpacTableUtil;
+
+import nom.tam.fits.FitsException;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryPoolMXBean;
+import java.lang.management.MemoryUsage;
+import java.util.List;
+
+/**
+ * @author loi
+ * @version $Id: IpacTableParser.java,v 1.18 2011/12/08 19:34:02 loi Exp $
+ */
+public class DataGroupReadTest {
+
+    private static final File largeIpacTable = FileLoader.resolveFile(DataGroupReadTest.class, "50k.tbl");
+    private static final File ipacTable = FileLoader.resolveFile(DataGroupReadTest.class, "DataGroupTest.tbl");
+    private static final File voTable = FileLoader.resolveFile(DataGroupReadTest.class, "p3am_cdd.xml");
+    private static final File fitsTable = FileLoader.resolveFile(DataGroupReadTest.class, "lsst-table.fits");
+
+    @Test
+    public void ipacTable() throws IOException {
+        DataGroup data = DataGroupReader.read(ipacTable);
+        Assert.assertNotNull(data);
+
+        // test col header
+        DataType ra = data.getDataDefintion("ra");
+        Assert.assertEquals("column name", "ra", ra.getKeyName());
+        Assert.assertEquals("column unit", "deg", ra.getDataUnit());
+        Assert.assertEquals("column desc", "null", ra.getNullString());
+        Assert.assertEquals("column type", "double", ra.getTypeDesc());
+
+        // test col attributes
+        String width = data.getAttribute(IpacTableUtil.makeAttribKey(IpacTableUtil.WIDTH_TAG, "designation")).getValue();
+        String prefWidth = data.getAttribute(IpacTableUtil.makeAttribKey(IpacTableUtil.PREF_WIDTH_TAG, "designation")).getValue();
+        String sortable = data.getAttribute(IpacTableUtil.makeAttribKey(IpacTableUtil.SORTABLE_TAG, "designation")).getValue();
+        String filterable = data.getAttribute(IpacTableUtil.makeAttribKey(IpacTableUtil.FILTERABLE_TAG, "designation")).getValue();
+        String visibility = data.getAttribute(IpacTableUtil.makeAttribKey(IpacTableUtil.VISI_TAG, "designation")).getValue();
+        Assert.assertEquals("column width", "100", width);
+        Assert.assertEquals("column prefWidth", "10", prefWidth);
+        Assert.assertEquals("column sortable", "false", sortable);
+        Assert.assertEquals("column filterable", "true", filterable);
+        Assert.assertEquals("column visibility", "hidden", visibility);
+
+        // random value test
+        Assert.assertEquals("cell (ra, 4)", 10.744471, data.get(4).getDataElement("ra"));           // in the middle
+        Assert.assertEquals("cell (h_msigcom, 0)", null, data.get(0).getDataElement("h_msigcom"));  // first row with null value
+        Assert.assertEquals("cell (err_min, 15)", 0.29, data.get(15).getDataElement("err_min"));    // last row
+    }
+
+    @Test
+    public void voTable() throws IOException {
+        DataGroup data = VoTableUtil.voToDataGroups(voTable.getPath())[0];
+        Assert.assertNotNull(data);
+        Assert.assertEquals("Number of rows", 40, data.size());
+        Assert.assertEquals("Number of columns", 20, data.getDataDefinitions().length);
+
+        // random value test
+        Assert.assertEquals("cell (magzip2, 0)", 0.006, data.get(0).getDataElement("magzip2"));       // first row
+        Assert.assertEquals("cell (magzip, 9)", 19.5, data.get(9).getDataElement("magzip"));       // in the middle
+        Assert.assertEquals("cell (magzip, 39)", 13.0, data.get(39).getDataElement("magzip"));       // last row
+    }
+
+    @Test
+    public void fitsTable() throws IOException, FitsException {
+        DataGroup data = FITSTableReader.convertFitsToDataGroup(fitsTable.getPath(), null, null, null, 0);
+        Assert.assertNotNull(data);
+        Assert.assertEquals("Number of rows", 765, data.size());
+        Assert.assertEquals("Number of columns", 52, data.getDataDefinitions().length);
+
+        // random value test
+        Assert.assertEquals("cell (coord_ra, 0)", 6.13602424, getDouble(data.get(0).getDataElement("coord_ra")), 0.00000001);       // first row
+        Assert.assertEquals("cell (footprint, 24)", 25, data.get(24).getDataElement("footprint"));       // in the middle
+        Assert.assertEquals("cell (base_GaussianCentroid_x, 764)", 850.043863, getDouble(data.get(764).getDataElement("base_GaussianCentroid_x")), 0.000001);       // last row
+    }
+
+    @Category({TestCategory.Perf.class})
+    @Test
+    public void perfTest() throws IOException {
+
+        Runtime.getRuntime().gc();
+        long beginMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
+        long start = System.currentTimeMillis();
+
+        DataGroup data = DataGroupReader.read(largeIpacTable);
+
+        long elapsed = System.currentTimeMillis() - start;
+        long usedB4Gc = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory() - beginMem;
+        Runtime.getRuntime().gc();
+        long usedAfGc = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory() - beginMem;
+
+        System.out.printf("Memory temp=%.2fMB  final:%.2fMB  elapsed:%.2fsecs %n", usedB4Gc/1000/1000.0, usedAfGc/1000/1000.0, elapsed/1000.0);
+        Assert.assertNotNull(data);
+        Assert.assertEquals("Number of rows", 49933, data.size());
+
+        List<MemoryPoolMXBean> pools = ManagementFactory.getMemoryPoolMXBeans();
+        for (MemoryPoolMXBean pool : pools) {
+            MemoryUsage peak = pool.getPeakUsage();
+            System.out.printf("Peak %s memory used: %,d %n", pool.getName(), peak.getUsed());
+        }
+    }
+
+
+    private static double getDouble(Object val) {
+        if (val instanceof Double) {
+            return (Double) val;
+        } else {
+            return Double.parseDouble(String.valueOf(val));
+        }
+    }
+
+}
diff --git a/.../server/util/ipactable/IpacTableTest.java → ...fly/server/util/tables/IpacTableTest.java b/.../server/util/ipactable/IpacTableTest.java → ...fly/server/util/tables/IpacTableTest.java
@@ -1,12 +1,16 @@
 /*
  * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
  */
-package edu.caltech.ipac.firefly.server.util.ipactable;
+package edu.caltech.ipac.firefly.server.util.tables;
 
 import edu.caltech.ipac.firefly.data.DecimateInfo;
 import edu.caltech.ipac.firefly.data.SortInfo;
 import edu.caltech.ipac.firefly.data.TableServerRequest;
 import edu.caltech.ipac.firefly.server.query.DecimationProcessor;
+import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
+import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupReader;
+import edu.caltech.ipac.firefly.server.util.ipactable.JsonTableUtil;
+import edu.caltech.ipac.firefly.server.util.ipactable.TableDef;
 import edu.caltech.ipac.firefly.util.FileLoader;
 import edu.caltech.ipac.util.DataGroup;
 import edu.caltech.ipac.util.FileUtil;
@@ -26,8 +30,8 @@
  */
 public class IpacTableTest {
 
-    private static final File ipacTable = FileLoader.resolveFile(IpacTableTest.class,  "test_data.tbl");
-    private static final File jsonResults = FileLoader.resolveFile(IpacTableTest.class,"test_data.json");
+    private static final File ipacTable = FileLoader.resolveFile(IpacTableTest.class,  "IpacTableTest.tbl");
+    private static final File jsonResults = FileLoader.resolveFile(IpacTableTest.class,"IpacTableTest.json");
     private static TableServerRequest request;
 
     @BeforeClass

diff --git a/src/firefly/test/edu/caltech/ipac/firefly/util/FileLoader.java b/src/firefly/test/edu/caltech/ipac/firefly/util/FileLoader.java
@@ -31,6 +31,14 @@ public class FileLoader {
 
     public static final String TEST_DATA_ROOT = "firefly_test_data/";
 
+    /**
+     * @return the path to firefly_test_data.
+     */
+    public static String getTestDataRoot() {
+        String rootPath = Paths.get("").toAbsolutePath().getParent().toUri().getPath();
+        return rootPath + TEST_DATA_ROOT;
+    }
+
     /**
      * This method returns the data path for where the test class is located and where is the data is stored.
      * @param cls
@@ -39,15 +47,13 @@ public class FileLoader {
      */
     public static String getDataPath(Class cls) {
 
-        String rootPath = Paths.get("").toAbsolutePath().getParent().toUri().getPath();
-        String testDataPath = TEST_DATA_ROOT+cls.getCanonicalName().replaceAll("\\.", "/")
+        String clsDataPath = cls.getCanonicalName().replaceAll("\\.", "/")
                 .replace(cls.getSimpleName(), "");
 
-        String dataPath = rootPath + testDataPath;
+        String dataPath = getTestDataRoot() + clsDataPath;
         return dataPath;
     }
 
-
     public static FitsRead loadFitsRead(Class cls, String fitsFile)  {
 
         try {