Precompute limit added.

antalbalint · Dec 15, 2015 · b8cd35f · b8cd35f
1 parent 82c5b99
commit b8cd35f
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 19 deletions.
diff --git a/src/main/java/org/mineotaur/common/GraphDatabaseUtils.java b/src/main/java/org/mineotaur/common/GraphDatabaseUtils.java
@@ -9,8 +9,6 @@
 import org.neo4j.graphdb.schema.IndexDefinition;
 import org.neo4j.graphdb.schema.Schema;
 
-import java.util.concurrent.TimeUnit;
-
 public class GraphDatabaseUtils {
 
     /**
@@ -68,11 +66,11 @@ public static void createIndex(GraphDatabaseService db, Label label, String name
             tx.success();
         }
 
-        try (Transaction tx = db.beginTx()) {
+        /*try (Transaction tx = db.beginTx()) {
             Schema schema = db.schema();
             schema.awaitIndexOnline( indexDefinition, 10, TimeUnit.SECONDS );
             tx.success();
-        }
+        }*/
     }
 
 }
diff --git a/src/main/java/org/mineotaur/importer/DatabaseGenerator.java b/src/main/java/org/mineotaur/importer/DatabaseGenerator.java
@@ -26,7 +26,9 @@
 import org.neo4j.graphdb.*;
 import org.neo4j.tooling.GlobalGraphOperations;
 
-import java.io.*;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.util.*;
 /**
  * Class containing fields and methods to generate a graph database from the input provided by the user.
@@ -171,10 +173,14 @@ protected void precomputeOptimized(GraphDatabaseService db, GlobalGraphOperation
         try {
             tx = db.beginTx();
             RelationshipType rt = relationships.get(group).get(descriptive);
+            List<Node> groupNodes = new ArrayList<>();
             Iterator<Node> nodes = ggo.getAllNodesWithLabel(groupLabel).iterator();
             while (nodes.hasNext()) {
+                groupNodes.add(nodes.next());
+            }
+            for (Node node: groupNodes) {
                 Mineotaur.LOGGER.info("Precomputing: " + (count++));
-                Node node = nodes.next();
+//                Node node = nodes.next();
                 Iterator<Relationship> rels = node.getRelationships(rt).iterator();
                 Map<String, List<Double>> features = new HashMap<>();
                 Map<String, List<String>> innerfilter = new HashMap<>();

diff --git a/src/main/java/org/mineotaur/importer/DatabaseGeneratorFromOmero.java b/src/main/java/org/mineotaur/importer/DatabaseGeneratorFromOmero.java
@@ -56,6 +56,7 @@ public class DatabaseGeneratorFromOmero extends DatabaseGenerator{
     private String geneForLabel;
     private String hitColumn;
     private int rowsToFetch;
+    private int precomputeLimit;
 
     /*public DatabaseGeneratorFromOmero(String hostName, String userName, String password, Long screenId) {
         this.password = password;
@@ -111,6 +112,12 @@ protected void processProperties() {
         else {
             limit = (Integer) DefaultProperty.LIMIT.getValue();
         }
+        if (properties.containsKey("precompute_limit")) {
+            precomputeLimit = Integer.valueOf(properties.getString("precompute_limit"));
+        }
+        else {
+            precomputeLimit = (Integer) DefaultProperty.PRECOMPUTE_LIMIT.getValue();
+        }
         if (properties.containsKey("row_prefetch")) {
             rowsToFetch = Integer.valueOf(properties.getString("row_prefetch"));
         }
@@ -129,12 +136,7 @@ protected void processProperties() {
         else {
             throw new IllegalArgumentException("No data file name provided.");
         }
-        if (properties.containsKey("dataFile")) {
-            dataFileName = properties.getString("dataFile");
-        }
-        else {
-            throw new IllegalArgumentException("No data file name provided.");
-        }
+
         if (properties.containsKey("groupID")) {
             groupId = properties.getString("groupID");
         }
@@ -400,7 +402,10 @@ else if ("reference".equals(colName) || "name".equals(colName) || "alternativeNa
             long[] experimentColumn = ((ImageColumn) cols[experimentID]).values;
             double[][] descriptiveColumn = ((DoubleArrayColumn) cols[descriptiveID]).values;
             double[] filterColumn = descriptiveColumn[filter];
-            filterProps.add(descriptiveHeader[filter]);
+            if (!filterProps.contains(descriptiveHeader[filter])) {
+                filterProps.add(descriptiveHeader[filter]);
+            }
+
             int count = 0;
             Transaction tx = null;
             try {
@@ -416,9 +421,9 @@ else if ("reference".equals(colName) || "name".equals(colName) || "alternativeNa
                         strain.setProperty("strainID", sid);
                         addProperties(strain, cols, strainIDs, i);
                         strains.put(sid, strain);
-                        Mineotaur.LOGGER.info("Strain created with imageID: " + sid);
+                        Mineotaur.LOGGER.info("Strain created with ID: " + sid);
                     } else {
-                        Mineotaur.LOGGER.info("Strain loaded with imageID: " + sid);
+                        Mineotaur.LOGGER.info("Strain loaded with ID: " + sid);
                     }
                     Long imageID = experimentColumn[i];
                     Node experiment = images.get(imageID);
@@ -518,21 +523,20 @@ public void generateDatabase() {
             readTable(dataFile);
             Mineotaur.LOGGER.info("Processing label data.");
             labelGenes();
+            closeConnection();
             Mineotaur.LOGGER.info(filterProps.toString());
             if (filterProps != null && !filterProps.isEmpty()) {
                 Mineotaur.LOGGER.info("Processing descriptive filters.");
                 createFilters(db, ggo, groupLabel, descriptiveLabel, DefaultRelationships.GROUP_CELL.getRelationshipType(), filterProps, limit);
             }
             Mineotaur.LOGGER.info("Connecting images to strains.");
             getImageIDs(DefaultRelationships.GROUP_EXPERIMENT.getRelationshipType());
-            Mineotaur.LOGGER.info("Precomputing nodes.");
-            precomputedLabel = DynamicLabel.label(group+COLLECTED);
-            precomputeOptimized(db, ggo, groupLabel, descriptiveLabel, precomputedLabel, relationships, filterProps, group, descriptive, limit);
+
             Mineotaur.LOGGER.info("Creating indices.");
             GraphDatabaseUtils.createIndex(db, groupLabel, groupName);
             Mineotaur.LOGGER.info("Precomputing nodes.");
             precomputedLabel = DynamicLabel.label(group+COLLECTED);
-            precomputeOptimized(db, ggo, groupLabel, descriptiveLabel, precomputedLabel, relationships, filterProps, group, descriptive, limit);
+            precomputeOptimized(db, ggo, groupLabel, descriptiveLabel, precomputedLabel, relationships, filterProps, group, descriptive, precomputeLimit);
             Mineotaur.LOGGER.info("Generating property files.");
             storeConfigurationFiles();
             /*generateFeatureNameList();

diff --git a/src/main/java/org/mineotaur/importer/DefaultProperty.java b/src/main/java/org/mineotaur/importer/DefaultProperty.java
@@ -10,6 +10,7 @@ public enum DefaultProperty {
     SEPARATOR("\t"),
     TOTAL_MEMORY("4G"),
     LIMIT(5000),
+    PRECOMPUTE_LIMIT(1000),
     ROW_PREFETCH(1000),
     OVERWRITE(false);