Merge pull request #58 from vsreekanti/source_key

[GROUND-3] Allow for user-generated unique IDs

ground-core/scripts/cassandra/cassandra.cql

@@ -32,6 +32,7 @@ create table version_history_dag (
 
 create table structure (
     item_id bigint,
+    source_key varchar,
     name varchar,
     PRIMARY KEY (item_id, name)
 );
@@ -71,6 +72,7 @@ create table rich_version_tag (
 
 create table edge (
     item_id bigint,
+    source_key varchar,
     from_node_id bigint,
     to_node_id bigint,
     name varchar,
@@ -79,12 +81,14 @@ create table edge (
 
 create table node (
     item_id bigint,
+    source_key varchar,
     name varchar,
     PRIMARY KEY (item_id, name)
 );
 
 create table graph (
     item_id bigint,
+    source_key varchar,
     name varchar,
     PRIMARY KEY (item_id, name)
 );
@@ -118,12 +122,14 @@ create table graph_version_edge (
 
 create table principal (
     node_id bigint,
+    source_key varchar,
     name varchar,
     PRIMARY KEY (node_id, name)
 );
 
 create table lineage_edge (
     item_id bigint,
+    source_key varchar,
     name varchar,
     PRIMARY KEY (item_id, name)
 );
@@ -138,6 +144,7 @@ create table lineage_edge_version (
 
 create table lineage_graph (
     item_id bigint,
+    source_key varchar,
     name varchar,
     PRIMARY KEY (item_id, name)
 );

ground-core/scripts/postgres/postgres.sql

@@ -34,7 +34,8 @@ create table version_history_dag (
 
 create table structure (
     item_id bigint NOT NULL PRIMARY KEY REFERENCES item(id),
-    name varchar NOT NULL UNIQUE
+    source_key varchar UNIQUE,
+    name varchar
 );
 
 create table structure_version (
@@ -72,20 +73,23 @@ create table rich_version_tag (
 
 create table node (
     item_id bigint NOT NULL PRIMARY KEY REFERENCES item(id),
-    name varchar NOT NULL UNIQUE
+    source_key varchar UNIQUE,
+    name varchar
 );
 
 create table edge (
     item_id bigint NOT NULL PRIMARY KEY REFERENCES item(id),
+    source_key varchar UNIQUE,
     from_node_id bigint NOT NULL REFERENCES node(item_id),
     to_node_id bigint NOT NULL REFERENCES node(item_id),
-    name varchar NOT NULL UNIQUE
+    name varchar
 );
 
 
 create table graph (
     item_id bigint NOT NULL PRIMARY KEY REFERENCES item(id),
-    name varchar NOT NULL UNIQUE
+    source_key varchar UNIQUE,
+    name varchar
 );
 
 create table node_version (
@@ -118,12 +122,14 @@ create table graph_version_edge (
 
 create table principal (
     node_id bigint NOT NULL PRIMARY KEY REFERENCES node(item_id),
-    name varchar NOT NULL UNIQUE REFERENCES node(name)
+    source_key varchar UNIQUE,
+    name varchar
 );
 
 create table lineage_edge (
     item_id bigint NOT NULL PRIMARY KEY REFERENCES item(id),
-    name varchar NOT NULL UNIQUE
+    source_key varchar UNIQUE,
+    name varchar
 );
 
 create table lineage_edge_version (
@@ -136,7 +142,8 @@ create table lineage_edge_version (
 
 create table lineage_graph (
     item_id bigint NOT NULL PRIMARY KEY REFERENCES item(id),
-    name varchar NOT NULL UNIQUE
+    source_key varchar UNIQUE,
+    name varchar
 );
 
 create table lineage_graph_version (

ground-core/src/main/java/edu/berkeley/ground/dao/models/EdgeFactory.java

@@ -22,8 +22,11 @@
 import java.util.Map;
 
 public abstract class EdgeFactory {
-  public abstract Edge create(String name, long fromNodeId, long toNodeId, Map<String, Tag> tags)
-      throws GroundException;
+  public abstract Edge create(String name,
+                              String sourceKey,
+                              long fromNodeId,
+                              long toNodeId,
+                              Map<String, Tag> tags) throws GroundException;
 
   public abstract Edge retrieveFromDatabase(String name) throws GroundException;
 
@@ -33,10 +36,11 @@ public abstract Edge create(String name, long fromNodeId, long toNodeId, Map<Str
 
   protected static Edge construct(long id,
                                   String name,
+                                  String sourceKey,
                                   long fromNodeId,
                                   long toNodeId,
                                   Map<String, Tag> tags) {
 
-    return new Edge(id, name, fromNodeId, toNodeId, tags);
+    return new Edge(id, name, sourceKey, fromNodeId, toNodeId, tags);
   }
 }

ground-core/src/main/java/edu/berkeley/ground/dao/models/GraphFactory.java

@@ -22,13 +22,15 @@
 import java.util.Map;
 
 public abstract class GraphFactory {
-  public abstract Graph create(String name, Map<String, Tag> tags) throws GroundException;
+  public abstract Graph create(String name,
+                               String sourceKey,
+                               Map<String, Tag> tags) throws GroundException;
 
   public abstract Graph retrieveFromDatabase(String name) throws GroundException;
 
   public abstract void update(long itemId, long childId, List<Long> parentIds) throws GroundException;
 
-  protected static Graph construct(long id, String name, Map<String, Tag> tags) {
-    return new Graph(id, name, tags);
+  protected static Graph construct(long id, String name, String sourceKey, Map<String, Tag> tags) {
+    return new Graph(id, name, sourceKey, tags);
   }
 }

ground-core/src/main/java/edu/berkeley/ground/dao/models/NodeFactory.java

@@ -22,15 +22,18 @@
 import java.util.Map;
 
 public abstract class NodeFactory {
-  public abstract Node create(String name, Map<String, Tag> tags) throws GroundException;
+  public abstract Node create(String name,
+                              String sourceKey,
+                              Map<String, Tag> tags)
+      throws GroundException;
 
   public abstract Node retrieveFromDatabase(String name) throws GroundException;
 
   public abstract void update(long itemId, long childId, List<Long> parentIds) throws GroundException;
 
   public abstract List<Long> getLeaves(String name) throws GroundException;
 
-  protected static Node construct(long id, String name, Map<String, Tag> tags) {
-    return new Node(id, name, tags);
+  protected static Node construct(long id, String name, String sourceKey, Map<String, Tag> tags) {
+    return new Node(id, name, sourceKey, tags);
   }
 }

ground-core/src/main/java/edu/berkeley/ground/dao/models/StructureFactory.java

@@ -22,15 +22,21 @@
 import java.util.Map;
 
 public abstract class StructureFactory {
-  public abstract Structure create(String name, Map<String, Tag> tags) throws GroundException;
+  public abstract Structure create(String name,
+                                   String sourceKey,
+                                   Map<String, Tag> tags)
+      throws GroundException;
 
   public abstract Structure retrieveFromDatabase(String name) throws GroundException;
 
   public abstract void update(long itemId, long childId, List<Long> parentIds) throws GroundException;
 
   public abstract List<Long> getLeaves(String name) throws GroundException;
 
-  protected static Structure construct(long id, String name, Map<String, Tag> tags) {
-    return new Structure(id, name, tags);
+  protected static Structure construct(long id,
+                                       String name,
+                                       String sourceKey,
+                                       Map<String, Tag> tags) {
+    return new Structure(id, name, sourceKey, tags);
   }
 }

ground-core/src/main/java/edu/berkeley/ground/dao/models/cassandra/CassandraEdgeFactory.java

@@ -62,8 +62,11 @@ public void setEdgeVersionFactory(CassandraEdgeVersionFactory edgeVersionFactory
     this.edgeVersionFactory = edgeVersionFactory;
   }
 
-  public Edge create(String name, long fromNodeId, long toNodeId, Map<String, Tag> tags)
-      throws GroundException {
+  public Edge create(String name,
+                     String sourceKey,
+                     long fromNodeId,
+                     long toNodeId,
+                     Map<String, Tag> tags) throws GroundException {
     try {
       long uniqueId = this.idGenerator.generateItemId();
 
@@ -74,12 +77,13 @@ public Edge create(String name, long fromNodeId, long toNodeId, Map<String, Tag>
       insertions.add(new DbDataContainer("item_id", GroundType.LONG, uniqueId));
       insertions.add(new DbDataContainer("from_node_id", GroundType.LONG, fromNodeId));
       insertions.add(new DbDataContainer("to_node_id", GroundType.LONG, toNodeId));
+      insertions.add(new DbDataContainer("source_key", GroundType.STRING, sourceKey));
 
       this.dbClient.insert("edge", insertions);
 
       this.dbClient.commit();
       LOGGER.info("Created edge " + name + ".");
-      return EdgeFactory.construct(uniqueId, name, fromNodeId, toNodeId, tags);
+      return EdgeFactory.construct(uniqueId, name, sourceKey, fromNodeId, toNodeId, tags);
     } catch (GroundException e) {
       this.dbClient.abort();
 
@@ -118,15 +122,18 @@ private Edge retrieveByPredicate(String fieldName, Object value, GroundType valu
       }
 
       long id = resultSet.getLong(0);
-      String name = resultSet.getString("name");
-      Map<String, Tag> tags = this.itemFactory.retrieveFromDatabase(id).getTags();
       long fromNodeId = resultSet.getLong("from_node_id");
       long toNodeId = resultSet.getLong("to_node_id");
 
+      String name = resultSet.getString("name");
+      String sourceKey = resultSet.getString("source_key");
+
+      Map<String, Tag> tags = this.itemFactory.retrieveFromDatabase(id).getTags();
+
       this.dbClient.commit();
       LOGGER.info("Retrieved edge " + value + ".");
 
-      return EdgeFactory.construct(id, name, fromNodeId, toNodeId, tags);
+      return EdgeFactory.construct(id, name, sourceKey, fromNodeId, toNodeId, tags);
     } catch (GroundException e) {
       this.dbClient.abort();
 

ground-core/src/main/java/edu/berkeley/ground/dao/models/cassandra/CassandraGraphFactory.java

@@ -47,7 +47,7 @@ public CassandraGraphFactory(CassandraItemFactory itemFactory, CassandraClient d
     this.idGenerator = idGenerator;
   }
 
-  public Graph create(String name, Map<String, Tag> tags) throws GroundException {
+  public Graph create(String name, String sourceKey, Map<String, Tag> tags) throws GroundException {
     try {
       long uniqueId = this.idGenerator.generateItemId();
 
@@ -56,13 +56,14 @@ public Graph create(String name, Map<String, Tag> tags) throws GroundException {
       List<DbDataContainer> insertions = new ArrayList<>();
       insertions.add(new DbDataContainer("name", GroundType.STRING, name));
       insertions.add(new DbDataContainer("item_id", GroundType.LONG, uniqueId));
+      insertions.add(new DbDataContainer("source_key", GroundType.STRING, sourceKey));
 
       this.dbClient.insert("graph", insertions);
 
       this.dbClient.commit();
       LOGGER.info("Created graph " + name + ".");
 
-      return GraphFactory.construct(uniqueId, name, tags);
+      return GraphFactory.construct(uniqueId, name, sourceKey, tags);
     } catch (GroundException e) {
       this.dbClient.abort();
 
@@ -91,12 +92,14 @@ public Graph retrieveFromDatabase(String name) throws GroundException {
       }
 
       long id = resultSet.getLong(0);
+      String sourceKey = resultSet.getString("source_key");
+
       Map<String, Tag> tags = this.itemFactory.retrieveFromDatabase(id).getTags();
 
       this.dbClient.commit();
       LOGGER.info("Retrieved graph " + name + ".");
 
-      return GraphFactory.construct(id, name, tags);
+      return GraphFactory.construct(id, name, sourceKey, tags);
     } catch (GroundException e) {
       this.dbClient.abort();
 

ground-core/src/main/java/edu/berkeley/ground/dao/models/cassandra/CassandraNodeFactory.java

@@ -47,7 +47,7 @@ public CassandraNodeFactory(CassandraItemFactory itemFactory, CassandraClient db
     this.idGenerator = idGenerator;
   }
 
-  public Node create(String name, Map<String, Tag> tags) throws GroundException {
+  public Node create(String name, String sourceKey, Map<String, Tag> tags) throws GroundException {
     try {
       long uniqueId = this.idGenerator.generateItemId();
 
@@ -56,13 +56,14 @@ public Node create(String name, Map<String, Tag> tags) throws GroundException {
       List<DbDataContainer> insertions = new ArrayList<>();
       insertions.add(new DbDataContainer("name", GroundType.STRING, name));
       insertions.add(new DbDataContainer("item_id", GroundType.LONG, uniqueId));
+      insertions.add(new DbDataContainer("source_key", GroundType.STRING, sourceKey));
 
       this.dbClient.insert("node", insertions);
 
       this.dbClient.commit();
       LOGGER.info("Created node " + name + ".");
 
-      return NodeFactory.construct(uniqueId, name, tags);
+      return NodeFactory.construct(uniqueId, name, sourceKey, tags);
     } catch (GroundException e) {
       this.dbClient.abort();
 
@@ -106,12 +107,14 @@ public Node retrieveFromDatabase(String name) throws GroundException {
       }
 
       long id = resultSet.getLong(0);
+      String sourceKey = resultSet.getString("source_key");
+
       Map<String, Tag> tags = this.itemFactory.retrieveFromDatabase(id).getTags();
 
       this.dbClient.commit();
       LOGGER.info("Retrieved node " + name + ".");
 
-      return NodeFactory.construct(id, name, tags);
+      return NodeFactory.construct(id, name, sourceKey, tags);
     } catch (GroundException e) {
       this.dbClient.abort();
 

ground-core/src/main/java/edu/berkeley/ground/dao/models/cassandra/CassandraStructureFactory.java

@@ -47,7 +47,8 @@ public CassandraStructureFactory(CassandraItemFactory itemFactory, CassandraClie
     this.idGenerator = idGenerator;
   }
 
-  public Structure create(String name, Map<String, Tag> tags) throws GroundException {
+  public Structure create(String name, String sourceKey, Map<String, Tag> tags)
+      throws GroundException {
     try {
       long uniqueId = this.idGenerator.generateItemId();
 
@@ -56,13 +57,14 @@ public Structure create(String name, Map<String, Tag> tags) throws GroundExcepti
       List<DbDataContainer> insertions = new ArrayList<>();
       insertions.add(new DbDataContainer("name", GroundType.STRING, name));
       insertions.add(new DbDataContainer("item_id", GroundType.LONG, uniqueId));
+      insertions.add(new DbDataContainer("source_key", GroundType.STRING, sourceKey));
 
       this.dbClient.insert("structure", insertions);
 
       this.dbClient.commit();
       LOGGER.info("Created structure " + name + ".");
 
-      return StructureFactory.construct(uniqueId, name, tags);
+      return StructureFactory.construct(uniqueId, name, sourceKey, tags);
     } catch (GroundException e) {
       this.dbClient.abort();
 
@@ -106,12 +108,14 @@ public Structure retrieveFromDatabase(String name) throws GroundException {
       }
 
       long id = resultSet.getLong(0);
+      String sourceKey = resultSet.getString("source_key");
+
       Map<String, Tag> tags = this.itemFactory.retrieveFromDatabase(id).getTags();
 
       this.dbClient.commit();
       LOGGER.info("Retrieved structure " + name + ".");
 
-      return StructureFactory.construct(id, name, tags);
+      return StructureFactory.construct(id, name, sourceKey, tags);
     } catch (GroundException e) {
       this.dbClient.abort();