Neo4j Integration

README.md

@@ -303,7 +303,7 @@ For relevant files see `package.json` (section scripts), directories `.test` and
 Zendro is the product of a joint effort between the Forschungszentrum Jülich, Germany and the Comisión Nacional para el Conocimiento y Uso de la Biodiversidad, México, to generate a tool that allows efficiently building data warehouses capable of dealing with diverse data generated by different research groups in the context of the FAIR principles and multidisciplinary projects. The name Zendro comes from the words Zenzontle and Drossel, which are Mexican and German words denoting a mockingbird, a bird capable of “talking” different languages, similar to how Zendro can connect your data warehouse from any programming language or data analysis pipeline.
 
 ### Zendro contributors in alphabetical order
-Francisca Acevedo<sup>1</sup>, Vicente Arriaga<sup>1</sup>, Katja Dohm<sup>3</sup>, Constantin Eiteneuer<sup>2</sup>, Sven Fahrner<sup>2</sup>, Frank Fischer<sup>4</sup>, Asis Hallab<sup>2</sup>, Alicia Mastretta-Yanes<sup>1</sup>, Roland Pieruschka<sup>2</sup>, Alejandro Ponce<sup>1</sup>, Yaxal Ponce<sup>2</sup>, Francisco Ramírez<sup>1</sup>, Irene Ramos<sup>1</sup>, Bernardo Terroba<sup>1</sup>, Tim Rehberg<sup>3</sup>, Verónica Suaste<sup>1</sup>, Björn Usadel<sup>2</sup>, David Velasco<sup>2</sup>, Thomas Voecking<sup>3</sup>
+Francisca Acevedo<sup>1</sup>, Vicente Arriaga<sup>1</sup>, Katja Dohm<sup>3</sup>, Constantin Eiteneuer<sup>2</sup>, Sven Fahrner<sup>2</sup>, Frank Fischer<sup>4</sup>, Asis Hallab<sup>2</sup>, Alicia Mastretta-Yanes<sup>1</sup>, Roland Pieruschka<sup>2</sup>, Alejandro Ponce<sup>1</sup>, Yaxal Ponce<sup>2</sup>, Francisco Ramírez<sup>1</sup>, Irene Ramos<sup>1</sup>, Bernardo Terroba<sup>1</sup>, Tim Rehberg<sup>3</sup>, Verónica Suaste<sup>1</sup>, Björn Usadel<sup>2</sup>, David Velasco<sup>2</sup>, Thomas Voecking<sup>3</sup>, Dan Wang<sup>2</sup>
 
 #### Author affiliations
 1. CONABIO - Comisión Nacional para el Conocimiento y Uso de la Biodiversidad, México

funks.js

@@ -462,8 +462,8 @@ writeAcls = async function (dir_write, models, adapters) {
   //set file name
   let file_name = dir_write + "/acl_rules.js";
   //set names
-  const modelsNames = models.map(item => item[0]);
-  let adminModelsNames = ['role', 'user', 'role_to_user'];
+  const modelsNames = models.map((item) => item[0]);
+  let adminModelsNames = ["role", "user", "role_to_user"];
   //generate
   await generateSection(
     "acl_rules",
@@ -744,7 +744,8 @@ module.exports.parseAssociations = function (dataModel) {
       if (isStandardAssociation) {
         //standard
         associations_info.associations.push(association);
-        association.targetStorageType = association.targetStorageType.toLowerCase();
+        association.targetStorageType =
+          association.targetStorageType.toLowerCase();
         associations_info.foreignKeyAssociations[name] = association.targetKey;
       } else {
         //generic
@@ -977,6 +978,7 @@ generateSections = async function (sections, opts, dir_write) {
       case "models-mongodb":
       case "models-amazonS3":
       case "models-trino":
+      case "models-neo4j":
       //adapters
       case "sql-adapter":
       case "zendro-adapters":
@@ -985,6 +987,7 @@ generateSections = async function (sections, opts, dir_write) {
       case "mongodb-adapter":
       case "amazonS3-adapter":
       case "trino-adapter":
+      case "neo4j-adapter":
         file_name =
           dir_write + "/" + section.dir + "/" + section.fileName + ".js";
         break;
@@ -1090,6 +1093,7 @@ getStorageType = function (dataModel) {
         case "amazon-s3":
         case "trino":
         case "presto":
+        case "neo4j":
         //adapters
         case "sql-adapter":
         case "ddm-adapter":
@@ -1100,6 +1104,7 @@ getStorageType = function (dataModel) {
         case "amazon-s3-adapter":
         case "trino-adapter":
         case "presto-adapter":
+        case "neo4j-adapter":
           //ok
           break;
 
@@ -1111,8 +1116,8 @@ getStorageType = function (dataModel) {
               `ERROR: The attribute 'storageType' has an invalid value. \n
               One of the following types is expected: [sql, distributed-data-model, 
                 zendro-server, generic, sql-adapter, ddm-adapter, zendro-webservice-adapter, generic-adapter,
-                cassandra, mongodb, amazon-s3, trino, presto, cassandra-adapter, mongodb-adapter,
-                amazon-s3-adapter, trino-adapter, presto-adapter].
+                cassandra, mongodb, amazon-s3, trino, presto, neo4j, cassandra-adapter, mongodb-adapter,
+                amazon-s3-adapter, trino-adapter, presto-adapter, neo4j-adapter].
                  But '${dataModel.storageType}' was obtained on ${
                 dataModel.adapterName !== undefined ? "adapter" : "model"
               } '${
@@ -1164,6 +1169,7 @@ module.exports.generateCode = async function (json_dir, dir_write, options) {
     "models/amazonS3",
     "models/trino",
     "models/presto",
+    "models/neo4j",
   ];
   let models = [];
   let adapters = [];
@@ -1504,6 +1510,24 @@ module.exports.generateCode = async function (json_dir, dir_write, options) {
         ];
         break;
 
+      case "neo4j":
+        sections = [
+          { dir: "schemas", template: "schemas", fileName: opts.nameLc },
+          { dir: "resolvers", template: "resolvers", fileName: opts.nameLc },
+          {
+            dir: "models/neo4j",
+            template: "models-neo4j",
+            fileName: opts.nameLc,
+          },
+          {
+            dir: "validations",
+            template: "validations",
+            fileName: opts.nameLc,
+          },
+          { dir: "patches", template: "patches", fileName: opts.nameLc },
+        ];
+        break;
+
       case "zendro-webservice-adapter":
         sections = [
           {
@@ -1608,6 +1632,18 @@ module.exports.generateCode = async function (json_dir, dir_write, options) {
           { dir: "patches", template: "patches", fileName: opts.adapterName },
         ];
         break;
+
+      case "neo4j-adapter":
+        sections = [
+          {
+            dir: "models/adapters",
+            template: "neo4j-adapter",
+            fileName: opts.adapterName,
+          },
+          { dir: "patches", template: "patches", fileName: opts.adapterName },
+        ];
+        break;
+
       default:
         break;
     }
@@ -1643,7 +1679,7 @@ module.exports.generateCode = async function (json_dir, dir_write, options) {
     ) {
       adapters.push(opts.adapterName);
     } else {
-      models.push([opts.name , opts.namePl, opts.nameLc]);
+      models.push([opts.name, opts.namePl, opts.nameLc]);
     }
   }
   //msg

lib/generators-aux.js

@@ -18,6 +18,8 @@ exports.getModelDatabase = function (dataModel) {
     "trino-adapter": "default-trino",
     presto: "default-presto",
     "presto-adapter": "default-presto",
+    neo4j: "default-neo4j",
+    "neo4j-adapter": "default-neo4j",
   };
 
   const storageType = dataModel.storageType.toLowerCase();

test/integration_test_misc/data_models_storage_config2.json

@@ -20,5 +20,13 @@
     "schema":"public",
     "presto_host": "gql_presto",
     "presto_port": "8080"
+  },
+  "default-neo4j": {
+    "storageType": "neo4j",
+    "username": "neo4j",
+    "password": "sciencedb",
+    "database": "neo4j",
+    "host": "gql_neo4j",
+    "port": "7687"
   }
 }

test/integration_test_misc/docker-compose-test.yml

@@ -58,6 +58,8 @@ services:
     depends_on:
       - gql_postgres1
       - gql_mongodb1
+      - gql_cassandra1
+      - gql_minio1
     build:
       context: .
       dockerfile: Dockerfile.graphql_server
@@ -139,10 +141,27 @@ services:
     networks:
       - instance2
 
+  gql_neo4j:
+    image: neo4j
+    container_name: neo4j1
+    user: ${UID_GID}
+    ports:
+      - 7474:7474
+      - 7687:7687
+    volumes:
+      - ./import:/var/lib/neo4j/import
+    environment:
+      NEO4J_AUTH: neo4j/sciencedb
+    networks:
+      - instance2
+
   gql_science_db_graphql_server2:
     container_name: server2
     depends_on:
       - gql_postgres2
+      - gql_presto
+      - gql_trino
+      - gql_neo4j
     build:
       context: .
       dockerfile: Dockerfile.graphql_server

test/integration_test_misc/import/movie.csv

@@ -0,0 +1,7 @@
+movie_id,release,runtime,box_office,is_adult,genres,votes
+m1,2008-12-03T10:15:30Z,130,17845632.32,true,action;thriller,50;200;140;1200;150
+m2,2018-12-03T10:15:30Z,110,15645632.32,true,comedy,1;2;14;1200;15000
+m3,2017-12-03T10:15:30Z,120,12345632.32,false,crime;horror,10;21;40;1000;10000
+m4,2018-12-03T10:15:30Z,120,15645632.32,false,crime;horror,1;2;14;1000;15000
+m5,2002-12-03T10:15:30Z,100,145632.32,false,action;mystery,5;20;150;1300;1500
+m6,2021-12-03T10:15:30Z,109,175632.32,true,wuxia;mystery,2;50;140;800;14000

test/integration_test_misc/integration_test_models_instance2/actor.json

@@ -0,0 +1,27 @@
+{
+    "model" : "actor",
+    "storageType" : "neo4j",
+    "attributes" : {
+        "actor_id": "String",
+        "actor_name": "String",
+        "movie_ids": "[String]"
+    },
+
+    "associations":{
+      "movies":{
+        "type": "to_many",
+        "reverseAssociationType": "to_many",
+        "target": "movie",
+        "targetKey": "actor_ids",
+        "sourceKey": "movie_ids",
+        "keyIn": "actor",
+        "targetStorageType": "neo4j"
+      }
+    },
+
+    "internalId": "actor_id",
+    "id":{
+        "name": "actor_id",
+        "type": "String" 
+    }
+}

test/integration_test_misc/integration_test_models_instance2/director.json

@@ -0,0 +1,24 @@
+{
+    "model": "director",
+    "storageType": "neo4j",
+    "attributes": {
+        "director_id": "String",
+        "director_name": "String",
+        "nationality": "String"
+    },
+    "associations": {
+        "movies": {
+            "type": "to_many",
+            "target": "movie",
+            "targetKey": "director_id",
+            "keyIn": "movie",
+            "targetStorageType": "neo4j",
+            "label": "movie_id"
+        }
+    },
+    "internalId": "director_id",
+    "id": {
+        "name": "director_id",
+        "type": "String"
+    }
+}

test/integration_test_misc/integration_test_models_instance2/dist_director.json

@@ -0,0 +1,20 @@
+{
+    "model": "dist_director",
+    "storageType" : "distributed-data-model",
+    "registry": ["dist_director_instance1"],
+    "attributes": {
+        "director_id": "String",
+        "director_name": "String",
+        "nationality": "String"
+    },
+    "associations": {
+        "dist_movies": {
+            "type": "to_many",
+            "target": "dist_movie",
+            "targetKey": "director_id",
+            "keyIn": "dist_movie",
+            "targetStorageType": "distributed-data-model"
+        }
+    },
+    "internalId": "director_id"
+}

test/integration_test_misc/integration_test_models_instance2/dist_director_instance1.json

@@ -0,0 +1,22 @@
+{
+    "model": "dist_director",
+    "storageType": "neo4j-adapter",
+    "adapterName": "dist_director_instance1",
+    "regex": "instance1",
+    "attributes": {
+      "director_id": "String",
+      "director_name": "String",
+      "nationality": "String"
+    },
+    "associations": {
+      "dist_movies": {
+        "type": "to_many",
+        "target": "dist_movie",
+        "targetKey": "director_id",
+        "keyIn": "dist_movie",
+        "targetStorageType": "distributed-data-model"
+      }
+    },
+    "internalId": "director_id"
+}
+

test/integration_test_misc/integration_test_models_instance2/dist_movie.json

@@ -0,0 +1,27 @@
+{
+    "model": "dist_movie",
+    "storageType" : "distributed-data-model",
+    "registry": ["dist_movie_instance1"],
+    "attributes": {
+        "movie_id": "String",
+        "release": "DateTime",
+        "runtime": "Int",
+        "box_office": "Float",
+        "is_adult": "Boolean",
+        "genres": "[String]",
+        "votes": "[Int]",
+        "director_id": "String"
+    },
+
+    "associations": {
+        "dist_director": {
+            "type": "to_one",
+            "target": "dist_director",
+            "targetKey": "director_id",
+            "keyIn" : "dist_movie",
+            "targetStorageType": "distributed-data-model"
+        }
+    },
+
+    "internalId" : "movie_id"
+}

test/integration_test_misc/integration_test_models_instance2/dist_movie_instance1.json

@@ -0,0 +1,28 @@
+{
+  "model": "dist_movie",
+  "storageType": "neo4j-adapter",
+  "adapterName": "dist_movie_instance1",
+  "regex": "instance1",
+  "attributes": {
+    "movie_id": "String",
+    "release": "DateTime",
+    "runtime": "Int",
+    "box_office": "Float",
+    "is_adult": "Boolean",
+    "genres": "[String]",
+    "votes": "[Int]",
+    "director_id": "String"
+  },
+  "associations": {
+    "dist_director": {
+      "type": "to_one",
+      "target": "dist_director",
+      "targetKey": "director_id",
+      "keyIn": "dist_movie",
+      "targetStorageType": "distributed-data-model"
+    }
+  },
+
+  "internalId": "movie_id"
+}
+