Skip to content

Commit

Permalink
Fixes neo4j-contrib#2333: Add systemdb export metadata procedure (neo…
Browse files Browse the repository at this point in the history
…4j-contrib#2374)

* Fixes neo4j-contrib#2333: Add systemdb export metadata procedure

* added System annotation - formatProperties
  • Loading branch information
vga91 committed Mar 23, 2022
1 parent 512ad98 commit 42cab04
Show file tree
Hide file tree
Showing 28 changed files with 680 additions and 16 deletions.
3 changes: 3 additions & 0 deletions core/src/main/java/apoc/SystemPropertyKeys.java
Expand Up @@ -20,6 +20,9 @@ public enum SystemPropertyKeys {
params,
paused,

// dv
data,

// uuid handler
label,
addToSetLabel,
Expand Down
Expand Up @@ -106,7 +106,7 @@ public static String formatNotUniqueLabels(String id, Node node, Map<String, Set
result.append(label(labelName));
}
}
return result.length() > 0 ? result.substring(2) : "";
return formatToString(result);
}

private static String getNodeIdLabels(Node node, Map<String, Set<String>> uniqueConstraints, Set<String> indexNames) {
Expand Down Expand Up @@ -145,12 +145,12 @@ public static String formatNodeProperties(String id, Node node, Map<String, Set<
result.append(", ");
result.append(formatPropertyName(id, UNIQUE_ID_PROP, node.getId(), jsonStyle));
}
return result.length() > 0 ? result.substring(2) : "";
return formatToString(result);
}

public static String formatRelationshipProperties(String id, Relationship relationship, boolean jsonStyle) {
StringBuilder result = formatProperties(id, relationship.getAllProperties(), jsonStyle);
return result.length() > 0 ? result.substring(2) : "";
return formatToString(result);
}

public static String formatNotUniqueProperties(String id, Node node, Map<String, Set<String>> uniqueConstraints, Set<String> indexedProperties, boolean jsonStyle) {
Expand All @@ -171,16 +171,26 @@ public static String formatNotUniqueProperties(String id, Node node, Map<String,
result.append(", ");
result.append(formatPropertyName(id, key, properties.get(key), jsonStyle));
}
return formatToString(result);
}

public static String formatToString(StringBuilder result) {
return result.length() > 0 ? result.substring(2) : "";
}

public static StringBuilder formatProperties(Map<String, Object> properties) {
return formatProperties("", properties, true);
}

public static StringBuilder formatProperties(String id, Map<String, Object> properties, boolean jsonStyle) {
StringBuilder result = new StringBuilder(100);
List<String> keys = Iterables.asList(properties.keySet());
Collections.sort(keys);
for (String prop : keys) {
result.append(", ");
result.append(formatPropertyName(id, prop, properties.get(prop), jsonStyle));
if (properties != null) {
List<String> keys = Iterables.asList(properties.keySet());
Collections.sort(keys);
for (String prop : keys) {
result.append(", ");
result.append(formatPropertyName(id, prop, properties.get(prop), jsonStyle));
}
}
return result;
}
Expand Down
10 changes: 10 additions & 0 deletions core/src/main/java/apoc/util/Util.java
Expand Up @@ -6,6 +6,9 @@
import apoc.export.util.CountingInputStream;
import apoc.result.VirtualNode;
import apoc.result.VirtualRelationship;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.collections.api.iterator.LongIterator;
Expand Down Expand Up @@ -109,6 +112,8 @@
import org.neo4j.procedure.TerminationGuard;

import static apoc.ApocConfig.apocConfig;
import static apoc.export.cypher.formatter.CypherFormatterUtils.formatProperties;
import static apoc.export.cypher.formatter.CypherFormatterUtils.formatToString;
import static apoc.util.DateFormatUtil.getOrCreate;
import static java.lang.String.format;
import static org.eclipse.jetty.util.URIUtil.encodePath;
Expand Down Expand Up @@ -969,6 +974,11 @@ public static Map<String, Object> extractCredentialsIfNeeded(String url, boolean
public static boolean isSelfRel(Relationship rel) {
return rel.getStartNodeId() == rel.getEndNodeId();
}

public static String toCypherMap(Map<String, Object> map) {
final StringBuilder builder = formatProperties(map);
return "{" + formatToString(builder) + "}";
}

public static PointValue toPoint(Map<String, Object> pointMap, Map<String, Object> defaultPointMap) {
double x;
Expand Down
Expand Up @@ -312,6 +312,7 @@
¦procedure¦apoc.static.set¦apoc.static.set(key :: STRING?, value :: ANY?) :: (value :: ANY?)¦apoc.static.set(name, value) - stores value under key for server lifetime storage, returns previously stored or configured value¦false¦xref::misc/static-values.adoc
¦procedure¦apoc.stats.degrees¦apoc.stats.degrees(types = :: STRING?) :: (type :: STRING?, direction :: STRING?, total :: INTEGER?, p50 :: INTEGER?, p75 :: INTEGER?, p90 :: INTEGER?, p95 :: INTEGER?, p99 :: INTEGER?, p999 :: INTEGER?, max :: INTEGER?, min :: INTEGER?, mean :: FLOAT?)¦¦true¦
¦procedure¦apoc.systemdb.execute¦apoc.systemdb.execute(DDL commands :: STRING?, params = {} :: MAP?) :: (row :: MAP?)¦¦false¦xref::database-introspection/systemdb.adoc
¦procedure¦apoc.systemdb.export.metadata¦apoc.systemdb.export.metadata(config = {} :: MAP?) :: (file :: STRING?, source :: STRING?, format :: STRING?, nodes :: INTEGER?, relationships :: INTEGER?, properties :: INTEGER?, time :: INTEGER?, rows :: INTEGER?, batchSize :: INTEGER?, batches :: INTEGER?, done :: BOOLEAN?, data :: STRING?)¦¦true¦
¦procedure¦apoc.systemdb.graph¦apoc.systemdb.graph() :: (nodes :: LIST? OF NODE?, relationships :: LIST? OF RELATIONSHIP?)¦¦false¦xref::database-introspection/systemdb.adoc
¦procedure¦apoc.text.doubleMetaphone¦apoc.text.doubleMetaphone(value :: ANY?) :: (value :: STRING?)¦apoc.text.doubleMetaphone(value) yield value - Compute the Double Metaphone phonetic encoding of all words of the text value which can be a single string or a list of strings¦true¦xref::misc/text-functions.adoc
¦procedure¦apoc.text.phonetic¦apoc.text.phonetic(value :: ANY?) :: (value :: STRING?)¦apoc.text.phonetic(value) yield value - Compute the US_ENGLISH phonetic soundex encoding of all words of the text value which can be a single string or a list of strings¦true¦xref::misc/text-functions.adoc
Expand Down
Expand Up @@ -342,3 +342,11 @@ impact the original one.
Heavy operations should be processed in this phase without blocking the original transaction.
Please note that 'after' and 'before' phases can sometimes block transactions, so generally, `afterAsync` phase is preferred
|===

=== Export metadata

[NOTE]
====
To import triggers in another database (for example after a `./neo4j-admin backup` and `/neo4j-admin restore`),
please see the xref::overview/apoc.systemdb/apoc.systemdb.export.metadata.adoc[apoc.systemdb.export.metadata] procedure.
====
Expand Up @@ -179,3 +179,11 @@ In order to replicate the procedure/function in a cluster environment you can tu
| `apoc.custom.procedures.refresh` | long (default `60000`) | the refresh time that allows replicating the procedure/function
changes to each cluster member
|===

=== Export metadata

[NOTE]
====
To import custom procedures in another database (for example after a `./neo4j-admin backup` and `/neo4j-admin restore`),
please see the xref::overview/apoc.systemdb/apoc.systemdb.export.metadata.adoc[apoc.systemdb.export.metadata] procedure.
====
Expand Up @@ -19,6 +19,12 @@ and might change within minor release updates.
¦Qualified Name¦Type¦Release
include::example$generated-documentation/apoc.systemdb.graph.adoc[]
include::example$generated-documentation/apoc.systemdb.execute.adoc[]
¦xref::overview/apoc.systemdb/apoc.systemdb.export.metadata.adoc[apoc.systemdb.export.metadata icon:book[]] +


¦label:procedure[]
¦label:apoc-full[]

|===

.isType example
Expand Down
8 changes: 8 additions & 0 deletions docs/asciidoc/modules/ROOT/pages/graph-updates/uuid.adoc
Expand Up @@ -159,3 +159,11 @@ The result is:
| label | installed | properties
| "Person" | false | {uuidProperty -> "uuid", addToExistingNodes -> true}
|===

=== Export metadata

[NOTE]
====
To import uuids in another database (for example after a `./neo4j-admin backup` and `/neo4j-admin restore`),
please see the xref::overview/apoc.systemdb/apoc.systemdb.export.metadata.adoc[apoc.systemdb.export.metadata] procedure.
====
@@ -0,0 +1,48 @@
////
This file is generated by DocsTest, so don't change it!
////

= apoc.systemdb.export.metadata
:description: This section contains reference documentation for the apoc.systemdb.export.metadata procedure.

label:procedure[] label:apoc-core[]

== Signature

[source]
----
apoc.systemdb.export.metadata(config = {} :: MAP?) :: (file :: STRING?, source :: STRING?, format :: STRING?, nodes :: INTEGER?, relationships :: INTEGER?, properties :: INTEGER?, time :: INTEGER?, rows :: INTEGER?, batchSize :: INTEGER?, batches :: INTEGER?, done :: BOOLEAN?, data :: STRING?)
----

== Input parameters
[.procedures, opts=header]
|===
| Name | Type | Default
|config|MAP?|{}
|===

== Config parameters
include::partial$usage/config/apoc.systemdb.export.metadata.adoc[]

== Output parameters
[.procedures, opts=header]
|===
| Name | Type
|file|STRING?
|source|STRING?
|format|STRING?
|nodes|INTEGER?
|relationships|INTEGER?
|properties|INTEGER?
|time|INTEGER?
|rows|INTEGER?
|batchSize|INTEGER?
|batches|INTEGER?
|done|BOOLEAN?
|data|STRING?
|===

[[usage-apoc.systemdb.export.metadata]]
== Usage Examples
include::partial$usage/apoc.systemdb.export.metadata.adoc[]

Expand Up @@ -13,6 +13,11 @@ This file is generated by DocsTest, so don't change it!

|label:procedure[]
|label:apoc-full[]
|xref::overview/apoc.systemdb/apoc.systemdb.export.metadata.adoc[apoc.systemdb.export.metadata icon:book[]]


|label:procedure[]
|label:apoc-core[]
|xref::overview/apoc.systemdb/apoc.systemdb.graph.adoc[apoc.systemdb.graph icon:book[]]


Expand Down
8 changes: 8 additions & 0 deletions docs/asciidoc/modules/ROOT/pages/virtual-resource/index.adoc
Expand Up @@ -129,6 +129,14 @@ When a Virtualized Resource is no longer needed it can be removed from the catal
CALL apoc.dv.catalog.remove("vr-name")
----

=== Export metadata

[NOTE]
====
To import dv catalogs in another database (for example after a `./neo4j-admin backup` and `/neo4j-admin restore`),
please see the xref::overview/apoc.systemdb/apoc.systemdb.export.metadata.adoc[apoc.systemdb.export.metadata] procedure.
====

== Managing a Virtualized Resource via CSV files

=== Creating a Virtualized Resource (CSV)
Expand Down
Expand Up @@ -3050,6 +3050,11 @@ apoc.static.getAll(prefix) - returns statically stored values from config (apoc.

|label:procedure[]
|label:apoc-full[]
|xref::overview/apoc.systemdb/apoc.systemdb.export.metadata.adoc[apoc.systemdb.export.metadata icon:book[]]


|label:procedure[]
|label:apoc-core[]
|xref::overview/apoc.systemdb/apoc.systemdb.graph.adoc[apoc.systemdb.graph icon:book[]]


Expand Down
Expand Up @@ -584,6 +584,7 @@ This file is generated by DocsTest, so don't change it!
*** xref::overview/apoc.stats/apoc.stats.degrees.adoc[]
** xref::overview/apoc.systemdb/index.adoc[]
*** xref::overview/apoc.systemdb/apoc.systemdb.execute.adoc[]
*** xref::overview/apoc.systemdb/apoc.systemdb.export.metadata.adoc[]
*** xref::overview/apoc.systemdb/apoc.systemdb.graph.adoc[]
** xref::overview/apoc.temporal/index.adoc[]
*** xref::overview/apoc.temporal/apoc.temporal.format.adoc[]
Expand Down
@@ -0,0 +1,104 @@
To use this procedure, we have to enable it by setting the following property in `apoc.conf`:

.apoc.conf
[source,properties]
----
apoc.export.file.enabled=true
----


If we execute in a database `neo4j` the following queries:
[source,cypher]
----
CALL apoc.trigger.add('trig','RETURN $alpha', {phase: 'after'}, {params: {alpha: 1} });
CALL apoc.trigger.add('trigTwo','RETURN 1', null);
CALL apoc.trigger.pause('trigTwo');
CALL apoc.custom.declareFunction('funNameOne(val = 2 :: INTEGER) :: NODE ', 'MATCH (t:Target {value : $val}) RETURN t');
CALL apoc.custom.declareProcedure('procNameOne(one = 2 ::INTEGER?, two = 3 :: INTEGER?) :: (sum :: INTEGER) ', 'RETURN $one + $two as sum');
CALL apoc.custom.asProcedure('procName','RETURN $input as answer','read',[['answer','number']],[['input','int','42']], 'Procedure that answer to the Ultimate Question of Life, the Universe, and Everything');
CALL apoc.custom.asFunction('funName','RETURN $input as answer','long', [['input','number']], false);
CREATE CONSTRAINT person_cons ON (p:Person) ASSERT p.alpha IS UNIQUE;
CALL apoc.uuid.install('Person', {addToSetLabels: true, uuidProperty: 'alpha'});
CALL apoc.dv.catalog.add("dvName", {type: 'CSV', url: 'file://myUrl', query: 'map.name = $name and map.age = $age', desc: "person's details", labels: ['Person']});
----

and in a database `another`:
[source,cypher]
----
CALL apoc.trigger.add('trigAnother','RETURN 1', null);
----


If we execute:

[source,cypher]
----
CALL apoc.systemdb.export.metadata()
----

we obtain the following files:

.metadata.customProcedures.neo4j.cypher
[source,cypher]
----
CALL apoc.custom.declareFunction('funNameOne(val = 2 :: INTEGER?) :: (NODE?)', 'MATCH (t:Target {value : $val}) RETURN t' , false, '');
CALL apoc.custom.declareProcedure('procNameOne(one = 2 :: INTEGER?, two = 3 :: INTEGER?) :: (sum :: INTEGER?)', 'RETURN $one + $two as sum' , 'READ', '');
CALL apoc.custom.declareProcedure('procName(input = 42 :: INTEGER?) :: (answer :: NUMBER?)', 'RETURN $input as answer' , 'READ', 'Procedure that answer to the Ultimate Question of Life, the Universe, and Everything');
CALL apoc.custom.declareFunction('funName(input :: NUMBER?) :: (INTEGER?)', 'RETURN $input as answer' , false, '');
----

.metadata.dvCatalogs.neo4j.cypher
[source,cypher]
----
CALL apoc.dv.catalog.add('dvName', {name:"dvName",url:"file://myUrl",desc:"person's details",labels:["Person"],query:"map.name = $name and map.age = $age",params:["$name","$age"],type:"CSV"});
----

.metadata.triggers.neo4j.cypher
[source,cypher]
----
CALL apoc.trigger.add('trig', 'RETURN $alpha', {phase:"after"},{params: {alpha:1}});
CALL apoc.trigger.add('trigTwo', 'RETURN 1', null,{params: {}});
CALL apoc.trigger.pause('trigTwo');
----

.metadata.uuids.neo4j.cypher
[source,cypher]
----
CALL apoc.uuid.install('Person', {uuidProperty:"alpha",addToSetLabels:true}) YIELD label RETURN label;
CALL apoc.uuid.install('Person', {uuidProperty:"beta",addToSetLabels:null}) YIELD label RETURN label;
----

.metadata.uuids.schema.neo4j.cypher
[source,cypher]
----
CREATE CONSTRAINT IF NOT EXISTS ON (n:Person) ASSERT n.alpha IS UNIQUE;
CREATE CONSTRAINT IF NOT EXISTS ON (n:Person) ASSERT n.beta IS UNIQUE;
----


So that we can import everything in another db, with the `apoc.cypher.run*` procedures:

.metadata.uuids.schema.neo4j.cypher
[source,cypher]
----
CALL apoc.cypher.runSchemaFile("metadata.uuids.schema.neo4j.cypher");
CALL apoc.cypher.runFiles(["metadata.customProcedures.neo4j.cypher", "metadata.dvCatalogs.neo4j.cypher", "metadata.triggers.neo4j.cypher", "metadata.uuids.neo4j.cypher"])
----


We can choose what we want to export with config parameter `features` (this is a list of strings with possible values "customProcedures", "triggers", "uuids", "dvCatalogs").
For example with:

[source,cypher]
----
CALL apoc.systemdb.export.metadata({features: ["triggers", "uuids"]})
----

will be exported only `metadata.triggers.neo4j.cypher` and `metadata.uuids.neo4j.cypher` files.

Therefore, we can choose the file name prefix, for example if we want to export files with names `customName.triggers.neo4j.cypher`, etc..., we can do:

[source,cypher]
----
CALL apoc.systemdb.export.metadata({filename: "customName"})
----
@@ -0,0 +1,9 @@
The procedure support the following config parameters:

.Config parameters
[opts=header, cols="1,1,1,5"]
|===
| name | type | default | description
| filename | String | "metadata" | The filename prefix. For example, metadata.customProcedures.dbName.cypher
| features | List<String> | ["CypherProcedure", "CypherFunction", "Uuid", "Trigger", "DataVirtualizationCatalog"] | A list indicating which functions are to be exported. Possible values are "CypherProcedure", "CypherFunction", "Uuid", "Trigger", "DataVirtualizationCatalog".
|===
4 changes: 2 additions & 2 deletions full/src/main/java/apoc/custom/CypherProceduresHandler.java
Expand Up @@ -283,7 +283,7 @@ private String serializeSignatures(List<FieldSignature> signatures) {
return Util.toJson(mapped);
}

private List<FieldSignature> deserializeSignatures(String s) {
public static List<FieldSignature> deserializeSignatures(String s) {
List<Map<String, Object>> mapped = Util.fromJson(s, List.class);
return mapped.stream().map(map -> {
String typeString = (String) map.get("type");
Expand Down Expand Up @@ -450,7 +450,7 @@ public List<FieldSignature> outputSignatures(@Name(value = "outputs", defaultVal
outputs.stream().map(pair -> FieldSignature.outputField(pair.get(0), typeof(pair.get(1)))).collect(Collectors.toList());
}

private Neo4jTypes.AnyType typeof(String typeName) {
private static Neo4jTypes.AnyType typeof(String typeName) {
typeName = typeName.replaceAll("\\?", "");
typeName = typeName.toUpperCase();
if (typeName.startsWith("LIST OF ")) return NTList(typeof(typeName.substring(8)));
Expand Down

0 comments on commit 42cab04

Please sign in to comment.