Skip to content

Commit

Permalink
Merge branch 'gh-2630-example-real-federatedstore' into gh-2632-Feder…
Browse files Browse the repository at this point in the history
…atedStore-RemoveGraphAndDeleteAccumuloTable
  • Loading branch information
GCHQDev404 committed Oct 24, 2022
2 parents 8e1fc24 + 6124630 commit 7a78406
Show file tree
Hide file tree
Showing 42 changed files with 1,555 additions and 802 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright 2022 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package uk.gov.gchq.gaffer.data.generator;

import uk.gov.gchq.gaffer.data.element.IdentifierType;

import java.util.LinkedHashMap;

public abstract class CsvFormat {
public static final String ENTITY_GROUP = "ENTITY_GROUP";
public static final String EDGE_GROUP = "EDGE_GROUP";
private static LinkedHashMap<String, String> identifiers = new LinkedHashMap<String, String>();
public abstract String getVertex();
public abstract String getEntityGroup();
public abstract String getEdgeGroup();
public abstract String getSource();
public abstract String getDestination();
public static LinkedHashMap<String, String> getIdentifiers(final CsvFormat csvFormat) {
identifiers.put(String.valueOf(IdentifierType.VERTEX), csvFormat.getVertex());
identifiers.put(ENTITY_GROUP, csvFormat.getEntityGroup());
identifiers.put(EDGE_GROUP, csvFormat.getEdgeGroup());
identifiers.put(String.valueOf(IdentifierType.SOURCE), csvFormat.getSource());
identifiers.put(String.valueOf(IdentifierType.DESTINATION), csvFormat.getDestination());
return identifiers;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
@Since("1.0.0")
@Summary("Generates a CSV string for each element")
public class CsvGenerator implements OneToOneObjectGenerator<String> {
public static final String GROUP = "GROUP";
public static final String COMMA = ",";
private static final Pattern COMMA_PATTERN = Pattern.compile(COMMA);
private static final String COMMA_REPLACEMENT_DEFAULT = " ";
Expand All @@ -68,15 +67,19 @@ public class CsvGenerator implements OneToOneObjectGenerator<String> {
* @param key the name of the field to be retrieved
* @return the value of the field
*/
protected Object getFieldValue(final Element element, final String key) {

private Object getFieldValue(final Element element, final String key) {
final IdentifierType idType = IdentifierType.fromName(key);
final Object value;
if (null == idType) {
if (GROUP.equals(key)) {
if (key.contains(CsvFormat.ENTITY_GROUP) && element.getClassName().contains("Entity")) {
value = element.getGroup();
} else if (key.contains(CsvFormat.EDGE_GROUP) && element.getClassName().contains("Edge")) {
value = element.getGroup();
} else {
value = element.getProperty(key);
}

} else {
value = element.getIdentifier(idType);
}
Expand Down Expand Up @@ -175,7 +178,6 @@ private String quoteString(final Object s) {
return value;
}


public boolean isQuoted() {
return quoted;
}
Expand All @@ -195,18 +197,44 @@ public void setCommaReplacement(final String commaReplacement) {
public static class Builder {
private LinkedHashMap<String, String> fields = new LinkedHashMap<>();
private LinkedHashMap<String, String> constants = new LinkedHashMap<>();

private LinkedHashMap<String, String> propertiesFromSchema = new LinkedHashMap<>();
private String commaReplacement = COMMA_REPLACEMENT_DEFAULT;
private Boolean quoted;

/**
* Stores any additional properties of an {@link Element}.
*
* @param propertyHeadersFromSchema the name of the property headers to be added
* @return a new {@link CsvGenerator.Builder}
*/
public Builder propertyHeadersFromSchema(final LinkedHashMap<String, String> propertyHeadersFromSchema) {
for (final String key: propertyHeadersFromSchema.keySet()) {
fields.put(key, key + ":" + propertyHeadersFromSchema.get(key));
}
return this;
}

/**
* Adds the main identifiers of an {@link Element}, ie VERTEX, GROUP, SOURCE, DEST to the fields LinkedHashMap.
* These identifiers are predefined in a {@link CsvFormat} e.g. {@link Neo4jFormat}
*
* @param identifiersFromFormat the name of the headers to be added
* @return a new {@link CsvGenerator.Builder}
*/
public Builder identifiersFromFormat(final LinkedHashMap<String, String> identifiersFromFormat) {
this.fields = identifiersFromFormat;
return this;
}

/**
* Stores the group of an {@link Element}.
*
* @param columnHeader the group of the {@code Element}
* @return a new {@link Builder}
*/
public Builder group(final String columnHeader) {
fields.put(GROUP, columnHeader);
return this;
return identifier(IdentifierType.GROUP, columnHeader);
}

/**
Expand Down Expand Up @@ -312,7 +340,7 @@ public Builder quoted(final boolean quoted) {
}

/**
* Passes all of the configured fields and constants about an {@link Element} to a new {@link CsvGenerator},
* Passes all the configured fields and constants about an {@link Element} to a new {@link CsvGenerator},
* including the comma replacement String, and the flag for whether values should be quoted.
*
* @return a new {@code CsvGenerator}, containing all configured information
Expand All @@ -325,7 +353,6 @@ public CsvGenerator build() {
if (null != quoted) {
generator.setQuoted(quoted);
}

return generator;
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Copyright 2022 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package uk.gov.gchq.gaffer.data.generator;

import org.codehaus.jackson.annotate.JsonIgnore;

public class Neo4jFormat extends CsvFormat {

@JsonIgnore
@Override
public String getVertex() {
return "_id";
}
@JsonIgnore
@Override
public String getEntityGroup() {
return "_labels";
}
@JsonIgnore
@Override
public String getEdgeGroup() {
return "_type";
}
@JsonIgnore
@Override
public String getSource() {
return "_start";
}
@JsonIgnore
@Override
public String getDestination() {
return "_end";
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
* Copyright 2022 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package uk.gov.gchq.gaffer.data.generator;

import org.codehaus.jackson.annotate.JsonIgnore;

public class NeptuneFormat extends CsvFormat {

@Override
@JsonIgnore
public String getVertex() {
return ":ID";
}

@Override
@JsonIgnore
public String getEntityGroup() {
return ":LABEL";
}

@Override
@JsonIgnore
public String getEdgeGroup() {
return ":TYPE";
}

@Override
@JsonIgnore
public String getSource() {
return ":START_ID";
}

@Override
@JsonIgnore
public String getDestination() {
return ":END_ID";
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,19 @@ public class OpenCypherCsvElementGenerator implements ElementGenerator<String>,
private static final long serialVersionUID = -821376598172364516L;
private static final Logger LOGGER = LoggerFactory.getLogger(OpenCypherCsvElementGenerator.class);

public static final String VERTEX = ":ID";
public static final String NEPTUNE_VERTEX = ":ID";
public static final String NEO4J_VERTEX = "_id";
public static final String ENTITY_GROUP = ":LABEL";
public static final String NEPTUNE_ENTITY_GROUP = ":LABEL";
public static final String NEO4J_ENTITY_GROUP = "_labels";

public static final String SOURCE = ":START_ID";
public static final String NEPTUNE_SOURCE = ":START_ID";
public static final String NEO4J_SOURCE = "_start";
public static final String DESTINATION = ":END_ID";
public static final String NEPTUNE_DESTINATION = ":END_ID";
public static final String NEO4J_DESTINATION = "_end";

public static final String EDGE_GROUP = ":TYPE";
public static final String NEPTUNE_EDGE_GROUP = ":TYPE";
public static final String NEO4J_EDGE_GROUP = "_type";
private static final List<String> ELEMENT_COLUMN_NAMES = ImmutableList.of(VERTEX, ENTITY_GROUP, EDGE_GROUP, SOURCE, DESTINATION);
private static final List<String> ELEMENT_COLUMN_NAMES = ImmutableList.of(NEPTUNE_VERTEX, NEPTUNE_ENTITY_GROUP, NEPTUNE_EDGE_GROUP, NEPTUNE_SOURCE, NEPTUNE_DESTINATION);
private String header;
private int firstRow = 0;
private Boolean trim = false;
Expand All @@ -85,11 +85,11 @@ public Iterable<? extends Element> apply(final Iterable<? extends String> string

FunctionChain.Builder<Tuple<String>, Tuple<String>> transformTuplesBuilder = new FunctionChain.Builder<>();

ElementTupleDefinition entityDefinition = new ElementTupleDefinition(ENTITY_GROUP).vertex(VERTEX);
ElementTupleDefinition edgeDefinition = new ElementTupleDefinition(EDGE_GROUP)
.source(SOURCE)
.destination(DESTINATION)
.property("edge-id", VERTEX);
ElementTupleDefinition entityDefinition = new ElementTupleDefinition(NEPTUNE_ENTITY_GROUP).vertex(NEPTUNE_VERTEX);
ElementTupleDefinition edgeDefinition = new ElementTupleDefinition(NEPTUNE_EDGE_GROUP)
.source(NEPTUNE_SOURCE)
.destination(NEPTUNE_DESTINATION)
.property("edge-id", NEPTUNE_VERTEX);
for (final String columnHeader : parseCsv.getHeader()) {
if (!ELEMENT_COLUMN_NAMES.contains(columnHeader)) {
String propertyName = columnHeader.split(":")[0];
Expand Down Expand Up @@ -158,11 +158,11 @@ public Iterable<? extends Element> apply(final Iterable<? extends String> string
}

private String parseHeader(final String header) {
String parsedHeader = header.replace(NEO4J_VERTEX, VERTEX);
parsedHeader = parsedHeader.replace(NEO4J_ENTITY_GROUP, ENTITY_GROUP);
parsedHeader = parsedHeader.replace(NEO4J_EDGE_GROUP, EDGE_GROUP);
parsedHeader = parsedHeader.replace(NEO4J_SOURCE, SOURCE);
parsedHeader = parsedHeader.replace(NEO4J_DESTINATION, DESTINATION);
String parsedHeader = header.replace(NEO4J_VERTEX, NEPTUNE_VERTEX);
parsedHeader = parsedHeader.replace(NEO4J_ENTITY_GROUP, NEPTUNE_ENTITY_GROUP);
parsedHeader = parsedHeader.replace(NEO4J_EDGE_GROUP, NEPTUNE_EDGE_GROUP);
parsedHeader = parsedHeader.replace(NEO4J_SOURCE, NEPTUNE_SOURCE);
parsedHeader = parsedHeader.replace(NEO4J_DESTINATION, NEPTUNE_DESTINATION);
return parsedHeader;
}

Expand Down
Loading

0 comments on commit 7a78406

Please sign in to comment.