Skip to content
Permalink
Browse files
ATLAS-4454 : Upgrade janusgraph version to 0.6.0
Signed-off-by: Pinal Shah <pinal.shah@freestoneinfotech.com>
  • Loading branch information
pinal-shah committed Nov 1, 2021
1 parent 7581672 commit 6721c0d949863931896e6e0737470d387f71431b
Show file tree
Hide file tree
Showing 12 changed files with 98 additions and 47 deletions.
@@ -34,7 +34,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-core</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>com.codahale.metrics</groupId>
@@ -40,7 +40,9 @@
import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator;
import org.janusgraph.diskstorage.keycolumnvalue.KeyRangeQuery;
import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.KeySlicesIterator;
import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.MultiSlicesQuery;
import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction;
import org.janusgraph.diskstorage.util.RecordIterator;
import org.janusgraph.diskstorage.util.StaticArrayBuffer;
@@ -150,6 +152,11 @@ public KeyIterator getKeys(SliceQuery query, StoreTransaction txh) throws Backen
return executeKeySliceQuery(new FilterList(FilterList.Operator.MUST_PASS_ALL), query);
}

@Override
public KeySlicesIterator getKeys(MultiSlicesQuery queries, StoreTransaction txh) throws BackendException {
throw new UnsupportedOperationException();
}

public static Filter getFilter(SliceQuery query) {
byte[] colStartBytes = query.getSliceStart().length() > 0 ? query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY) : null;
byte[] colEndBytes = query.getSliceEnd().length() > 0 ? query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY) : null;
@@ -440,7 +440,7 @@ public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, St
throw new TemporaryBackendException(e);
}

sleepAfterWrite(txh, commitTime);
this.sleepAfterWrite(commitTime);
}

@Override
@@ -65,15 +65,15 @@
</dependency>

<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<scope>provided</scope>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
<version>${commons-conf2.version}</version>
</dependency>

<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-server</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>com.codahale.metrics</groupId>
@@ -101,7 +101,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-berkeleyje</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -113,7 +113,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-cassandra</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.cassandra.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -129,7 +129,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-cql</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -147,7 +147,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-es</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -159,7 +159,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-hbase</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -171,7 +171,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-solr</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -191,7 +191,13 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-lucene</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
@@ -269,7 +275,6 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>

</dependencies>

<build>
@@ -29,6 +29,7 @@
import org.apache.atlas.repository.graphdb.janus.serializer.TypeCategorySerializer;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationConverter;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
import org.janusgraph.core.JanusGraph;
import org.janusgraph.core.JanusGraphException;
@@ -51,6 +52,7 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

import static org.apache.atlas.ApplicationProperties.DEFAULT_INDEX_RECOVERY;
import static org.apache.atlas.ApplicationProperties.INDEX_RECOVERY_CONF;
@@ -100,18 +102,18 @@ public static Configuration getConfiguration() throws AtlasException {
Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);

//add serializers for non-standard property value types that Atlas uses
janusConfig.addProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName());
janusConfig.addProperty("attributes.custom.attribute1.serializer-class", TypeCategorySerializer.class.getName());
janusConfig.setProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName());
janusConfig.setProperty("attributes.custom.attribute1.serializer-class", TypeCategorySerializer.class.getName());

//not ideal, but avoids making large changes to Atlas
janusConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
janusConfig.addProperty("attributes.custom.attribute2.serializer-class", SerializableSerializer.class.getName());
janusConfig.setProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
janusConfig.setProperty("attributes.custom.attribute2.serializer-class", SerializableSerializer.class.getName());

janusConfig.addProperty("attributes.custom.attribute3.attribute-class", BigInteger.class.getName());
janusConfig.addProperty("attributes.custom.attribute3.serializer-class", BigIntegerSerializer.class.getName());
janusConfig.setProperty("attributes.custom.attribute3.attribute-class", BigInteger.class.getName());
janusConfig.setProperty("attributes.custom.attribute3.serializer-class", BigIntegerSerializer.class.getName());

janusConfig.addProperty("attributes.custom.attribute4.attribute-class", BigDecimal.class.getName());
janusConfig.addProperty("attributes.custom.attribute4.serializer-class", BigDecimalSerializer.class.getName());
janusConfig.setProperty("attributes.custom.attribute4.attribute-class", BigDecimal.class.getName());
janusConfig.setProperty("attributes.custom.attribute4.serializer-class", BigDecimalSerializer.class.getName());

return janusConfig;
}
@@ -188,14 +190,16 @@ public static JanusGraph getGraphInstance() {

@VisibleForTesting
static JanusGraph initJanusGraph(Configuration config) {

org.apache.commons.configuration2.Configuration conf2 = createConfiguration2(config);
try {
return JanusGraphFactory.open(config);
return JanusGraphFactory.open(conf2);
} catch (JanusGraphException e) {
LOG.warn("JanusGraphException: {}", e.getMessage());
if (e.getMessage().startsWith(OLDER_STORAGE_EXCEPTION)) {
LOG.info("Newer client is being used with older janus storage version. Setting allow-upgrade=true and reattempting connection");
config.addProperty("graph.allow-upgrade", true);
return JanusGraphFactory.open(config);
return JanusGraphFactory.open(conf2);
} else {
throw new RuntimeException(e);
}
@@ -252,7 +256,9 @@ public static JanusGraph getBulkLoadingGraphInstance() {
try {
Configuration cfg = getConfiguration();
cfg.setProperty("storage.batch-loading", true);
return JanusGraphFactory.open(cfg);

org.apache.commons.configuration2.Configuration conf2 = createConfiguration2(cfg);
return JanusGraphFactory.open(conf2);
} catch (IllegalArgumentException ex) {
LOG.error("getBulkLoadingGraphInstance: Failed!", ex);
} catch (AtlasException ex) {
@@ -262,6 +268,12 @@ public static JanusGraph getBulkLoadingGraphInstance() {
return null;
}

private static org.apache.commons.configuration2.Configuration createConfiguration2(Configuration conf) {
Properties properties = ConfigurationConverter.getProperties(conf);

return org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties);
}

public static void unload() {
synchronized (AtlasJanusGraphDatabase.class) {

@@ -262,7 +262,7 @@ public AtlasGraphSONReader create() {
}

final GraphSONMapper.Builder builder = GraphSONMapper.build();
final GraphSONMapper mapper = builder.typeInfo(TypeInfo.NO_TYPES).create();
final GraphSONMapper mapper = builder.create();

return new AtlasGraphSONReader(mapper.createMapper(), relationshipCache, graph, bulkLoadGraph,
numWorkers, batchSize, suppliedStartIndex);
@@ -236,7 +236,7 @@ public void has(String propertyName, QueryOperator op, Object value) {

@Override
public void orderBy(final String propertyName, final AtlasGraphQuery.SortOrder sortOrder) {
Order order = sortOrder == AtlasGraphQuery.SortOrder.ASC ? Order.incr : Order.decr;
Order order = sortOrder == AtlasGraphQuery.SortOrder.ASC ? Order.asc : Order.desc;
query.orderBy(propertyName, order);
}

@@ -672,6 +672,27 @@ public Stream<String> query(IndexQuery query, KeyInformation.IndexRetriever info
doc -> doc.getFieldValue(keyIdField).toString());
}

@Override
public Long queryCount(IndexQuery query, KeyInformation.IndexRetriever information, BaseTransaction tx) throws BackendException {
try {
String collection = query.getStore();
String keyIdField = this.getKeyFieldId(collection);
SolrQuery solrQuery = new SolrQuery("*:*");
solrQuery.set("fl", new String[]{keyIdField});
String queryFilter = this.buildQueryFilter(query.getCondition(), information.get(collection));
solrQuery.addFilterQuery(new String[]{queryFilter});
QueryResponse response = this.solrClient.query(collection, solrQuery);
logger.debug("Executed query [{}] in {} ms", query, response.getElapsedTime());
return response.getResults().getNumFound();
} catch (IOException ex) {
logger.error("Query did not complete : ", ex);
throw new PermanentBackendException(ex);
} catch (SolrServerException ex) {
logger.error("Unable to query Solr index.", ex);
throw new PermanentBackendException(ex);
}
}

private void addOrderToQuery(SolrQuery solrQuery, List<IndexQuery.OrderEntry> orders) {
for (final IndexQuery.OrderEntry order1 : orders) {
final String item = order1.getKey();
@@ -765,10 +786,12 @@ public String buildQueryFilter(Condition<JanusGraphElement> condition, KeyInform
final String key = atom.getKey();
final JanusGraphPredicate predicate = atom.getPredicate();

if (value instanceof Number) {
if (value == null && predicate == Cmp.NOT_EQUAL) {
return key + ":*";
} else if (value instanceof Number) {
final String queryValue = escapeValue(value);
Preconditions.checkArgument(predicate instanceof Cmp,
"Relation not supported on numeric types: " + predicate);
"Relation not supported on numeric types: %s", predicate);
final Cmp numRel = (Cmp) predicate;
switch (numRel) {
case EQUAL:
@@ -804,18 +827,18 @@ public String buildQueryFilter(Condition<JanusGraphElement> condition, KeyInform
return (key + ":" + escapeValue(value) + "*");
} else if (predicate == Text.REGEX || predicate == Text.CONTAINS_REGEX) {
return (key + ":/" + value + "/");
} else if (predicate == Cmp.EQUAL) {
} else if (predicate == Cmp.EQUAL || predicate == Cmp.NOT_EQUAL) {
final String tokenizer =
ParameterType.STRING_ANALYZER.findParameter(information.get(key).getParameters(), null);
if(tokenizer != null){
return tokenize(information, value, key, predicate,tokenizer);
} else {
if (tokenizer != null) {
return tokenize(information, value, key, predicate, tokenizer);
} else if (predicate == Cmp.EQUAL) {
return (key + ":\"" + escapeValue(value) + "\"");
} else { // Cmp.NOT_EQUAL case
return ("-" + key + ":\"" + escapeValue(value) + "\"");
}
} else if (predicate == Cmp.NOT_EQUAL) {
return ("-" + key + ":\"" + escapeValue(value) + "\"");
} else if (predicate == Text.FUZZY || predicate == Text.CONTAINS_FUZZY) {
return (key + ":"+escapeValue(value)+"~");
return (key + ":"+escapeValue(value)+"~"+Text.getMaxEditDistance(value.toString()));
} else if (predicate == Cmp.LESS_THAN) {
return (key + ":[* TO \"" + escapeValue(value) + "\"}");
} else if (predicate == Cmp.LESS_THAN_EQUAL) {
@@ -830,9 +853,9 @@ public String buildQueryFilter(Condition<JanusGraphElement> condition, KeyInform
} else if (value instanceof Geoshape) {
final Mapping map = Mapping.getMapping(information.get(key));
Preconditions.checkArgument(predicate instanceof Geo && predicate != Geo.DISJOINT,
"Relation not supported on geo types: " + predicate);
"Relation not supported on geo types: %s", predicate);
Preconditions.checkArgument(map == Mapping.PREFIX_TREE || predicate == Geo.WITHIN || predicate == Geo.INTERSECT,
"Relation not supported on geopoint types: " + predicate);
"Relation not supported on geopoint types: %s", predicate);
final Geoshape geo = (Geoshape)value;
if (geo.getType() == Geoshape.Type.CIRCLE && (predicate == Geo.INTERSECT || map == Mapping.DEFAULT)) {
final Geoshape.Point center = geo.getPoint();
@@ -852,8 +875,7 @@ public String buildQueryFilter(Condition<JanusGraphElement> condition, KeyInform
} else if (value instanceof Date || value instanceof Instant) {
final String s = value.toString();
final String queryValue = escapeValue(value instanceof Date ? toIsoDate((Date) value) : value.toString());
Preconditions.checkArgument(predicate instanceof Cmp, "Relation not supported on date types: "
+ predicate);
Preconditions.checkArgument(predicate instanceof Cmp, "Relation not supported on date types: %s", predicate);
final Cmp numRel = (Cmp) predicate;

switch (numRel) {
@@ -944,7 +966,11 @@ private String tokenize(KeyInformation.StoreRetriever information, Object value,
if (terms.isEmpty()) {
return "";
} else if (terms.size() == 1) {
return (key + ":(" + escapeValue(terms.get(0)) + ")");
if (janusgraphPredicate == Cmp.NOT_EQUAL) {
return ("-" + key + ":(" + escapeValue(terms.get(0)) + ")");
} else {
return (key + ":(" + escapeValue(terms.get(0)) + ")");
}
} else {
final And<JanusGraphElement> andTerms = new And<>();
for (final String term : terms) {
@@ -999,7 +1025,7 @@ public BaseTransactionConfigurable beginTransaction(BaseTransactionConfig config

@Override
public void close() throws BackendException {
logger.trace("Shutting down connection to Solr", solrClient);
logger.trace("Shutting down connection to Solr {}", solrClient);
try {
solrClient.close();
} catch (final IOException e) {
@@ -73,7 +73,7 @@ protected Object getId(JsonNode node) {

private JsonNode getEntityNode(String json) throws IOException {
GraphSONMapper.Builder builder = GraphSONMapper.build();
final ObjectMapper mapper = builder.typeInfo(TypeInfo.NO_TYPES).create().createMapper();
final ObjectMapper mapper = builder.create().createMapper();
return mapper.readTree(json);
}

@@ -717,7 +717,8 @@
<httpcomponents-httpcore.version>4.4.13</httpcomponents-httpcore.version>
<jackson.databind.version>2.11.3</jackson.databind.version>
<jackson.version>2.11.3</jackson.version>
<janus.version>0.5.3</janus.version>
<janusgraph.version>0.6.0</janusgraph.version>
<janusgraph.cassandra.version>0.5.3</janusgraph.cassandra.version>
<javax-inject.version>1</javax-inject.version>
<javax.servlet.version>3.1.0</javax.servlet.version>
<jersey-spring.version>1.19.4</jersey-spring.version>
@@ -767,7 +768,7 @@
<surefire.forkCount>2C</surefire.forkCount>
<surefire.version>3.0.0-M5</surefire.version>
<testng.version>6.9.4</testng.version>
<tinkerpop.version>3.4.10</tinkerpop.version>
<tinkerpop.version>3.5.1</tinkerpop.version>
<woodstox-core.version>5.0.3</woodstox-core.version>
<zookeeper.version>3.5.7</zookeeper.version>
</properties>

0 comments on commit 6721c0d

Please sign in to comment.