From 8ac2f1f480f06d5daf555fe86b2fae809800a1a0 Mon Sep 17 00:00:00 2001 From: Gaurav Sehgal Date: Fri, 3 Jul 2020 15:25:02 +0530 Subject: [PATCH 01/12] Fixed es dependency issue --- .gitignore | 3 + pom.xml | 2 +- .../resources/atlas-application.properties | 262 ++++++++++++++++++ webapp/src/main/resources/atlas-log4j.xml | 131 +++++++++ .../resources/users-credentials.properties | 1 + 5 files changed, 398 insertions(+), 1 deletion(-) create mode 100644 webapp/src/main/resources/atlas-application.properties create mode 100755 webapp/src/main/resources/atlas-log4j.xml create mode 100644 webapp/src/main/resources/users-credentials.properties diff --git a/.gitignore b/.gitignore index 90ca27646d9..6a8b21b9ebd 100755 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,6 @@ distro/solr/*.tgz # emacs files *# *~ + +dashboardv3/node_modules +docs/package-lock.json diff --git a/pom.xml b/pom.xml index 5e0442ae55c..d5ef5f4cc63 100644 --- a/pom.xml +++ b/pom.xml @@ -672,7 +672,7 @@ 1.4.6.2.3.99.0-195 1.2.0 4.0.1 - 5.6.4 + 7.6.2 3.2.11 1.2.17 diff --git a/webapp/src/main/resources/atlas-application.properties b/webapp/src/main/resources/atlas-application.properties new file mode 100644 index 00000000000..bdc411f9b67 --- /dev/null +++ b/webapp/src/main/resources/atlas-application.properties @@ -0,0 +1,262 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +######### Graph Database Configs ######### +# Graph Database +#Configures the graph database to use. Defaults to JanusGraph +#atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase +# Graph Storage +# Set atlas.graph.storage.backend to the correct value for your desired storage +# backend. Possible values: +# +# hbase +# cassandra +# embeddedcassandra - Should only be set by building Atlas with -Pdist,embedded-cassandra-solr +# berkeleyje +# +# See the configuration documentation for more information about configuring the various storage backends. +# +# atlas.graph.storage.backend=hbase2 +# atlas.graph.storage.hbase.table=apache_atlas_janus +#Hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here +# atlas.graph.storage.hostname= +# atlas.graph.storage.hbase.regions-per-server=1 +# atlas.graph.storage.lock.wait-time=10000 +#In order to use Cassandra as a backend, comment out the hbase specific properties above, and uncomment the +#the following properties +#atlas.graph.storage.clustername= +#atlas.graph.storage.port= +atlas.graph.storage.backend=cql +atlas.graph.storage.hostname=localhost +atlas.graph.storage.cql.keyspace=atlandata +atlas.graph.storage.cql.replication-factor=1 +atlas.graph.storage.clustername=cassandra +atlas.graph.storage.port=9042 +atlas.graph.query.fast-property=true +atlas.graph.query.batch=true +atlas.use.index.query.to.find.entity.by.unique.attributes=true +# Gremlin Query Optimizer +# +# Enables rewriting gremlin queriesqueries to maximize performance. This flag is provided as +# a possible way to work around any defects that are found in the optimizer until they +# are resolved. +#atlas.query.gremlinOptimizerEnabled=true +# Delete handler +# +# This allows the default behavior of doing "soft" deletes to be changed. +# +# Allowed Values: +# org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1 - all deletes are "soft" deletes +# org.apache.atlas.repository.store.graph.v1.HardDeleteHandlerV1 - all deletes are "hard" deletes +# +atlas.DeleteHandlerV1.impl=org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1 +# Entity audit repository +# +# This allows the default behavior of logging entity changes to hbase to be changed. +# +# Allowed Values: +# org.apache.atlas.repository.audit.HBaseBasedAuditRepository - log entity changes to hbase +# org.apache.atlas.repository.audit.CassandraBasedAuditRepository - log entity changes to cassandra +# org.apache.atlas.repository.audit.NoopEntityAuditRepository - disable the audit repository +# +atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.CassandraBasedAuditRepository +atlas.EntityAuditRepository.keyspace=atlas_audit +atlas.EntityAuditRepository.replicationFactor=1 +# if Cassandra is used as a backend for audit from the above property, uncomment and set the following +# properties appropriately. If using the embedded cassandra profile, these properties can remain +# commented out. +# atlas.EntityAuditRepository.keyspace=atlas_audit +# atlas.EntityAuditRepository.replicationFactor=1 +# Graph Search Index +atlas.graph.index.search.backend=elasticsearch +atlas.graph.index.search.elasticsearch.retry_on_conflict=5 +#Solr +#Solr cloud mode properties +# atlas.graph.index.search.solr.mode=cloud +# atlas.graph.index.search.solr.zookeeper-url= +# atlas.graph.index.search.solr.zookeeper-connect-timeout=60000 +# atlas.graph.index.search.solr.zookeeper-session-timeout=60000 +# atlas.graph.index.search.solr.wait-searcher=true +#Solr http mode properties +#atlas.graph.index.search.solr.mode=http +#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr +# ElasticSearch support (Tech Preview) +# Comment out above solr configuration, and uncomment the following two lines. Additionally, make sure the +# hostname field is set to a comma delimited set of elasticsearch master nodes, or an ELB that fronts the masters. +# +# Elasticsearch does not provide authentication out of the box, but does provide an option with the X-Pack product +# https://www.elastic.co/products/x-pack/security +# +# Alternatively, the JanusGraph documentation provides some tips on how to secure Elasticsearch without additional +# plugins: http://docs.janusgraph.org/latest/elasticsearch.html +atlas.graph.index.search.hostname=localhost:9200 +atlas.graph.index.search.elasticsearch.client-only=true +# Solr-specific configuration property +# atlas.graph.index.search.max-result-set-size=150 +######### Notification Configs ######### +atlas.notification.embedded=false +atlas.kafka.data=${sys:atlas.home}/data/kafka +atlas.kafka.zookeeper.connect=localhost:2181 +atlas.kafka.bootstrap.servers=localhost:9092 +atlas.kafka.zookeeper.session.timeout.ms=400 +atlas.kafka.zookeeper.connection.timeout.ms=200 +atlas.kafka.zookeeper.sync.time.ms=20 +atlas.kafka.auto.commit.interval.ms=1000 +atlas.kafka.hook.group.id=atlas +atlas.kafka.enable.auto.commit=false +atlas.kafka.auto.offset.reset=earliest +atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1 +atlas.kafka.poll.timeout.ms=1000 +atlas.notification.create.topics=true +atlas.notification.replicas=1 +atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES +atlas.notification.log.failed.messages=true +atlas.notification.consumer.retry.interval=500 +atlas.notification.hook.retry.interval=1000 +# Enable for Kerberized Kafka clusters +#atlas.notification.kafka.service.principal=kafka/_HOST@EXAMPLE.COM +#atlas.notification.kafka.keytab.location=/etc/security/keytabs/kafka.service.keytab +## Server port configuration +#atlas.server.http.port=21000 +#atlas.server.https.port=21443 +######### Security Properties ######### +# SSL config +atlas.enableTLS=false +#truststore.file=/path/to/truststore.jks +#cert.stores.credential.provider.path=jceks://file/path/to/credentialstore.jceks +#following only required for 2-way SSL +#keystore.file=/path/to/keystore.jks +# Authentication config +atlas.authentication.method.kerberos=false +atlas.authentication.method.file=true +#atlas.authentication.method.keycloak=true +#atlas.authentication.method.keycloak.file=${sys:atlas.home}/conf/keycloak.json +#atlas.authentication.method.keycloak.ugi-groups=false +#atlas.authentication.method.keycloak.groups_claim=groups +#### ldap.type= LDAP or AD +atlas.authentication.method.ldap.type=none +#### user credentials file +atlas.authentication.method.file.filename=users-credentials.properties +### groups from UGI +#atlas.authentication.method.ldap.ugi-groups=true +######## LDAP properties ######### +#atlas.authentication.method.ldap.url=ldap://:389 +#atlas.authentication.method.ldap.userDNpattern=uid={0},ou=People,dc=example,dc=com +#atlas.authentication.method.ldap.groupSearchBase=dc=example,dc=com +#atlas.authentication.method.ldap.groupSearchFilter=(member=uid={0},ou=Users,dc=example,dc=com) +#atlas.authentication.method.ldap.groupRoleAttribute=cn +#atlas.authentication.method.ldap.base.dn=dc=example,dc=com +#atlas.authentication.method.ldap.bind.dn=cn=Manager,dc=example,dc=com +#atlas.authentication.method.ldap.bind.password= +#atlas.authentication.method.ldap.referral=ignore +#atlas.authentication.method.ldap.user.searchfilter=(uid={0}) +#atlas.authentication.method.ldap.default.role= +######### Active directory properties ####### +#atlas.authentication.method.ldap.ad.domain=example.com +#atlas.authentication.method.ldap.ad.url=ldap://:389 +#atlas.authentication.method.ldap.ad.base.dn=(sAMAccountName={0}) +#atlas.authentication.method.ldap.ad.bind.dn=CN=team,CN=Users,DC=example,DC=com +#atlas.authentication.method.ldap.ad.bind.password= +#atlas.authentication.method.ldap.ad.referral=ignore +#atlas.authentication.method.ldap.ad.user.searchfilter=(sAMAccountName={0}) +#atlas.authentication.method.ldap.ad.default.role= +######### JAAS Configuration ######## +#atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +#atlas.jaas.KafkaClient.loginModuleControlFlag = required +#atlas.jaas.KafkaClient.option.useKeyTab = true +#atlas.jaas.KafkaClient.option.storeKey = true +#atlas.jaas.KafkaClient.option.serviceName = kafka +#atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab +#atlas.jaas.KafkaClient.option.principal = atlas/_HOST@EXAMPLE.COM +######### Server Properties ######### +atlas.rest.address=http://localhost:21000 +# If enabled and set to true, this will run setup steps when the server starts +#atlas.server.run.setup.on.start=false +######### Entity Audit Configs ######### +atlas.audit.hbase.tablename=apache_atlas_entity_audit +atlas.audigit.zookeeper.session.timeout.ms=1000 +atlas.audit.hbase.zookeeper.quorum=localhost:2181 +######### High Availability Configuration ######## +atlas.server.ha.enabled=false +#### Enabled the configs below as per need if HA is enabled ##### +#atlas.server.ids=id1 +#atlas.server.address.id1=localhost:21000 +#atlas.server.ha.zookeeper.connect=localhost:2181 +#atlas.server.ha.zookeeper.retry.sleeptime.ms=1000 +#atlas.server.ha.zookeeper.num.retries=3 +#atlas.server.ha.zookeeper.session.timeout.ms=20000 +## if ACLs need to be set on the created nodes, uncomment these lines and set the values ## +#atlas.server.ha.zookeeper.acl=: +#atlas.server.ha.zookeeper.auth=: +######### Atlas Authorization ######### +atlas.authorizer.impl=none +######### Type Cache Implementation ######## +# A type cache class which implements +# org.apache.atlas.typesystem.types.cache.TypeCache. +# The default implementation is org.apache.atlas.typesystem.types.cache.DefaultTypeCache which is a local in-memory type cache. +#atlas.TypeCache.impl= +######### Performance Configs ######### +atlas.graph.storage.lock.retries=5 +atlas.graph.cache.db-cache-time=120000 +atlas.graph.cache.db-cache-size=0.6 +atlas.graph.cache.db-cache=true +######### CSRF Configs ######### +atlas.rest-csrf.enabled=true +atlas.rest-csrf.browser-useragents-regex=^Mozilla.*,^Opera.*,^Chrome.* +atlas.rest-csrf.methods-to-ignore=GET,OPTIONS,HEAD,TRACE +atlas.rest-csrf.custom-header=X-XSRF-HEADER +############ KNOX Configs ################ +#atlas.sso.knox.browser.useragent=Mozilla,Chrome,Opera +#atlas.sso.knox.enabled=true +#atlas.sso.knox.providerurl=https://:8443/gateway/knoxsso/api/v1/websso +#atlas.sso.knox.publicKey= +############ Atlas Metric/Stats configs ################ +# Format: atlas.metric.query.. +atlas.metric.query.cache.ttlInSecs=900 +#atlas.metric.query.general.typeCount= +#atlas.metric.query.general.typeUnusedCount= +#atlas.metric.query.general.entityCount= +#atlas.metric.query.general.tagCount= +#atlas.metric.query.general.entityDeleted= +# +#atlas.metric.query.entity.typeEntities= +#atlas.metric.query.entity.entityTagged= +# +#atlas.metric.query.tags.entityTags= +######### Compiled Query Cache Configuration ######### +# The size of the compiled query cache. Older queries will be evicted from the cache +# when we reach the capacity. +#atlas.CompiledQueryCache.capacity=1000 +# Allows notifications when items are evicted from the compiled query +# cache because it has become full. A warning will be issued when +# the specified number of evictions have occurred. If the eviction +# warning threshold <= 0, no eviction warnings will be issued. +#atlas.CompiledQueryCache.evictionWarningThrottle=0 +######### Full Text Search Configuration ######### +#Set to false to disable full text search. +atlas.search.fulltext.enable=true + +######### Gremlin Search Configuration ######### +#Set to false to disable gremlin search. +atlas.search.gremlin.enable=false +########## Add http headers ########### +#atlas.headers.Access-Control-Allow-Origin=* +#atlas.headers.Access-Control-Allow-Methods=GET,OPTIONS,HEAD,PUT,POST +#atlas.headers.= \ No newline at end of file diff --git a/webapp/src/main/resources/atlas-log4j.xml b/webapp/src/main/resources/atlas-log4j.xml new file mode 100755 index 00000000000..5c908e897b7 --- /dev/null +++ b/webapp/src/main/resources/atlas-log4j.xml @@ -0,0 +1,131 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/webapp/src/main/resources/users-credentials.properties b/webapp/src/main/resources/users-credentials.properties new file mode 100644 index 00000000000..6e99ee7cc61 --- /dev/null +++ b/webapp/src/main/resources/users-credentials.properties @@ -0,0 +1 @@ +admin=ADMIN::a4a88c0872bf652bb9ed803ece5fd6e82354838a9bf59ab4babb1dab322154e1 \ No newline at end of file From c93f3e192ebc89148420499cd599e4e45eefaccb Mon Sep 17 00:00:00 2001 From: Nishant Arora Date: Wed, 8 Jul 2020 19:04:24 +0530 Subject: [PATCH 02/12] glossary import categories, classifications with terms, and assignedEntities with terms --- .../atlas/glossary/GlossaryCategoryUtils.java | 91 +++++++- .../atlas/glossary/GlossaryService.java | 56 +++++ .../atlas/glossary/GlossaryTermUtils.java | 217 +++++++++++++++++- .../apache/atlas/web/rest/GlossaryREST.java | 75 ++++++ 4 files changed, 429 insertions(+), 10 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java index e218c2905f1..dcef6287430 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java @@ -27,8 +27,10 @@ import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.ogm.DataAccess; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.type.AtlasRelationshipType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.collections.CollectionUtils; @@ -37,12 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; public class GlossaryCategoryUtils extends GlossaryUtils { @@ -576,4 +573,86 @@ private void updateChildCategories(AtlasGlossaryCategory parentCategory, Collect } } + public static String getGlossaryCategoryHeaders() { + List ret = new ArrayList<>(); + + ret.add("GlossaryName"); + ret.add("CategoryName"); + ret.add("ShortDescription"); + ret.add("LongDescription"); + + return String.join(", ", ret); + } + + protected List getGlossaryCategoryDataList(List fileData, List failedTermMsgs) throws AtlasBaseException { + List glossaryCategories = new ArrayList<>(); + Map glossaryNameCache = new HashMap<>(); + + for (String[] record : fileData) { + AtlasGlossaryCategory glossaryCategory = new AtlasGlossaryCategory(); + + if ((record.length < 1) || StringUtils.isBlank(record[0])) { + LOG.error("The GlossaryName is blank for the record : ", Arrays.toString(record)); + failedTermMsgs.add("The GlossaryName is blank for the record : " + Arrays.toString(record)); + } + + String glossaryName = record[0]; + String glossaryGuid; + + if (glossaryNameCache.get(glossaryName) != null) { + glossaryGuid = glossaryNameCache.get(glossaryName); + + } else { + AtlasVertex vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(GlossaryUtils.ATLAS_GLOSSARY_TYPENAME, GlossaryUtils.ATLAS_GLOSSARY_TYPENAME + "." + QUALIFIED_NAME_ATTR, glossaryName); + + glossaryGuid = (vertex != null) ? AtlasGraphUtilsV2.getIdFromVertex(vertex) : null; + } + + if (glossaryGuid == null) { + if (GlossaryService.isNameInvalid(glossaryName)) { + LOG.error("The provided Glossary Name is invalid : " + glossaryName); + failedTermMsgs.add("The provided Glossary Name is invalid : " + glossaryName); + } else { + AtlasGlossary glossary = new AtlasGlossary(); + glossary.setQualifiedName(glossaryName); + glossary.setName(glossaryName); + + glossary = dataAccess.save(glossary); + glossaryGuid = glossary.getGuid(); + } + } + + if (glossaryGuid != null) { + glossaryNameCache.put(glossaryName, glossaryGuid); + glossaryCategory = populateGlossaryCategoryObject(failedTermMsgs, record, glossaryGuid); + glossaryCategories.add(glossaryCategory); + } + } + + if (failedTermMsgs.size() == 0) { + return glossaryCategories; + } else { + throw new AtlasBaseException("The uploaded file has not been processed due to the following errors : " + "\n" + failedTermMsgs.toString()); + } + } + + protected AtlasGlossaryCategory populateGlossaryCategoryObject(List failedCategoryMsgList, String[] record, String glossaryGuid) { + AtlasGlossaryCategory ret = new AtlasGlossaryCategory(); + int i = 0; + int length = record.length; + + ret.setName((length > ++i) ? record[i] : null); + + if (!StringUtils.isNotBlank(ret.getName())) { + failedCategoryMsgList.add("\n" + "The CategoryName is blank for provided record: " + Arrays.toString(record)); + } else { + ret.setShortDescription((length > ++i) ? record[i] : null); + + ret.setLongDescription((length > ++i) ? record[i] : null); + + ret.setAnchor(new AtlasGlossaryHeader(glossaryGuid)); + } + + return ret; + } } diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java index d630f669eef..1815aa99e4c 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java @@ -1101,6 +1101,28 @@ public List importGlossaryData(InputStream inputStream, Strin return ret; } + public List importGlossaryTermEntitiesData(InputStream inputStream, String fileName) throws AtlasBaseException { + List ret; + + try { + if (StringUtils.isBlank(fileName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_FILE_TYPE, fileName); + } + + List fileData = FileUtils.readFileData(fileName, inputStream); + List failedTermMsgs = new ArrayList<>(); + + ret = glossaryTermUtils.getGlossaryTermEntitiesDataList(fileData, failedTermMsgs); + for (AtlasGlossaryTerm term: ret) { + assignTermToEntities(term.getGuid(), new ArrayList<>(term.getAssignedEntities())); + } + } catch (IOException e) { + throw new AtlasBaseException(AtlasErrorCode.FAILED_TO_UPLOAD, e); + } + + return null; + } + private List createGlossaryTerms(List glossaryTerms) throws AtlasBaseException { List ret = new ArrayList<>(); @@ -1114,4 +1136,38 @@ private List createGlossaryTerms(List glos return ret; } + + public List importGlossaryCategoryData(InputStream inputStream, String fileName) throws AtlasBaseException { + List ret; + + try { + if (StringUtils.isBlank(fileName)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_FILE_TYPE, fileName); + } + + List fileData = FileUtils.readFileData(fileName, inputStream); + List failedCategoryMsgs = new ArrayList<>(); + + ret = glossaryCategoryUtils.getGlossaryCategoryDataList(fileData, failedCategoryMsgs); + ret = createGlossaryCategories(ret); + } catch (IOException e) { + throw new AtlasBaseException(AtlasErrorCode.FAILED_TO_UPLOAD, e); + } + + return ret; + } + + private List createGlossaryCategories(List glossaryCategories) throws AtlasBaseException { + List ret = new ArrayList<>(); + + for (AtlasGlossaryCategory glossaryCategory : glossaryCategories) { + try { + ret.add(createCategory(glossaryCategory)); + } catch (AtlasBaseException e) { + throw new AtlasBaseException(AtlasErrorCode.FAILED_TO_CREATE_GLOSSARY_TERM, e); + } + } + + return ret; + } } diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java index 2c84ec750e6..e1e0202f072 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java @@ -24,10 +24,7 @@ import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader; import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader; import org.apache.atlas.model.glossary.relations.AtlasTermCategorizationHeader; -import org.apache.atlas.model.instance.AtlasObjectId; -import org.apache.atlas.model.instance.AtlasRelatedObjectId; -import org.apache.atlas.model.instance.AtlasRelationship; -import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.*; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.ogm.DataAccess; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; @@ -581,6 +578,58 @@ protected List getGlossaryTermDataList(List fileDat } } + protected List getGlossaryTermEntitiesDataList(List fileData, List failedTermMsgs) throws AtlasBaseException { + List glossaryTerms = new ArrayList<>(); + Map glossaryNameCache = new HashMap<>(); + + for (String[] record : fileData) { + AtlasGlossaryTerm glossaryTerm = new AtlasGlossaryTerm(); + + if ((record.length < 1) || StringUtils.isBlank(record[0])) { + LOG.error("The GlossaryName is blank for the record : ", Arrays.toString(record)); + failedTermMsgs.add("The GlossaryName is blank for the record : " + Arrays.toString(record)); + } + + String glossaryName = record[0]; + String glossaryGuid; + + if (glossaryNameCache.get(glossaryName) != null) { + glossaryGuid = glossaryNameCache.get(glossaryName); + + } else { + AtlasVertex vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(GlossaryUtils.ATLAS_GLOSSARY_TYPENAME, GlossaryUtils.ATLAS_GLOSSARY_TYPENAME + "." + QUALIFIED_NAME_ATTR, glossaryName); + + glossaryGuid = (vertex != null) ? AtlasGraphUtilsV2.getIdFromVertex(vertex) : null; + } + + if (glossaryGuid == null) { + if (GlossaryService.isNameInvalid(glossaryName)) { + LOG.error("The provided Glossary Name is invalid : " + glossaryName); + failedTermMsgs.add("The provided Glossary Name is invalid : " + glossaryName); + } else { + AtlasGlossary glossary = new AtlasGlossary(); + glossary.setQualifiedName(glossaryName); + glossary.setName(glossaryName); + + glossary = dataAccess.save(glossary); + glossaryGuid = glossary.getGuid(); + } + } + + if (glossaryGuid != null) { + glossaryNameCache.put(glossaryName, glossaryGuid); + glossaryTerm = populateGlossaryTermEntitiesObject(failedTermMsgs, record, glossaryName, glossaryGuid); + glossaryTerms.add(glossaryTerm); + } + } + + if (failedTermMsgs.size() == 0) { + return glossaryTerms; + } else { + throw new AtlasBaseException("The uploaded file has not been processed due to the following errors : " + "\n" + failedTermMsgs.toString()); + } + } + public static String getGlossaryTermHeaders() { List ret = new ArrayList<>(); @@ -588,6 +637,8 @@ public static String getGlossaryTermHeaders() { ret.add("TermName"); ret.add("ShortDescription"); ret.add("LongDescription"); + ret.add("Categories"); + ret.add("Classifications"); ret.add("Examples"); ret.add("Abbreviation"); ret.add("Usage"); @@ -609,6 +660,16 @@ public static String getGlossaryTermHeaders() { return String.join(", ", ret); } + public static String getGlossaryTermEntitiesHeaders() { + List ret = new ArrayList<>(); + + ret.add("GlossaryName"); + ret.add("TermName"); + ret.add("EntityGUIDs"); + + return String.join(", ", ret); + } + protected Map getMapValue(String csvRecord, List failedTermMsgs) { Map ret = null; @@ -677,6 +738,131 @@ protected Set getAtlasRelatedTermHeaderSet(String csvRecord, String termName, St return ret; } + protected Set getAtlasRelatedObjectIdSet(String csvRecord, String termName, String glossaryName, List failedTermMsgs) { + Set ret = null; + + if (StringUtils.isNotBlank(csvRecord)) { + ret = new HashSet(); + String csvRecordArray[] = csvRecord.split(FileUtils.ESCAPE_CHARACTER + FileUtils.PIPE_CHARACTER); + AtlasRelatedObjectId relatedObjectId; + + for (String data : csvRecordArray) { + AtlasVertex vertex = null; + + if (data != "") { + vertex = AtlasGraphUtilsV2.findByGuid(data); + } else { + failedTermMsgs.add("\n" + "Either incorrect data specified for Term or Entity does not exist : " +termName); + } + + if (vertex != null) { + String entityGuid = AtlasGraphUtilsV2.getIdFromVertex(vertex); + relatedObjectId = new AtlasRelatedObjectId(); + relatedObjectId.setGuid(entityGuid); + ret.add(relatedObjectId); + } else { + failedTermMsgs.add("\n" + "The provided Reference Glossary and TermName does not exist in the system " + + data + " for record with TermName : " + termName + " and GlossaryName : " + glossaryName); + } + } + + return ret; + } + + return ret; + } + + protected String getTermGuidByQualifiedName(String qualifiedName, List failedTermMsgs) { + + String termGuid = ""; + + AtlasVertex vertex = null; + + if (qualifiedName != "") { + vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME, + GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME + invalidNameChars[1] + QUALIFIED_NAME_ATTR, qualifiedName); + } else { + failedTermMsgs.add("\n" + "Either incorrect data specified for Term or Term does not exist : " + qualifiedName); + } + + if (vertex != null) { + termGuid = AtlasGraphUtilsV2.getIdFromVertex(vertex); + } else { + failedTermMsgs.add("\n" + "The provided Reference Glossary and TermName does not exist in the system " + + qualifiedName); + } + + return termGuid; + } + + protected Set getAtlasCategoriesHeaderSet(String csvRecord, String termName, String glossaryName, List failedTermMsgs) { + Set ret = null; + + if (StringUtils.isNotBlank(csvRecord)) { + ret = new HashSet(); + String csvRecordArray[] = csvRecord.split(FileUtils.ESCAPE_CHARACTER + FileUtils.PIPE_CHARACTER); + AtlasTermCategorizationHeader termCategoryHeader; + + for (String data : csvRecordArray) { + AtlasVertex vertex = null; + String dataArray[] = data.split(FileUtils.ESCAPE_CHARACTER + FileUtils.COLON_CHARACTER); + + if ((dataArray.length % 2) == 0) { + vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(GlossaryUtils.ATLAS_GLOSSARY_CATEGORY_TYPENAME, + GlossaryUtils.ATLAS_GLOSSARY_CATEGORY_TYPENAME + invalidNameChars[1] + QUALIFIED_NAME_ATTR, dataArray[1] + invalidNameChars[0] + dataArray[0]); + } else { + failedTermMsgs.add("\n" + "Either incorrect data specified for Term or Category does not exist : " +termName); + } + + if (vertex != null) { + String glossaryCategoryGuid = AtlasGraphUtilsV2.getIdFromVertex(vertex); + termCategoryHeader = new AtlasTermCategorizationHeader(); + termCategoryHeader.setCategoryGuid(glossaryCategoryGuid); + ret.add(termCategoryHeader); + } else { + failedTermMsgs.add("\n" + "The provided Reference Glossary and Category does not exist in the system " + + dataArray[1] + FileUtils.COLON_CHARACTER + dataArray[0] + " for record with TermName : " + termName + " and GlossaryName : " + glossaryName); + } + } + + return ret; + } + + return ret; + } + + protected List getAtlasClassificationList(String csvRecord, String termName, String glossaryName, List failedTermMsgs) { + List ret = null; + + if (StringUtils.isNotBlank(csvRecord)) { + ret = new ArrayList<>(); + String csvRecordArray[] = csvRecord.split(FileUtils.ESCAPE_CHARACTER + FileUtils.PIPE_CHARACTER); + AtlasClassification termClassification; + + for (String data : csvRecordArray) { + + // TODO check for the existence of the classification here + + if (data == "") { + failedTermMsgs.add("\n" + "Either incorrect data specified for Term or Classification does not exist : " +termName); + } + + if (data != "") { + termClassification = new AtlasClassification(); + termClassification.setTypeName(data); + ret.add(termClassification); + } else { + failedTermMsgs.add("\n" + "The provided Reference Glossary and Category does not exist in the system " + + data + " for record with TermName : " + termName + " and GlossaryName : " + glossaryName); + } + } + + return ret; + } + + return ret; + } + protected AtlasGlossaryTerm populateGlossaryTermObject(List failedTermMsgList, String[] record, String glossaryGuid) { AtlasGlossaryTerm ret = new AtlasGlossaryTerm(); int i = 0; @@ -691,6 +877,10 @@ protected AtlasGlossaryTerm populateGlossaryTermObject(List failedTermMs ret.setLongDescription((length > ++i) ? record[i] : null); + ret.setCategories((length > ++i) ? (Set) getAtlasCategoriesHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null); + + ret.setClassifications((length > ++i) ? (List) getAtlasClassificationList(record[i], ret.getName(), record[0], failedTermMsgList) : null); + ret.setExamples((length > ++i) ? (List) getListValue(record[i]) : null); ret.setAbbreviation((length > ++i) ? record[i] : null); @@ -730,4 +920,23 @@ protected AtlasGlossaryTerm populateGlossaryTermObject(List failedTermMs return ret; } + + protected AtlasGlossaryTerm populateGlossaryTermEntitiesObject(List failedTermMsgList, String[] record, String glossaryName, String glossaryGuid) { + AtlasGlossaryTerm ret = new AtlasGlossaryTerm(); + int i = 0; + int length = record.length; + + ret.setName((length > ++i) ? record[i] : null); + + if (!StringUtils.isNotBlank(ret.getName())) { + failedTermMsgList.add("\n" + "The TermName is blank for provided record: " + Arrays.toString(record)); + } else { + ret.setQualifiedName(ret.getName()+ "@" + glossaryName); + ret.setAnchor(new AtlasGlossaryHeader(glossaryGuid)); + ret.setAssignedEntities((length > ++i) ? (Set) getAtlasRelatedObjectIdSet(record[i], ret.getName(), record[0], failedTermMsgList) : null); + ret.setGuid(getTermGuidByQualifiedName(ret.getQualifiedName(), failedTermMsgList)); + } + + return ret; + } } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java index 78d52adef82..174d65cbcc9 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java @@ -22,6 +22,7 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.SortOrder; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.glossary.GlossaryCategoryUtils; import org.apache.atlas.glossary.GlossaryService; import org.apache.atlas.glossary.GlossaryTermUtils; import org.apache.atlas.model.glossary.AtlasGlossary; @@ -969,6 +970,42 @@ public void write(OutputStream outputStream) throws IOException, WebApplicationE }; } + /** + * Get sample template for uploading/creating bulk AtlasGlossaryCategory + * + * @return Template File + * @HTTP 400 If the provided fileType is not supported + */ + @GET + @Path("/import/categories/template") + @Produces(MediaType.APPLICATION_OCTET_STREAM) + public StreamingOutput produceCategoriesTemplate() { + return new StreamingOutput() { + @Override + public void write(OutputStream outputStream) throws IOException, WebApplicationException { + outputStream.write(GlossaryCategoryUtils.getGlossaryCategoryHeaders().getBytes()); + } + }; + } + + /** + * Get sample template for uploading/creating bulk AtlasGlossaryCategory + * + * @return Template File + * @HTTP 400 If the provided fileType is not supported + */ + @GET + @Path("/import/termentities/template") + @Produces(MediaType.APPLICATION_OCTET_STREAM) + public StreamingOutput produceTermEntitiesTemplate() { + return new StreamingOutput() { + @Override + public void write(OutputStream outputStream) throws IOException, WebApplicationException { + outputStream.write(GlossaryTermUtils.getGlossaryTermEntitiesHeaders().getBytes()); + } + }; + } + /** * Upload glossary file for creating AtlasGlossaryTerms in bulk * @@ -987,4 +1024,42 @@ public List importGlossaryData(@FormDataParam("file") InputSt @FormDataParam("file") FormDataContentDisposition fileDetail) throws AtlasBaseException { return glossaryService.importGlossaryData(inputStream, fileDetail.getFileName()); } + + /** + * Upload glossary file for creating AtlasGlossaryTerms in bulk + * + * @param inputStream InputStream of file + * @param fileDetail FormDataContentDisposition metadata of file + * @return + * @throws AtlasBaseException + * @HTTP 200 If glossary term creation was successful + * @HTTP 400 If Glossary term definition has invalid or missing information + * @HTTP 409 If Glossary term already exists (duplicate qualifiedName) + */ + @POST + @Path("/import/termentities") + @Consumes(MediaType.MULTIPART_FORM_DATA) + public List importGlossaryTermEntitiesData(@FormDataParam("file") InputStream inputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail) throws AtlasBaseException { + return glossaryService.importGlossaryTermEntitiesData(inputStream, fileDetail.getFileName()); + } + + /** + * Upload glossary file for creating AtlasGlossaryTerms in bulk + * + * @param inputStream InputStream of file + * @param fileDetail FormDataContentDisposition metadata of file + * @return + * @throws AtlasBaseException + * @HTTP 200 If glossary term creation was successful + * @HTTP 400 If Glossary term definition has invalid or missing information + * @HTTP 409 If Glossary term already exists (duplicate qualifiedName) + */ + @POST + @Path("/import/categories") + @Consumes(MediaType.MULTIPART_FORM_DATA) + public List importGlossaryCategoriesData(@FormDataParam("file") InputStream inputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail) throws AtlasBaseException { + return glossaryService.importGlossaryCategoryData(inputStream, fileDetail.getFileName()); + } } \ No newline at end of file From 9898d7e580ab3aaaedbe8295d0ea8f830479bb85 Mon Sep 17 00:00:00 2001 From: Nishant Arora Date: Wed, 8 Jul 2020 19:23:04 +0530 Subject: [PATCH 03/12] updated comments --- .../apache/atlas/web/rest/GlossaryREST.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java index 174d65cbcc9..26165a77502 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java @@ -989,7 +989,7 @@ public void write(OutputStream outputStream) throws IOException, WebApplicationE } /** - * Get sample template for uploading/creating bulk AtlasGlossaryCategory + * Get sample template for assigning entities to AtlasGlossaryTerm * * @return Template File * @HTTP 400 If the provided fileType is not supported @@ -1026,15 +1026,15 @@ public List importGlossaryData(@FormDataParam("file") InputSt } /** - * Upload glossary file for creating AtlasGlossaryTerms in bulk + * Upload glossary file for assigning entities to AtlasGlossaryTerm in bulk * * @param inputStream InputStream of file * @param fileDetail FormDataContentDisposition metadata of file * @return * @throws AtlasBaseException - * @HTTP 200 If glossary term creation was successful - * @HTTP 400 If Glossary term definition has invalid or missing information - * @HTTP 409 If Glossary term already exists (duplicate qualifiedName) + * @HTTP 200 If glossary term assignment was successful + * @HTTP 400 If Glossary term assignment definition has invalid or missing information + * @HTTP 409 If Glossary term assignment already exists (duplicate qualifiedName) */ @POST @Path("/import/termentities") @@ -1045,15 +1045,15 @@ public List importGlossaryTermEntitiesData(@FormDataParam("fi } /** - * Upload glossary file for creating AtlasGlossaryTerms in bulk + * Upload glossary file for creating AtlasGlossaryCategory in bulk * * @param inputStream InputStream of file * @param fileDetail FormDataContentDisposition metadata of file * @return * @throws AtlasBaseException - * @HTTP 200 If glossary term creation was successful - * @HTTP 400 If Glossary term definition has invalid or missing information - * @HTTP 409 If Glossary term already exists (duplicate qualifiedName) + * @HTTP 200 If glossary category creation was successful + * @HTTP 400 If Glossary category definition has invalid or missing information + * @HTTP 409 If Glossary category already exists (duplicate qualifiedName) */ @POST @Path("/import/categories") From 9dda90ac4bef676adae5cd4d230c8032347ab2f4 Mon Sep 17 00:00:00 2001 From: Gaurav Sehgal Date: Thu, 9 Jul 2020 03:28:57 +0530 Subject: [PATCH 04/12] Entity search process powered directly from es --- .../apache/atlas/repository/Constants.java | 8 + graphdb/api/pom.xml | 13 + .../atlas/repository/graphdb/AtlasGraph.java | 3 + graphdb/janus/pom.xml | 6 + .../janus/AtlasElasticsearchDatabase.java | 58 ++++ .../janus/AtlasElasticsearchIndexQuery.java | 96 ++++++ .../graphdb/janus/AtlasJanusGraph.java | 66 +++-- .../janus/AtlasJanusGraphDatabase.java | 2 +- .../graphdb/janus/AtlasJanusIndexQuery.java | 1 + pom.xml | 2 +- .../atlas/discovery/ESIndexQueryBuilder.java | 273 ++++++++++++++++++ .../discovery/EntitySearchProcessor.java | 94 +++--- .../apache/atlas/discovery/SearchContext.java | 185 +++++++----- .../atlas/discovery/SearchProcessor.java | 6 +- .../resources/atlas-application.properties | 5 +- 15 files changed, 668 insertions(+), 150 deletions(-) create mode 100644 graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java create mode 100644 graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchIndexQuery.java create mode 100644 repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index a71787bc010..3096889263d 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -49,6 +49,12 @@ public final class Constants { public static final String TYPE_NAME_INTERNAL = INTERNAL_PROPERTY_KEY_PREFIX + "internal"; public static final String ASSET_ENTITY_TYPE = "Asset"; public static final String OWNER_ATTRIBUTE = "owner"; + public static final String REF_ASSET_TYPE = "Referenceable"; + + /** + * Atlan constants + */ + public static final String ATLAN_ASSET_TYPE = "AtlanAsset"; /** * Entity type's super types property key. @@ -148,6 +154,8 @@ public final class Constants { */ public static final String VERTEX_INDEX = "vertex_index"; + public static final String INDEX_PREFIX = "janusgraph_"; + /** * search backing index name for edge labels. */ diff --git a/graphdb/api/pom.xml b/graphdb/api/pom.xml index 0d8acc0dc52..22084dc08f3 100644 --- a/graphdb/api/pom.xml +++ b/graphdb/api/pom.xml @@ -46,11 +46,24 @@ org.codehaus.jettison jettison + org.apache.tinkerpop gremlin-core ${tinkerpop.version} + + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java index c016f634085..855a2c8b3fd 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java @@ -21,6 +21,7 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.groovy.GroovyExpression; import org.apache.atlas.type.AtlasType; +import org.elasticsearch.search.builder.SearchSourceBuilder; import javax.script.ScriptEngine; import javax.script.ScriptException; @@ -194,6 +195,8 @@ public interface AtlasGraph { */ AtlasIndexQuery indexQuery(GraphIndexQueryParameters indexQueryParameters); + AtlasIndexQuery esIndexQuery(String indexName, SearchSourceBuilder sourceBuilder); + /** * Gets the management object associated with this graph and opens a transaction * for changes that are made. diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index 6deef5181b1..f15443077de 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -127,6 +127,12 @@ ${elasticsearch.version} + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + + org.janusgraph janusgraph-es diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java new file mode 100644 index 00000000000..a69baa2e6f9 --- /dev/null +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java @@ -0,0 +1,58 @@ +package org.apache.atlas.repository.graphdb.janus; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.commons.configuration.Configuration; +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class AtlasElasticsearchDatabase { + private static final Logger LOG = LoggerFactory.getLogger(RestHighLevelClient.class); + private static volatile RestHighLevelClient searchClient; + public static final String INDEX_BACKEND_CONF = "atlas.graph.index.search.hostname"; + + public static List getHttpHosts() throws AtlasException { + List httpHosts = new ArrayList<>(); + Configuration configuration = ApplicationProperties.get(); + String indexConf = configuration.getString(INDEX_BACKEND_CONF); + String[] hosts = indexConf.split(","); + for (String host: hosts) { + host = host.trim(); + String[] hostAndPort = host.split(":"); + if (hostAndPort.length == 1) { + httpHosts.add(new HttpHost(hostAndPort[0])); + } else if (hostAndPort.length == 2) { + httpHosts.add(new HttpHost(hostAndPort[0], Integer.parseInt(hostAndPort[1]))); + } else { + throw new AtlasException("Invalid config"); + } + } + return httpHosts; + } + + public static RestHighLevelClient getClient() { + if (searchClient == null) { + synchronized (AtlasElasticsearchDatabase.class) { + if (searchClient == null) { + try { + List httpHosts = getHttpHosts(); + + RestClientBuilder restClientBuilder = RestClient.builder(httpHosts.toArray(new HttpHost[0])); + searchClient = + new RestHighLevelClient(restClientBuilder); + } catch (AtlasException e) { + + } + } + } + } + return searchClient; + } +} diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchIndexQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchIndexQuery.java new file mode 100644 index 00000000000..643af3c0dda --- /dev/null +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchIndexQuery.java @@ -0,0 +1,96 @@ +package org.apache.atlas.repository.graphdb.janus; + +import org.apache.atlas.repository.graphdb.AtlasIndexQuery; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.commons.lang.NotImplementedException; +import org.apache.tinkerpop.gremlin.process.traversal.Order; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.janusgraph.util.encoding.LongEncoding; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.stream.Stream; + +public class AtlasElasticsearchIndexQuery implements AtlasIndexQuery { + private AtlasJanusGraph graph; + private RestHighLevelClient esClient; + private String index; + private SearchSourceBuilder sourceBuilder; + private SearchResponse searchResponse; + + public AtlasElasticsearchIndexQuery(AtlasJanusGraph graph, RestHighLevelClient esClient, String index, SearchSourceBuilder sourceBuilder) { + this.esClient = esClient; + this.sourceBuilder = sourceBuilder; + this.index = index; + this.graph = graph; + searchResponse = null; + } + + private SearchRequest getSearchRequest(String index, SearchSourceBuilder sourceBuilder) { + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.source(sourceBuilder); + return searchRequest; + } + + private Iterator> runQuery(SearchRequest searchRequest) { + Iterator> result = null; + try { + searchResponse = esClient.search(searchRequest, RequestOptions.DEFAULT); + Stream> resultStream = Arrays.stream(searchResponse.getHits().getHits()) + .map(ResultImpl::new); + result = resultStream.iterator(); + } catch (IOException e) { + e.printStackTrace(); + } + return result; + } + + @Override + public Iterator> vertices() { + SearchRequest searchRequest = getSearchRequest(index, sourceBuilder); + return runQuery(searchRequest); + } + + @Override + public Iterator> vertices(int offset, int limit, String sortBy, Order sortOrder) { + throw new NotImplementedException(); + } + + @Override + public Iterator> vertices(int offset, int limit) { + sourceBuilder.from(offset); + sourceBuilder.size(limit); + SearchRequest searchRequest = getSearchRequest(index, sourceBuilder); + return runQuery(searchRequest); + } + + @Override + public Long vertexTotals() { + return searchResponse.getHits().getTotalHits().value; + } + + public final class ResultImpl implements AtlasIndexQuery.Result { + private SearchHit hit; + + public ResultImpl(SearchHit hit) { + this.hit = hit; + } + + @Override + public AtlasVertex getVertex() { + long vertexId = LongEncoding.decode(hit.getId()); + return graph.getVertex(String.valueOf(vertexId)); + } + + @Override + public double getScore() { + return hit.getScore(); + } + } +} diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java index 0dd573b897a..f00b0ab1a22 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java @@ -57,6 +57,8 @@ import org.apache.tinkerpop.gremlin.structure.io.IoCore; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.janusgraph.core.Cardinality; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.JanusGraphFactory; @@ -83,9 +85,9 @@ import java.util.Map; import java.util.Set; -import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT; -import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase.getGraphInstance; +import static org.apache.atlas.repository.graphdb.janus.AtlasElasticsearchDatabase.getClient; import static org.apache.atlas.type.Constants.STATE_PROPERTY_KEY; /** @@ -93,28 +95,29 @@ */ public class AtlasJanusGraph implements AtlasGraph { private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraph.class); - private static final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0]; + private static final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0]; - private static Configuration APPLICATION_PROPERTIES = null; + private static Configuration APPLICATION_PROPERTIES = null; private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction(); - private final Set multiProperties = new HashSet<>(); - private final StandardJanusGraph janusGraph; + private final Set multiProperties = new HashSet<>(); + private final StandardJanusGraph janusGraph; + private final RestHighLevelClient esClient; private final ThreadLocal scriptEngine = ThreadLocal.withInitial(() -> { DefaultImportCustomizer.Builder builder = DefaultImportCustomizer.build() - .addClassImports(java.util.function.Function.class) - .addMethodImports(__.class.getMethods()) - .addMethodImports(P.class.getMethods()); + .addClassImports(java.util.function.Function.class) + .addMethodImports(__.class.getMethods()) + .addMethodImports(P.class.getMethods()); return new GremlinGroovyScriptEngine(builder.create()); }); public AtlasJanusGraph() { - this(getGraphInstance()); + this(getGraphInstance(), getClient()); } - public AtlasJanusGraph(JanusGraph graphInstance) { + public AtlasJanusGraph(JanusGraph graphInstance, RestHighLevelClient esClient) { //determine multi-properties once at startup JanusGraphManagement mgmt = null; @@ -135,6 +138,7 @@ public AtlasJanusGraph(JanusGraph graphInstance) { } janusGraph = (StandardJanusGraph) graphInstance; + this.esClient = esClient; } @Override @@ -142,9 +146,9 @@ public AtlasEdge addEdge(AtlasVertex inVertex, String edgeLabel) { try { - Vertex oV = outVertex.getV().getWrappedElement(); - Vertex iV = inVertex.getV().getWrappedElement(); - Edge edge = oV.addEdge(edgeLabel, iV); + Vertex oV = outVertex.getV().getWrappedElement(); + Vertex iV = inVertex.getV().getWrappedElement(); + Edge edge = oV.addEdge(edgeLabel, iV); return GraphDbObjectFactory.createEdge(this, edge); } catch (SchemaViolationException e) { @@ -186,7 +190,7 @@ public AtlasEdge getEdgeBetweenVertices(AtlasVertex fromVertex, AtlasVertex toVe @Override public AtlasEdge getEdge(String edgeId) { Iterator it = getGraph().edges(edgeId); - Edge e = getSingleElement(it, edgeId); + Edge e = getSingleElement(it, edgeId); return GraphDbObjectFactory.createEdge(this, e); } @@ -267,24 +271,29 @@ public AtlasIndexQuery indexQuery(String index * Creates an index query. * * @param indexQueryParameters the parameterObject containing the information needed for creating the index. - * */ public AtlasIndexQuery indexQuery(GraphIndexQueryParameters indexQueryParameters) { return indexQuery(indexQueryParameters.getIndexName(), indexQueryParameters.getGraphQueryString(), indexQueryParameters.getOffset(), indexQueryParameters.getIndexQueryParameters()); } private AtlasIndexQuery indexQuery(String indexName, String graphQuery, int offset, List indexQueryParameterList) { - String prefix = getIndexQueryPrefix(); - JanusGraphIndexQuery query = getGraph().indexQuery(indexName, graphQuery).setElementIdentifier(prefix).offset(offset); + String prefix = getIndexQueryPrefix(); + JanusGraphIndexQuery query = getGraph().indexQuery(indexName, graphQuery).setElementIdentifier(prefix).offset(offset); - if(indexQueryParameterList != null && indexQueryParameterList.size() > 0) { - for(AtlasIndexQueryParameter indexQueryParameter: indexQueryParameterList) { + if (indexQueryParameterList != null && indexQueryParameterList.size() > 0) { + for (AtlasIndexQueryParameter indexQueryParameter : indexQueryParameterList) { query = query.addParameter(new Parameter(indexQueryParameter.getParameterName(), indexQueryParameter.getParameterValue())); } } return new AtlasJanusIndexQuery(this, query); } + @Override + public AtlasIndexQuery esIndexQuery(String indexName, SearchSourceBuilder sourceBuilder) { + assert esClient != null; + return new AtlasElasticsearchIndexQuery(this, esClient, INDEX_PREFIX + indexName, sourceBuilder); + } + @Override public AtlasGraphManagement getManagementSystem() { return new AtlasJanusGraphManagement(this, getGraph().openManagement()); @@ -312,8 +321,8 @@ public Set getVertexIndexKeys() { @Override public AtlasVertex getVertex(String vertexId) { - Iterator it = getGraph().vertices(vertexId); - Vertex vertex = getSingleElement(it, vertexId); + Iterator it = getGraph().vertices(vertexId); + Vertex vertex = getSingleElement(it, vertexId); return GraphDbObjectFactory.createVertex(this, vertex); } @@ -353,7 +362,7 @@ public JanusGraph getGraph() { @Override public void exportToGson(OutputStream os) throws IOException { - GraphSONMapper mapper = getGraph().io(IoCore.graphson()).mapper().create(); + GraphSONMapper mapper = getGraph().io(IoCore.graphson()).mapper().create(); GraphSONWriter.Builder builder = GraphSONWriter.build(); builder.mapper(mapper); @@ -434,7 +443,7 @@ public Iterable> wrapVertices(Iter return Iterables.transform(it, (Function>) input -> - GraphDbObjectFactory.createVertex(AtlasJanusGraph.this, input)); + GraphDbObjectFactory.createVertex(AtlasJanusGraph.this, input)); } @@ -457,9 +466,9 @@ public void addMultiProperties(Set names) { } - String getIndexFieldName(AtlasPropertyKey propertyKey, JanusGraphIndex graphIndex, Parameter ... parameters) { + String getIndexFieldName(AtlasPropertyKey propertyKey, JanusGraphIndex graphIndex, Parameter... parameters) { PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey); - if(parameters == null) { + if (parameters == null) { parameters = EMPTY_PARAMETER_ARRAY; } return janusGraph.getIndexSerializer().getDefaultFieldName(janusKey, parameters, graphIndex.getBackingIndex()); @@ -523,9 +532,9 @@ private Object convertGremlinValue(Object rawValue) { } private Set getIndexKeys(Class janusGraphElementClass) { - JanusGraphManagement mgmt = getGraph().openManagement(); + JanusGraphManagement mgmt = getGraph().openManagement(); Iterable indices = mgmt.getGraphIndexes(janusGraphElementClass); - Set result = new HashSet(); + Set result = new HashSet(); for (JanusGraphIndex index : indices) { result.add(index.name()); @@ -574,6 +583,7 @@ public Object apply(Object input) { return convertGremlinValue(input); } } + private Edge getFirstActiveEdge(GraphTraversal gt) { while (gt.hasNext()) { Edge gremlinEdge = (Edge) gt.next(); diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java index e9b4b09ef4a..1e9d868b718 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java @@ -279,7 +279,7 @@ public AtlasGraph getGraph() { @Override public AtlasGraph getGraphBulkLoading() { - return new AtlasJanusGraph(getBulkLoadingGraphInstance()); + return new AtlasJanusGraph(getBulkLoadingGraphInstance(), null); } private static void startLocalSolr() { diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java index ff3fd4a78ea..f34bcdeb297 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java @@ -28,6 +28,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order; import org.janusgraph.core.JanusGraphIndexQuery; import org.janusgraph.core.JanusGraphVertex; +import org.janusgraph.util.encoding.LongEncoding; /** * Janus implementation of AtlasIndexQuery. diff --git a/pom.xml b/pom.xml index d5ef5f4cc63..e032722e8f6 100644 --- a/pom.xml +++ b/pom.xml @@ -658,7 +658,7 @@ 0.5.1 3.4.6 - 7.3.0 + 8.4.0 3.1.1 2.0.2 diff --git a/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java b/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java new file mode 100644 index 00000000000..b4f7be80b1f --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java @@ -0,0 +1,273 @@ +package org.apache.atlas.discovery; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.type.AtlasStructType; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.QueryStringQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; +import static org.apache.atlas.discovery.SearchProcessor.*; +import static org.apache.atlas.repository.Constants.*; + +@SuppressWarnings("DuplicatedCode") +public class ESIndexQueryBuilder { + SearchContext context; + private static final Logger LOG = LoggerFactory.getLogger(ESIndexQueryBuilder.class); + private static final Map OPERATOR_MAP = new HashMap<>(); + + static { + OPERATOR_MAP.put(SearchParameters.Operator.LT, "%s: [* TO %s}"); + + OPERATOR_MAP.put(SearchParameters.Operator.GT, "%s: {%s TO *]"); + + OPERATOR_MAP.put(SearchParameters.Operator.LTE, "%s: [* TO %s]"); + + OPERATOR_MAP.put(SearchParameters.Operator.GTE, "%s: [%s TO *]"); + + OPERATOR_MAP.put(SearchParameters.Operator.EQ, "%s: %s"); + + OPERATOR_MAP.put(SearchParameters.Operator.NEQ, "(*:* -" + "%s: %s)"); + + OPERATOR_MAP.put(SearchParameters.Operator.IN, "%s: (%s)"); // this should be a list of quoted strings + + OPERATOR_MAP.put(SearchParameters.Operator.LIKE, "%s: (%s)"); // this should be regex pattern + + OPERATOR_MAP.put(SearchParameters.Operator.STARTS_WITH, "%s: (%s*)"); + + OPERATOR_MAP.put(SearchParameters.Operator.ENDS_WITH, "%s: (*%s)"); + + OPERATOR_MAP.put(SearchParameters.Operator.CONTAINS, "%s: (*%s*)"); + + OPERATOR_MAP.put(SearchParameters.Operator.IS_NULL, "(*:* NOT " + "%s:[* TO *])"); + + OPERATOR_MAP.put(SearchParameters.Operator.NOT_NULL, "%s:[* TO *]"); + } + + ESIndexQueryBuilder(SearchContext context) { + this.context = context; + } + + void addClassificationTypeFilter(StringBuilder indexQuery) { + if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationNames())) { + String classificationNames = AtlasStructType.AtlasAttribute.escapeIndexQueryValue(context.getClassificationNames()); + if (indexQuery.length() != 0) { + indexQuery.append(" AND "); + } + + indexQuery.append("(").append(CLASSIFICATION_NAMES_KEY).append(':').append(classificationNames) + .append(" OR ").append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append(':').append(classificationNames).append(")"); + } + } + + void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) { + if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) { + String classificationTypesQryStr = context.getClassificationTypesQryStr(); + + if (indexQuery.length() != 0) { + indexQuery.append(" AND "); + } + + indexQuery.append("(").append(CLASSIFICATION_NAMES_KEY).append(":").append(classificationTypesQryStr).append(" OR ") + .append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append(":").append(classificationTypesQryStr).append(")"); + } + } + + void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) { + if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) { + if (context.getClassificationTypes().iterator().next() == MATCH_ALL_NOT_CLASSIFIED) { + if (indexQuery.length() != 0) { + indexQuery.append(" AND "); + } + indexQuery.append("( *:* ").append("-").append(CLASSIFICATION_NAMES_KEY) + .append(":" + "[* TO *]").append(" AND ").append("-") + .append(PROPAGATED_CLASSIFICATION_NAMES_KEY) + .append(":" + "[* TO *]").append(")"); + } + } + } + + void addActiveStateQueryFilter(StringBuilder indexQuery) { + if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery != null) { + if (indexQuery.length() != 0) { + indexQuery.append(" AND "); + } + indexQuery.append("(").append(STATE_PROPERTY_KEY).append(":" + "ACTIVE").append(")"); + } + } + + void addTypeAndSubTypesQueryFilter(StringBuilder indexQuery, String typeAndAllSubTypesQryStr) { + if (indexQuery != null && StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) { + if (indexQuery.length() > 0) { + indexQuery.append(" AND "); + } + + indexQuery.append("(").append(Constants.TYPE_NAME_PROPERTY_KEY) + .append(":").append(typeAndAllSubTypesQryStr).append(")"); + } + } + + void constructFilterQuery(StringBuilder indexQuery, Set structTypes, SearchParameters.FilterCriteria filterCriteria, Set indexAttributes) { + if (filterCriteria != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Processing Filters"); + } + + String filterQuery = toIndexQuery(structTypes, filterCriteria, indexAttributes, 0); + + if (org.apache.commons.lang.StringUtils.isNotEmpty(filterQuery)) { + if (indexQuery.length() > 0) { + indexQuery.append(AND_STR); + } + + indexQuery.append(filterQuery); + } + } + } + + private String toIndexQuery(Set structTypes, SearchParameters.FilterCriteria criteria, Set indexAttributes, int level) { + return toIndexQuery(structTypes, criteria, indexAttributes, new StringBuilder(), level); + } + + private String toIndexQuery(Set structTypes, SearchParameters.FilterCriteria criteria, Set indexAttributes, StringBuilder sb, int level) { + Set filterAttributes = new HashSet<>(indexAttributes); + + SearchParameters.FilterCriteria.Condition condition = criteria.getCondition(); + if (condition != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) { + StringBuilder nestedExpression = new StringBuilder(); + + for (SearchParameters.FilterCriteria filterCriteria : criteria.getCriterion()) { + String nestedQuery = toIndexQuery(structTypes, filterCriteria, filterAttributes, level + 1); + + if (org.apache.commons.lang.StringUtils.isNotEmpty(nestedQuery)) { + if (nestedExpression.length() > 0) { + nestedExpression.append(SPACE_STRING).append(condition).append(SPACE_STRING); + } + nestedExpression.append(nestedQuery); + } + } + + boolean needSurroundingBraces = level != 0 || (condition == SearchParameters.FilterCriteria.Condition.OR && criteria.getCriterion().size() > 1); + if (nestedExpression.length() > 0) { + return sb.append(needSurroundingBraces ? BRACE_OPEN_STR : EMPTY_STRING) + .append(nestedExpression) + .append(needSurroundingBraces ? BRACE_CLOSE_STR : EMPTY_STRING) + .toString(); + } else { + return EMPTY_STRING; + } + } else if (org.apache.commons.lang.StringUtils.isNotEmpty(criteria.getAttributeName())) { + try { + ArrayList orExpQuery = new ArrayList<>(); + for (AtlasStructType structType : structTypes) { + String name = structType.getVertexPropertyName(criteria.getAttributeName()); + + if (filterAttributes.contains(name)) { + String nestedQuery = toIndexExpression(structType, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue()); + orExpQuery.add(nestedQuery); + filterAttributes.remove(name); + } + } + + if (CollectionUtils.isNotEmpty(orExpQuery)) { + if (orExpQuery.size() > 1) { + String orExpStr = org.apache.commons.lang.StringUtils.join(orExpQuery, " " + SearchParameters.FilterCriteria.Condition.OR.name() + " "); + return BRACE_OPEN_STR + " " + orExpStr + " " + BRACE_CLOSE_STR; + } else { + return orExpQuery.iterator().next(); + } + } else { + return EMPTY_STRING; + } + } catch (AtlasBaseException e) { + LOG.warn(e.getMessage()); + } + } + return EMPTY_STRING; + } + + private String toIndexExpression(AtlasStructType type, String attrName, SearchParameters.Operator op, String attrVal) { + String ret = EMPTY_STRING; + + try { + if (OPERATOR_MAP.get(op) != null) { + String qualifiedName = type.getVertexPropertyName(attrName); + String escapeIndexQueryValue = AtlasStructType.AtlasAttribute.escapeIndexQueryValue(attrVal); + + // map '__customAttributes' 'CONTAINS' operator to 'EQ' operator (solr limitation for json serialized string search) + // map '__customAttributes' value from 'key1=value1' to '\"key1\":\"value1\"' (escape special characters and surround with quotes) + if (attrName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && op == SearchParameters.Operator.CONTAINS) { + ret = String.format(OPERATOR_MAP.get(op), qualifiedName, getCustomAttributeIndexQueryValue(escapeIndexQueryValue, false)); + } else { + ret = String.format(OPERATOR_MAP.get(op), qualifiedName, escapeIndexQueryValue); + } + } + } catch (AtlasBaseException ex) { + LOG.warn(ex.getMessage()); + } + + return ret; + } + + private String getCustomAttributeIndexQueryValue(String attrValue, boolean forGraphQuery) { + String ret = null; + + if (org.apache.commons.lang.StringUtils.isNotEmpty(attrValue)) { + int separatorIdx = attrValue.indexOf(CUSTOM_ATTR_SEPARATOR); + String key = separatorIdx != -1 ? attrValue.substring(0, separatorIdx) : null; + String value = key != null ? attrValue.substring(separatorIdx + 1) : null; + + if (key != null && value != null) { + if (forGraphQuery) { + ret = String.format(CUSTOM_ATTR_SEARCH_FORMAT_GRAPH, key, value); + } else { + ret = String.format(CUSTOM_ATTR_SEARCH_FORMAT, key, value); + } + } else { + ret = attrValue; + } + } + + return ret; + } + + void constructSearchSource(SearchSourceBuilder sourceBuilder, String queryString, + String fullTextQuery, String sortBy, org.apache.atlas.SortOrder sortOrder) { + if (StringUtils.isNotEmpty(fullTextQuery)) { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must().add(QueryBuilders.queryStringQuery(queryString)); + + QueryStringQueryBuilder fullTextQueryBuilder = QueryBuilders.queryStringQuery(fullTextQuery); + fullTextQueryBuilder.fields().put(ATLAN_ASSET_TYPE + ".integrationType", 100F); + fullTextQueryBuilder.fields().put(ASSET_ENTITY_TYPE + ".__s_name", 50F); + fullTextQueryBuilder.fields().put(ATLAN_ASSET_TYPE + ".displayName", 50F); + fullTextQueryBuilder.fields().put(ASSET_ENTITY_TYPE + ".__s_owner", 20F); + fullTextQueryBuilder.fields().put(REF_ASSET_TYPE + ".qualifiedName", 10F); + fullTextQueryBuilder.fields().put(ATLAN_ASSET_TYPE + ".status", 5F); + boolQueryBuilder.must().add(fullTextQueryBuilder); + + sourceBuilder.query(boolQueryBuilder); + } else { + sourceBuilder.query(QueryBuilders.queryStringQuery(queryString)); + } + + sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); + + if (StringUtils.isNotEmpty(sortBy)) { + SortOrder order = sortOrder == org.apache.atlas.SortOrder.ASCENDING ? SortOrder.ASC : SortOrder.DESC; + sourceBuilder.sort(new FieldSortBuilder(sortBy).order(order)); + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java index 5dcff3b1627..f4e5804151f 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java @@ -31,6 +31,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.Predicate; import org.apache.commons.collections.PredicateUtils; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,30 +47,35 @@ import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC; public class EntitySearchProcessor extends SearchProcessor { - private static final Logger LOG = LoggerFactory.getLogger(EntitySearchProcessor.class); + private static final Logger LOG = LoggerFactory.getLogger(EntitySearchProcessor.class); private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("EntitySearchProcessor"); private final AtlasIndexQuery indexQuery; private final AtlasGraphQuery graphQuery; - private Predicate graphQueryPredicate; - private Predicate filterGraphQueryPredicate; + private Predicate graphQueryPredicate; + private Predicate filterGraphQueryPredicate; + private SearchSourceBuilder sourceBuilder; public EntitySearchProcessor(SearchContext context) { super(context); - final Set entityTypes = context.getEntityTypes(); - final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters(); - final Set indexAttributes = new HashSet<>(); - final Set graphAttributes = new HashSet<>(); - final Set allAttributes = new HashSet<>(); - final Set typeAndSubTypes = context.getEntityTypeNames(); - final String typeAndSubTypesQryStr = context.getEntityTypesQryStr(); - final String sortBy = context.getSearchParameters().getSortBy(); - final SortOrder sortOrder = context.getSearchParameters().getSortOrder(); + sourceBuilder = new SearchSourceBuilder(); - final Set classificationTypes = context.getClassificationTypes(); - final Set classificationTypeAndSubTypes = context.getClassificationTypeNames(); - final boolean filterClassification; + final Set entityTypes = context.getEntityTypes(); + final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters(); + final Set indexAttributes = new HashSet<>(); + final Set graphAttributes = new HashSet<>(); + final Set allAttributes = new HashSet<>(); + final Set typeAndSubTypes = context.getEntityTypeNames(); + final String typeAndSubTypesQryStr = context.getEntityTypesQryStr(); + final String fullTextQuery = context.getSearchParameters().getQuery(); + final AtlasEntityType entityType = context.getEntityTypes().iterator().next(); + String sortBy = context.getSearchParameters().getSortBy(); + SortOrder sortOrder = context.getSearchParameters().getSortOrder(); + + final Set classificationTypes = context.getClassificationTypes(); + final Set classificationTypeAndSubTypes = context.getClassificationTypeNames(); + final boolean filterClassification; if (CollectionUtils.isNotEmpty(classificationTypes)) { filterClassification = !context.needClassificationProcessor(); @@ -78,9 +84,9 @@ public EntitySearchProcessor(SearchContext context) { } final Predicate typeNamePredicate; - final Predicate traitPredicate = buildTraitPredict(classificationTypes); - final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator() - .generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class); + final Predicate traitPredicate = buildTraitPredict(classificationTypes); + final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator() + .generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class); if (!isEntityRootType()) { @@ -100,11 +106,11 @@ public EntitySearchProcessor(SearchContext context) { inMemoryPredicate = typeNamePredicate; if (typeSearchByIndex) { - graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(indexQuery, typeAndSubTypesQryStr); + esIndexQueryBuilder.addTypeAndSubTypesQueryFilter(indexQuery, typeAndSubTypesQryStr); } if (attrSearchByIndex) { - constructFilterQuery(indexQuery, entityTypes, filterCriteria, indexAttributes); + esIndexQueryBuilder.constructFilterQuery(indexQuery, entityTypes, filterCriteria, indexAttributes); Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, indexAttributes); if (attributePredicate != null) { @@ -116,14 +122,26 @@ public EntitySearchProcessor(SearchContext context) { if (indexQuery.length() > 0) { - graphIndexQueryBuilder.addActiveStateQueryFilter(indexQuery); + esIndexQueryBuilder.addActiveStateQueryFilter(indexQuery); String indexQueryString = STRAY_AND_PATTERN.matcher(indexQuery).replaceAll(")"); indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")"); indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll(""); - this.indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, indexQueryString); + AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy); + if (sortByAttribute == null) { + sortBy = null; + } else { + sortBy = sortByAttribute.getVertexPropertyName(); + } + + if (sortOrder == null) { + sortOrder = ASCENDING; + } + + esIndexQueryBuilder.constructSearchSource(sourceBuilder, indexQueryString, fullTextQuery, sortBy, sortOrder); + this.indexQuery = context.getGraph().esIndexQuery(Constants.VERTEX_INDEX, sourceBuilder); } else { this.indexQuery = null; } @@ -145,7 +163,7 @@ public EntitySearchProcessor(SearchContext context) { orConditions.add(query.createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null) - .has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null)); + .has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null)); } else { orConditions.add(query.createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes)); orConditions.add(query.createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes)); @@ -188,7 +206,6 @@ public EntitySearchProcessor(SearchContext context) { } } if (sortBy != null && !sortBy.isEmpty()) { - final AtlasEntityType entityType = context.getEntityTypes().iterator().next(); AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy); if (sortByAttribute != null) { @@ -209,18 +226,18 @@ public EntitySearchProcessor(SearchContext context) { if (attributesPredicate != null) { filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate : - PredicateUtils.andPredicate(filterGraphQueryPredicate, attributesPredicate); + PredicateUtils.andPredicate(filterGraphQueryPredicate, attributesPredicate); } if (filterClassification) { filterGraphQueryPredicate = filterGraphQueryPredicate == null ? traitPredicate : - PredicateUtils.andPredicate(filterGraphQueryPredicate, traitPredicate); + PredicateUtils.andPredicate(filterGraphQueryPredicate, traitPredicate); } // Filter condition for the STATUS if (context.getSearchParameters().getExcludeDeletedEntities()) { filterGraphQueryPredicate = filterGraphQueryPredicate == null ? activePredicate : - PredicateUtils.andPredicate(filterGraphQueryPredicate, activePredicate); + PredicateUtils.andPredicate(filterGraphQueryPredicate, activePredicate); } } @@ -236,12 +253,12 @@ public List execute() { AtlasPerfTracer perf = null; if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntitySearchProcessor.execute(" + context + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntitySearchProcessor.execute(" + context + ")"); } try { final int startIdx = context.getSearchParameters().getOffset(); - final int limit = context.getSearchParameters().getLimit(); + final int limit = context.getSearchParameters().getLimit(); // when subsequent filtering stages are involved, query should start at 0 even though startIdx can be higher // @@ -251,19 +268,6 @@ public List execute() { final List entityVertices = new ArrayList<>(); - SortOrder sortOrder = context.getSearchParameters().getSortOrder(); - String sortBy = context.getSearchParameters().getSortBy(); - - final AtlasEntityType entityType = context.getEntityTypes().iterator().next(); - AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy); - if (sortByAttribute == null) { - sortBy = null; - } else { - sortBy = sortByAttribute.getVertexPropertyName(); - } - - if (sortOrder == null) { sortOrder = ASCENDING; } - for (; ret.size() < limit; qryOffset += limit) { entityVertices.clear(); @@ -276,7 +280,7 @@ public List execute() { final boolean isLastResultPage; if (indexQuery != null) { - Iterator idxQueryResult = executeIndexQuery(context, indexQuery, qryOffset, limit); + Iterator idxQueryResult = indexQuery.vertices(qryOffset, limit); getVerticesFromIndexQueryResult(idxQueryResult, entityVertices); @@ -343,6 +347,10 @@ public void filter(List entityVertices) { } } + public boolean isIndexQuery() { + return indexQuery != null; + } + @Override public long getResultCount() { if (indexQuery != null) { diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java index 04e82186b59..9b69de4c35b 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java @@ -55,38 +55,38 @@ * possible chaining of processor(s) */ public class SearchContext { - private static final Logger LOG = LoggerFactory.getLogger(SearchContext.class); - - private final AtlasTypeRegistry typeRegistry; - private final AtlasGraph graph; - private final Set entityTypes; - private final Set indexedKeys; - private final Set entityAttributes; - private final SearchParameters searchParameters; + private static final Logger LOG = LoggerFactory.getLogger(SearchContext.class); + + private final AtlasTypeRegistry typeRegistry; + private final AtlasGraph graph; + private final Set entityTypes; + private final Set indexedKeys; + private final Set entityAttributes; + private final SearchParameters searchParameters; private final Set classificationTypes; - private final Set classificationNames; - private final Set typeAndSubTypes; - private final Set classificationTypeAndSubTypes; - private final String typeAndSubTypesQryStr; - private final String classificationTypeAndSubTypesQryStr; - private boolean terminateSearch = false; - private SearchProcessor searchProcessor; + private final Set classificationNames; + private final Set typeAndSubTypes; + private final Set classificationTypeAndSubTypes; + private final String typeAndSubTypesQryStr; + private final String classificationTypeAndSubTypesQryStr; + private boolean terminateSearch = false; + private SearchProcessor searchProcessor; public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS)); - public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS)); - public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS)); - public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot(); - public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot(); - public final static String TYPENAME_DELIMITER = ","; + public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS)); + public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS)); + public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot(); + public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot(); + public final static String TYPENAME_DELIMITER = ","; public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set indexedKeys) throws AtlasBaseException { - this.searchParameters = searchParameters; - this.typeRegistry = typeRegistry; - this.graph = graph; - this.indexedKeys = indexedKeys; - this.entityAttributes = new HashSet<>(); - this.entityTypes = getEntityTypes(searchParameters.getTypeName()); + this.searchParameters = searchParameters; + this.typeRegistry = typeRegistry; + this.graph = graph; + this.indexedKeys = indexedKeys; + this.entityAttributes = new HashSet<>(); + this.entityTypes = getEntityTypes(searchParameters.getTypeName()); this.classificationNames = getClassificationNames(searchParameters.getClassification()); this.classificationTypes = getClassificationTypes(this.classificationNames); @@ -117,45 +117,48 @@ public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRe filterStructTypes(); //gather all classifications and its corresponding subtypes - Set classificationTypeAndSubTypes = new HashSet<>(); + Set classificationTypeAndSubTypes = new HashSet<>(); String classificationTypeAndSubTypesQryStr = null; - if (CollectionUtils.isNotEmpty(classificationTypes) && classificationTypes.iterator().next() != MATCH_ALL_NOT_CLASSIFIED ) { + if (CollectionUtils.isNotEmpty(classificationTypes) && classificationTypes.iterator().next() != MATCH_ALL_NOT_CLASSIFIED) { for (AtlasClassificationType classificationType : classificationTypes) { if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { - classificationTypeAndSubTypes = Collections.emptySet(); + classificationTypeAndSubTypes = Collections.emptySet(); classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY; break; } else { Set allTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName()); - classificationTypeAndSubTypes.addAll(allTypes); } + classificationTypeAndSubTypes.addAll(allTypes); + } } if (CollectionUtils.isNotEmpty(classificationTypeAndSubTypes)) { classificationTypeAndSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(classificationTypeAndSubTypes); } } else { - classificationTypeAndSubTypes = Collections.emptySet(); + classificationTypeAndSubTypes = Collections.emptySet(); classificationTypeAndSubTypesQryStr = ""; } - this.classificationTypeAndSubTypes = classificationTypeAndSubTypes; + this.classificationTypeAndSubTypes = classificationTypeAndSubTypes; this.classificationTypeAndSubTypesQryStr = classificationTypeAndSubTypesQryStr; //gather all types and its corresponding subtypes - Set typeAndSubTypes = new HashSet<>(); + Set typeAndSubTypes = new HashSet<>(); String typeAndSubTypesQryStr = null; + Set superTypes = new HashSet<>(); if (CollectionUtils.isNotEmpty(entityTypes)) { for (AtlasEntityType entityType : entityTypes) { if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) { - typeAndSubTypes = Collections.emptySet(); + typeAndSubTypes = Collections.emptySet(); typeAndSubTypesQryStr = ALL_TYPE_QUERY; break; } else { - Set allTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName()); + Set allTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName()); typeAndSubTypes.addAll(allTypes); + superTypes.addAll(entityType.getSuperTypes()); } } @@ -163,17 +166,23 @@ public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRe typeAndSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndSubTypes); } } else { - typeAndSubTypes = Collections.emptySet(); + typeAndSubTypes = Collections.emptySet(); typeAndSubTypesQryStr = ""; } - this.typeAndSubTypes = typeAndSubTypes; + this.typeAndSubTypes = typeAndSubTypes; this.typeAndSubTypesQryStr = typeAndSubTypesQryStr; if (glossaryTermVertex != null) { addProcessor(new TermSearchProcessor(this, getAssignedEntities(glossaryTermVertex))); } - if (needFullTextProcessor()) { + EntitySearchProcessor entitySearchProcessor = null; + if (needEntityProcessor()) { + entitySearchProcessor = new EntitySearchProcessor(this); + } + + if (needFullTextProcessor() && (needClassificationProcessor() || entitySearchProcessor == null + || !entitySearchProcessor.isIndexQuery() || !superTypes.contains(Constants.ATLAN_ASSET_TYPE))) { if (AtlasRepositoryConfiguration.isFreeTextSearchEnabled()) { LOG.debug("Using Free Text index based search."); @@ -189,36 +198,62 @@ public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRe addProcessor(new ClassificationSearchProcessor(this)); } - if (needEntityProcessor()) { - addProcessor(new EntitySearchProcessor(this)); + if (entitySearchProcessor != null) { + addProcessor(entitySearchProcessor); } } - public SearchParameters getSearchParameters() { return searchParameters; } + public SearchParameters getSearchParameters() { + return searchParameters; + } - public AtlasTypeRegistry getTypeRegistry() { return typeRegistry; } + public AtlasTypeRegistry getTypeRegistry() { + return typeRegistry; + } - public AtlasGraph getGraph() { return graph; } + public AtlasGraph getGraph() { + return graph; + } - public Set getIndexedKeys() { return indexedKeys; } + public Set getIndexedKeys() { + return indexedKeys; + } - public Set getEntityAttributes() { return entityAttributes; } + public Set getEntityAttributes() { + return entityAttributes; + } - public Set getClassificationTypes() { return classificationTypes; } + public Set getClassificationTypes() { + return classificationTypes; + } - public Set getEntityTypeNames() { return typeAndSubTypes; } + public Set getEntityTypeNames() { + return typeAndSubTypes; + } - public Set getClassificationTypeNames() { return classificationTypeAndSubTypes; } + public Set getClassificationTypeNames() { + return classificationTypeAndSubTypes; + } - public String getEntityTypesQryStr() { return typeAndSubTypesQryStr; } + public String getEntityTypesQryStr() { + return typeAndSubTypesQryStr; + } - public String getClassificationTypesQryStr() { return classificationTypeAndSubTypesQryStr; } + public String getClassificationTypesQryStr() { + return classificationTypeAndSubTypesQryStr; + } - public Set getEntityTypes() { return entityTypes; } + public Set getEntityTypes() { + return entityTypes; + } - public SearchProcessor getSearchProcessor() { return searchProcessor; } + public SearchProcessor getSearchProcessor() { + return searchProcessor; + } - public Set getClassificationNames() {return classificationNames;} + public Set getClassificationNames() { + return classificationNames; + } public boolean includeEntityType(String entityType) { return typeAndSubTypes.isEmpty() || typeAndSubTypes.contains(entityType); @@ -240,9 +275,13 @@ public boolean includeClassificationTypes(Collection traitNames) { return ret; } - public boolean terminateSearch() { return terminateSearch; } + public boolean terminateSearch() { + return terminateSearch; + } - public void terminateSearch(boolean terminateSearch) { this.terminateSearch = terminateSearch; } + public void terminateSearch(boolean terminateSearch) { + this.terminateSearch = terminateSearch; + } public StringBuilder toString(StringBuilder sb) { if (sb == null) { @@ -268,10 +307,10 @@ boolean needFullTextProcessor() { } boolean needClassificationProcessor() { - return (CollectionUtils.isNotEmpty(classificationTypes) && (CollectionUtils.isEmpty(entityTypes) || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ; + return (CollectionUtils.isNotEmpty(classificationTypes) && (CollectionUtils.isEmpty(entityTypes) || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch(); } - boolean isWildCardSearch () { + boolean isWildCardSearch() { if (CollectionUtils.isNotEmpty(classificationNames)) { return classificationNames.stream().anyMatch(classification -> classification.contains(WILDCARD_CLASSIFICATIONS)); } @@ -317,7 +356,7 @@ private void validateAttributes(final AtlasStructType structType, final String.. public boolean hasAttributeFilter(FilterCriteria filterCriteria) { return filterCriteria != null && - (CollectionUtils.isNotEmpty(filterCriteria.getCriterion()) || StringUtils.isNotEmpty(filterCriteria.getAttributeName())); + (CollectionUtils.isNotEmpty(filterCriteria.getCriterion()) || StringUtils.isNotEmpty(filterCriteria.getAttributeName())); } private void addProcessor(SearchProcessor processor) { @@ -337,7 +376,7 @@ private AtlasClassificationType getClassificationType(String classificationName) ret = MATCH_ALL_CLASSIFIED; } else if (StringUtils.equals(classificationName, MATCH_ALL_NOT_CLASSIFIED.getTypeName())) { ret = MATCH_ALL_NOT_CLASSIFIED; - } else if (StringUtils.equals(classificationName, ALL_CLASSIFICATION_TYPES)){ + } else if (StringUtils.equals(classificationName, ALL_CLASSIFICATION_TYPES)) { ret = MATCH_ALL_CLASSIFICATION_TYPES; } else { ret = typeRegistry.getClassificationTypeByName(classificationName); @@ -359,7 +398,7 @@ private Set getClassificationNames(String classification) throws AtlasBa Set classificationNames = new HashSet<>(); if (StringUtils.isNotEmpty(classification)) { - String[] types = classification.split(TYPENAME_DELIMITER); + String[] types = classification.split(TYPENAME_DELIMITER); Set names = new HashSet<>(Arrays.asList(types)); names.forEach(name -> { @@ -383,7 +422,7 @@ private Set getClassificationNames(String classification) throws AtlasBa private AtlasEntityType getEntityType(String entityName) { return StringUtils.equals(entityName, ALL_ENTITY_TYPES) ? MATCH_ALL_ENTITY_TYPES : - typeRegistry.getEntityTypeByName(entityName); + typeRegistry.getEntityTypeByName(entityName); } private Set getEntityTypes(String typeName) throws AtlasBaseException { @@ -392,10 +431,10 @@ private Set getEntityTypes(String typeName) throws AtlasBaseExc //split multiple typeNames by comma if (StringUtils.isNotEmpty(typeName)) { - String[] types = typeName.split(TYPENAME_DELIMITER); + String[] types = typeName.split(TYPENAME_DELIMITER); Set typeNames = new HashSet<>(Arrays.asList(types)); - entityTypes = typeNames.stream().map(n -> - getEntityType(n)).filter(Objects::nonNull).collect(Collectors.toSet()); + entityTypes = typeNames.stream().map(n -> + getEntityType(n)).filter(Objects::nonNull).collect(Collectors.toSet()); // Validate if the type name is incorrect if (CollectionUtils.isEmpty(entityTypes) || entityTypes.size() != typeNames.size()) { @@ -421,7 +460,7 @@ private Set getEntityTypes(String typeName) throws AtlasBaseExc return entityTypes; } - private void filterStructTypes(){ + private void filterStructTypes() { //if typeName contains ALL_ENTITY_TYPES, remove others as OR condition will not effect any other if (CollectionUtils.isNotEmpty(entityTypes) && entityTypes.contains(MATCH_ALL_ENTITY_TYPES)) { entityTypes.clear(); @@ -452,10 +491,10 @@ private AtlasVertex getGlossaryTermVertex(String termName) { if (StringUtils.isNotEmpty(termName)) { AtlasEntityType termType = getTermEntityType(); - AtlasAttribute attrName = termType.getAttribute(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ATTR_QNAME); - AtlasGraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, termType.getTypeName()) - .has(attrName.getVertexPropertyName(), termName) - .has(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()); + AtlasAttribute attrName = termType.getAttribute(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ATTR_QNAME); + AtlasGraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, termType.getTypeName()) + .has(attrName.getVertexPropertyName(), termName) + .has(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()); Iterator results = query.vertices().iterator(); @@ -466,10 +505,10 @@ private AtlasVertex getGlossaryTermVertex(String termName) { } private List getAssignedEntities(AtlasVertex glossaryTerm) { - List ret = new ArrayList<>(); - AtlasEntityType termType = getTermEntityType(); - AtlasAttribute attr = termType.getRelationshipAttribute(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ATTR_ASSIGNED_ENTITIES, EntityGraphRetriever.TERM_RELATION_NAME); - Iterator edges = GraphHelper.getEdgesForLabel(glossaryTerm, attr.getRelationshipEdgeLabel(), attr.getRelationshipEdgeDirection()); + List ret = new ArrayList<>(); + AtlasEntityType termType = getTermEntityType(); + AtlasAttribute attr = termType.getRelationshipAttribute(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ATTR_ASSIGNED_ENTITIES, EntityGraphRetriever.TERM_RELATION_NAME); + Iterator edges = GraphHelper.getEdgesForLabel(glossaryTerm, attr.getRelationshipEdgeLabel(), attr.getRelationshipEdgeDirection()); boolean excludeDeletedEntities = searchParameters.getExcludeDeletedEntities(); if (edges != null) { diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java index c9a605355d8..161f8a341da 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java @@ -127,10 +127,12 @@ public abstract class SearchProcessor { protected SearchProcessor nextProcessor; protected Predicate inMemoryPredicate; protected GraphIndexQueryBuilder graphIndexQueryBuilder; + protected ESIndexQueryBuilder esIndexQueryBuilder; protected SearchProcessor(SearchContext context) { this.context = context; this.graphIndexQueryBuilder = new GraphIndexQueryBuilder(context); + this.esIndexQueryBuilder = new ESIndexQueryBuilder(context); } public void addProcessor(SearchProcessor processor) { @@ -995,7 +997,7 @@ private static boolean hasIndexQuerySpecialChar(String attributeValue) { private static boolean isIndexQuerySpecialChar(char c) { switch (c) { case '+': - case '-': +// case '-': case '&': case '|': case '!': @@ -1011,7 +1013,7 @@ private static boolean isIndexQuerySpecialChar(char c) { case '*': case '?': case ':': - case '/': +// case '/': case '#': case '$': case '%': diff --git a/webapp/src/main/resources/atlas-application.properties b/webapp/src/main/resources/atlas-application.properties index bdc411f9b67..4fcaf3b9c9e 100644 --- a/webapp/src/main/resources/atlas-application.properties +++ b/webapp/src/main/resources/atlas-application.properties @@ -107,8 +107,9 @@ atlas.graph.index.search.elasticsearch.retry_on_conflict=5 # plugins: http://docs.janusgraph.org/latest/elasticsearch.html atlas.graph.index.search.hostname=localhost:9200 atlas.graph.index.search.elasticsearch.client-only=true +atlas.graph.index.search.max-result-set-size=1000 + # Solr-specific configuration property -# atlas.graph.index.search.max-result-set-size=150 ######### Notification Configs ######### atlas.notification.embedded=false atlas.kafka.data=${sys:atlas.home}/data/kafka @@ -255,7 +256,7 @@ atlas.search.fulltext.enable=true ######### Gremlin Search Configuration ######### #Set to false to disable gremlin search. -atlas.search.gremlin.enable=false +atlas.search.gremlin.enable=true ########## Add http headers ########### #atlas.headers.Access-Control-Allow-Origin=* #atlas.headers.Access-Control-Allow-Methods=GET,OPTIONS,HEAD,PUT,POST From a067ddd6bd4c3d3689b7cdcfd776c4826bb0df01 Mon Sep 17 00:00:00 2001 From: Gaurav Sehgal Date: Thu, 9 Jul 2020 04:14:25 +0530 Subject: [PATCH 05/12] Added apache licence to es classes --- .../janus/AtlasElasticsearchDatabase.java | 17 +++++++++++++++++ .../janus/AtlasElasticsearchIndexQuery.java | 17 +++++++++++++++++ .../atlas/discovery/ESIndexQueryBuilder.java | 17 +++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java index a69baa2e6f9..8561a6c7b68 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.atlas.repository.graphdb.janus; import org.apache.atlas.ApplicationProperties; diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchIndexQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchIndexQuery.java index 643af3c0dda..1ce7cc4728d 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchIndexQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchIndexQuery.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.atlas.repository.graphdb.janus; import org.apache.atlas.repository.graphdb.AtlasIndexQuery; diff --git a/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java b/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java index b4f7be80b1f..11b90e91595 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.atlas.discovery; import org.apache.atlas.exception.AtlasBaseException; From 6ff8968937b4593e7460634eef02efd40c60ae42 Mon Sep 17 00:00:00 2001 From: Gaurav Sehgal Date: Thu, 9 Jul 2020 17:54:44 +0530 Subject: [PATCH 06/12] Minor changes in es query --- .../apache/atlas/discovery/ESIndexQueryBuilder.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java b/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java index 11b90e91595..523defdf37a 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ESIndexQueryBuilder.java @@ -267,12 +267,12 @@ void constructSearchSource(SearchSourceBuilder sourceBuilder, String queryString boolQueryBuilder.must().add(QueryBuilders.queryStringQuery(queryString)); QueryStringQueryBuilder fullTextQueryBuilder = QueryBuilders.queryStringQuery(fullTextQuery); - fullTextQueryBuilder.fields().put(ATLAN_ASSET_TYPE + ".integrationType", 100F); - fullTextQueryBuilder.fields().put(ASSET_ENTITY_TYPE + ".__s_name", 50F); - fullTextQueryBuilder.fields().put(ATLAN_ASSET_TYPE + ".displayName", 50F); - fullTextQueryBuilder.fields().put(ASSET_ENTITY_TYPE + ".__s_owner", 20F); + fullTextQueryBuilder.fields().put(ATLAN_ASSET_TYPE + ".displayName", 500F); + fullTextQueryBuilder.fields().put(ASSET_ENTITY_TYPE + ".__s_name", 500F); + fullTextQueryBuilder.fields().put(ATLAN_ASSET_TYPE + ".integrationType", 50F); + fullTextQueryBuilder.fields().put(ASSET_ENTITY_TYPE + ".__s_owner", 50F); fullTextQueryBuilder.fields().put(REF_ASSET_TYPE + ".qualifiedName", 10F); - fullTextQueryBuilder.fields().put(ATLAN_ASSET_TYPE + ".status", 5F); + fullTextQueryBuilder.fields().put(ATLAN_ASSET_TYPE + ".status", 10F); boolQueryBuilder.must().add(fullTextQueryBuilder); sourceBuilder.query(boolQueryBuilder); From 32524a16ac281902f820080e4090beabc8ab0aef Mon Sep 17 00:00:00 2001 From: Nishant Arora <8947356+nshntarora@users.noreply.github.com> Date: Thu, 9 Jul 2020 19:07:18 +0530 Subject: [PATCH 07/12] policy evaluator api (#3) --- .../instance/AtlasEvaluatePolicyRequest.java | 98 +++++++++++++++ .../instance/AtlasEvaluatePolicyResponse.java | 112 ++++++++++++++++++ .../org/apache/atlas/web/rest/EntityREST.java | 39 +++++- 3 files changed, 244 insertions(+), 5 deletions(-) create mode 100644 intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyRequest.java create mode 100644 intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyResponse.java diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyRequest.java new file mode 100644 index 00000000000..8c81212cbf6 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyRequest.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Set; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + + +/** + * Request to run state-check of entities + */ +@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class AtlasEvaluatePolicyRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private String typeName; + private String entityGuid; + private String action; + + + public AtlasEvaluatePolicyRequest() { + } + + public String getEntityGuid() { + return entityGuid; + } + + public void setEntityGuid(String entityGuid) { + this.entityGuid = entityGuid; + } + + public String getTypeName() { + return typeName; + } + + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + + public StringBuilder toString(StringBuilder sb) { + if (sb == null) { + sb = new StringBuilder(); + } + + sb.append("AtlasEvaluatePolicyRequest{"); + sb.append("entityGuid="); + sb.append(entityGuid); + sb.append(", action=").append(action); + sb.append(", typeName=").append(typeName); + sb.append("}"); + + return sb; + } + + @Override + public String toString() { + return toString(new StringBuilder()).toString(); + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyResponse.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyResponse.java new file mode 100644 index 00000000000..020abec6465 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEvaluatePolicyResponse.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Set; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + + +/** + * Request to run state-check of entities + */ +@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class AtlasEvaluatePolicyResponse implements Serializable { + private static final long serialVersionUID = 1L; + + private String typeName; + private String entityGuid; + private String action; + private Boolean allowed; + + + public AtlasEvaluatePolicyResponse(String typeName, String entityGuid, String action, Boolean allowed) { + this.typeName = typeName; + this.entityGuid = entityGuid; + this.action = action; + this.allowed = allowed; + } + + public String getEntityGuid() { + return entityGuid; + } + + public void setEntityGuid(String entityGuid) { + this.entityGuid = entityGuid; + } + + public String getTypeName() { + return typeName; + } + + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + public Boolean getAllowed() { + return allowed; + } + + public void setAllowed(Boolean allowed) { + this.allowed = allowed; + } + + + public StringBuilder toString(StringBuilder sb) { + if (sb == null) { + sb = new StringBuilder(); + } + + sb.append("AtlasEvaluatePolicyResponse{"); + sb.append("entityGuid="); + sb.append(entityGuid); + sb.append(", action=").append(action); + sb.append(", typeName=").append(typeName); + sb.append(", allowed=").append(allowed); + sb.append("}"); + + return sb; + } + + @Override + public String toString() { + return toString(new StringBuilder()).toString(); + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 88de8b679f2..3910cab0fe4 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -21,18 +21,17 @@ import com.sun.jersey.multipart.FormDataParam; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.bulkimport.BulkImportResponse; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; -import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.instance.*; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasEntityHeaders; -import org.apache.atlas.model.instance.ClassificationAssociateRequest; -import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.repository.audit.EntityAuditRepository; import org.apache.atlas.repository.converters.AtlasInstanceConverter; @@ -136,6 +135,36 @@ public AtlasEntityWithExtInfo getById(@PathParam("guid") String guid, @QueryPara } } + /** + * Bulk API to create new entities or updates existing entities in Atlas. + * Existing entity is matched using its unique guid if supplied or by its unique attributes eg: qualifiedName + */ + @POST + @Path("/evaluator") + public List evaluatePolicies(List entities) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + List response = new ArrayList(); + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.evaluatePolicies()"); + } + + for (int i=0; i < entities.size(); i++) { + try { + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.valueOf(entities.get(i).getAction()), new AtlasEntityHeader(entities.get(i).getTypeName(), entities.get(i).getEntityGuid(), null)), entities.get(i).getAction() + "guid=", entities.get(i).getEntityGuid()); + response.add(new AtlasEvaluatePolicyResponse(entities.get(i).getTypeName(), entities.get(i).getEntityGuid(), entities.get(i).getAction(), true)); + } catch (AtlasBaseException e) { + response.add(new AtlasEvaluatePolicyResponse(entities.get(i).getTypeName(), entities.get(i).getEntityGuid(), entities.get(i).getAction(), false)); + } + } + } finally { + AtlasPerfTracer.log(perf); + } + + return response; + } /** * Get entity header given its GUID. * @param guid GUID for the entity From d51b4b60688afd66b00f76a8e118624992db03bd Mon Sep 17 00:00:00 2001 From: Gaurav Sehgal Date: Mon, 13 Jul 2020 16:22:02 +0530 Subject: [PATCH 08/12] Use entity search processor in case of AtlanAsset --- .../src/main/java/org/apache/atlas/discovery/SearchContext.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java index 9b69de4c35b..5d35af74838 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java @@ -159,6 +159,7 @@ public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRe Set allTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName()); typeAndSubTypes.addAll(allTypes); superTypes.addAll(entityType.getSuperTypes()); + superTypes.add(entityType.getTypeName()); } } From cb575fab3790e7a4cc733b023309fc8dd68e2a39 Mon Sep 17 00:00:00 2001 From: Gaurav Sehgal Date: Wed, 15 Jul 2020 17:26:14 +0530 Subject: [PATCH 09/12] Optimized apis to fetch terms and categories headers --- .../apache/atlas/repository/Constants.java | 15 + .../atlas/glossary/GlossaryService.java | 259 +++++++++++++----- .../apache/atlas/web/rest/GlossaryREST.java | 107 +++++++- 3 files changed, 301 insertions(+), 80 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 3096889263d..63657c03449 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -110,6 +110,16 @@ public final class Constants { public static final String PATCH_ACTION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.action"); public static final String PATCH_STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.state"); + /** + * Glossary property keys. + */ + public static final String TERM_DISPLAY_TEXT_KEY = "AtlasGlossaryTerm.name"; + public static final String CATEGORY_DISPLAY_TEXT_KEY = "AtlasGlossaryCategory.name"; + public static final String CATEGORY_PARENT_EDGE_LABEL = "r:AtlasGlossaryCategoryHierarchyLink"; + public static final String CATEGORY_TERMS_EDGE_LABEL = "r:AtlasGlossaryTermCategorization"; + public static final String GLOSSARY_TERMS_EDGE_LABEL = "r:AtlasGlossaryTermAnchor"; + public static final String GLOSSARY_CATEGORY_EDGE_LABEL = "r:AtlasGlossaryCategoryAnchor"; + /** * The homeId field is used when saving into Atlas a copy of an object that is being imported from another * repository. The homeId will be set to a String that identifies the other repository. The specific format @@ -209,6 +219,11 @@ public final class Constants { public static final String ATTR_NAME_REPLICATED_FROM = "replicatedFrom"; public static final Integer INCOMPLETE_ENTITY_VALUE = Integer.valueOf(1); + /* + * State values + */ + public static final String ACTIVE_STATE_VALUE = "ACTIVE"; + /* * All supported file-format extensions for Bulk Imports through file upload */ diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java index 1815aa99e4c..6f784167791 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java @@ -28,6 +28,9 @@ import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader; import org.apache.atlas.model.glossary.relations.AtlasTermCategorizationHeader; import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.ogm.DataAccess; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; @@ -37,6 +40,10 @@ import org.apache.atlas.util.FileUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.apache.tinkerpop.gremlin.process.traversal.Order; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -44,39 +51,32 @@ import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import static org.apache.atlas.glossary.GlossaryUtils.*; @Service public class GlossaryService { - private static final Logger LOG = LoggerFactory.getLogger(GlossaryService.class); - private static final boolean DEBUG_ENABLED = LOG.isDebugEnabled(); - private static final String QUALIFIED_NAME_ATTR = "qualifiedName"; - - private final DataAccess dataAccess; - private final GlossaryTermUtils glossaryTermUtils; - private final GlossaryCategoryUtils glossaryCategoryUtils; - private final AtlasTypeRegistry atlasTypeRegistry; + private static final Logger LOG = LoggerFactory.getLogger(GlossaryService.class); + private static final boolean DEBUG_ENABLED = LOG.isDebugEnabled(); + private static final String QUALIFIED_NAME_ATTR = "qualifiedName"; + + private final DataAccess dataAccess; + private final GlossaryTermUtils glossaryTermUtils; + private final GlossaryCategoryUtils glossaryCategoryUtils; + private final AtlasTypeRegistry atlasTypeRegistry; private final AtlasEntityChangeNotifier entityChangeNotifier; - private static final char[] invalidNameChars = { '@', '.' }; + private static final char[] invalidNameChars = {'@', '.'}; @Inject public GlossaryService(DataAccess dataAccess, final AtlasRelationshipStore relationshipStore, final AtlasTypeRegistry typeRegistry, AtlasEntityChangeNotifier entityChangeNotifier) { - this.dataAccess = dataAccess; - atlasTypeRegistry = typeRegistry; - glossaryTermUtils = new GlossaryTermUtils(relationshipStore, typeRegistry, dataAccess); - glossaryCategoryUtils = new GlossaryCategoryUtils(relationshipStore, typeRegistry, dataAccess); + this.dataAccess = dataAccess; + atlasTypeRegistry = typeRegistry; + glossaryTermUtils = new GlossaryTermUtils(relationshipStore, typeRegistry, dataAccess); + glossaryCategoryUtils = new GlossaryCategoryUtils(relationshipStore, typeRegistry, dataAccess); this.entityChangeNotifier = entityChangeNotifier; } @@ -95,11 +95,11 @@ public List getGlossaries(int limit, int offset, SortOrder sortOr LOG.debug("==> GlossaryService.getGlossaries({}, {}, {})", limit, offset, sortOrder); } - List glossaryGuids = AtlasGraphUtilsV2.findEntityGUIDsByType(GlossaryUtils.ATLAS_GLOSSARY_TYPENAME, sortOrder); + List glossaryGuids = AtlasGraphUtilsV2.findEntityGUIDsByType(GlossaryUtils.ATLAS_GLOSSARY_TYPENAME, sortOrder); PaginationHelper paginationHelper = new PaginationHelper<>(glossaryGuids, offset, limit); List ret; - List guidsToLoad = paginationHelper.getPaginatedList(); + List guidsToLoad = paginationHelper.getPaginatedList(); if (CollectionUtils.isNotEmpty(guidsToLoad)) { ret = guidsToLoad.stream().map(GlossaryUtils::getGlossarySkeleton).collect(Collectors.toList()); Iterable glossaries = dataAccess.load(ret); @@ -141,7 +141,7 @@ public AtlasGlossary createGlossary(AtlasGlossary atlasGlossary) throws AtlasBas if (StringUtils.isEmpty(atlasGlossary.getName())) { throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_QUALIFIED_NAME_CANT_BE_DERIVED); } - if (isNameInvalid(atlasGlossary.getName())){ + if (isNameInvalid(atlasGlossary.getName())) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } else { atlasGlossary.setQualifiedName(atlasGlossary.getName()); @@ -179,7 +179,7 @@ public AtlasGlossary getGlossary(String glossaryGuid) throws AtlasBaseException } AtlasGlossary atlasGlossary = getGlossarySkeleton(glossaryGuid); - AtlasGlossary ret = dataAccess.load(atlasGlossary); + AtlasGlossary ret = dataAccess.load(atlasGlossary); setInfoForRelations(ret); @@ -207,16 +207,16 @@ public AtlasGlossary.AtlasGlossaryExtInfo getDetailedGlossary(String glossaryGui } AtlasGlossary atlasGlossary = getGlossarySkeleton(glossaryGuid); - AtlasGlossary glossary = dataAccess.load(atlasGlossary); + AtlasGlossary glossary = dataAccess.load(atlasGlossary); AtlasGlossary.AtlasGlossaryExtInfo ret = new AtlasGlossary.AtlasGlossaryExtInfo(glossary); // Load all linked terms if (CollectionUtils.isNotEmpty(ret.getTerms())) { List termsToLoad = ret.getTerms() - .stream() - .map(id -> getAtlasGlossaryTermSkeleton(id.getTermGuid())) - .collect(Collectors.toList()); + .stream() + .map(id -> getAtlasGlossaryTermSkeleton(id.getTermGuid())) + .collect(Collectors.toList()); Iterable glossaryTerms = dataAccess.load(termsToLoad); glossaryTerms.forEach(ret::addTermInfo); } @@ -224,9 +224,9 @@ public AtlasGlossary.AtlasGlossaryExtInfo getDetailedGlossary(String glossaryGui // Load all linked categories if (CollectionUtils.isNotEmpty(ret.getCategories())) { List categoriesToLoad = ret.getCategories() - .stream() - .map(id -> getAtlasGlossaryCategorySkeleton(id.getCategoryGuid())) - .collect(Collectors.toList()); + .stream() + .map(id -> getAtlasGlossaryCategorySkeleton(id.getCategoryGuid())) + .collect(Collectors.toList()); Iterable glossaryCategories = dataAccess.load(categoriesToLoad); glossaryCategories.forEach(ret::addCategoryInfo); } @@ -310,7 +310,7 @@ public AtlasGlossaryTerm getTerm(String termGuid) throws AtlasBaseException { AtlasGlossaryTerm atlasGlossary = getAtlasGlossaryTermSkeleton(termGuid); - AtlasGlossaryTerm ret = dataAccess.load(atlasGlossary); + AtlasGlossaryTerm ret = dataAccess.load(atlasGlossary); setInfoForRelations(ret); @@ -335,12 +335,12 @@ public AtlasGlossaryTerm createTerm(AtlasGlossaryTerm glossaryTerm) throws Atlas throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_TERM_QUALIFIED_NAME_CANT_BE_DERIVED); } - if (isNameInvalid(glossaryTerm.getName())){ + if (isNameInvalid(glossaryTerm.getName())) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } else { // Derive the qualifiedName - String anchorGlossaryGuid = glossaryTerm.getAnchor().getGlossaryGuid(); - AtlasGlossary glossary = dataAccess.load(getGlossarySkeleton(anchorGlossaryGuid)); + String anchorGlossaryGuid = glossaryTerm.getAnchor().getGlossaryGuid(); + AtlasGlossary glossary = dataAccess.load(getGlossarySkeleton(anchorGlossaryGuid)); glossaryTerm.setQualifiedName(glossaryTerm.getName() + "@" + glossary.getQualifiedName()); if (LOG.isDebugEnabled()) { @@ -480,7 +480,6 @@ public void assignTermToEntities(String termGuid, List rel AtlasGlossaryTerm glossaryTerm = dataAccess.load(getAtlasGlossaryTermSkeleton(termGuid)); glossaryTermUtils.processTermAssignments(glossaryTerm, relatedObjectIds); - entityChangeNotifier.onTermAddedToEntities(glossaryTerm, relatedObjectIds); if (DEBUG_ENABLED) { @@ -519,7 +518,7 @@ public AtlasGlossaryCategory getCategory(String categoryGuid) throws AtlasBaseEx } AtlasGlossaryCategory atlasGlossary = getAtlasGlossaryCategorySkeleton(categoryGuid); - AtlasGlossaryCategory ret = dataAccess.load(atlasGlossary); + AtlasGlossaryCategory ret = dataAccess.load(atlasGlossary); setInfoForRelations(ret); @@ -544,13 +543,13 @@ public AtlasGlossaryCategory createCategory(AtlasGlossaryCategory glossaryCatego if (StringUtils.isEmpty(glossaryCategory.getName())) { throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_CATEGORY_QUALIFIED_NAME_CANT_BE_DERIVED); } - if (isNameInvalid(glossaryCategory.getName())){ + if (isNameInvalid(glossaryCategory.getName())) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } else { // Derive the qualifiedName String anchorGlossaryGuid = glossaryCategory.getAnchor().getGlossaryGuid(); AtlasGlossary glossary = dataAccess.load(getGlossarySkeleton(anchorGlossaryGuid)); - glossaryCategory.setQualifiedName(glossaryCategory.getName()+ "@" + glossary.getQualifiedName()); + glossaryCategory.setQualifiedName(glossaryCategory.getName() + "@" + glossary.getQualifiedName()); if (LOG.isDebugEnabled()) { LOG.debug("Derived qualifiedName = {}", glossaryCategory.getQualifiedName()); @@ -710,8 +709,8 @@ public List getGlossaryTermsHeaders(String glossaryGuid, List terms = new ArrayList<>(glossary.getTerms()); if (sortOrder != null) { terms.sort((o1, o2) -> sortOrder == SortOrder.ASCENDING ? - o1.getDisplayText().compareTo(o2.getDisplayText()) : - o2.getDisplayText().compareTo(o1.getDisplayText())); + o1.getDisplayText().compareTo(o2.getDisplayText()) : + o2.getDisplayText().compareTo(o1.getDisplayText())); } ret = new PaginationHelper<>(terms, offset, limit).getPaginatedList(); } else { @@ -725,6 +724,128 @@ public List getGlossaryTermsHeaders(String glossaryGuid, return ret; } + + @GraphTransaction + public List getCategoryTermsHeaders(String categoryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException { + if (Objects.isNull(categoryGuid)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "categoryGuid is null/empty"); + } + + if (DEBUG_ENABLED) { + LOG.debug("==> GlossaryService.getCategoryTermsHeaders({}, {}, {}, {})", categoryGuid, offset, limit, sortOrder); + } + + List ret = new ArrayList<>(); + AtlasGraph graph = AtlasGraphProvider.getGraphInstance(); + + GraphTraversal query = graph.V().has(Constants.GUID_PROPERTY_KEY, categoryGuid) + .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)) + .out(Constants.CATEGORY_TERMS_EDGE_LABEL) + .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)); + + runPaginatedTermsQuery(offset, limit, sortOrder, ret, query); + + return ret; + } + + @GraphTransaction + public List getGlossaryTermsHeaders(String glossaryGuid, int offset, int limit, SortOrder sortOrder, boolean isRoot) throws AtlasBaseException { + if (Objects.isNull(glossaryGuid)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryGuid is null/empty"); + } + + if (DEBUG_ENABLED) { + LOG.debug("==> GlossaryService.getGlossaryTermsHeaders({}, {}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder, isRoot); + } + + List ret = new ArrayList<>(); + AtlasGraph graph = AtlasGraphProvider.getGraphInstance(); + + GraphTraversal query = graph.V().has(Constants.GUID_PROPERTY_KEY, glossaryGuid) + .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)) + .out(Constants.GLOSSARY_TERMS_EDGE_LABEL) + .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)); + + if (isRoot) { + query = query.where(__.inE(Constants.CATEGORY_TERMS_EDGE_LABEL).count().is(P.eq(0))); + } + + runPaginatedTermsQuery(offset, limit, sortOrder, ret, query); + + return ret; + } + + private void runPaginatedTermsQuery(int offset, int limit, SortOrder sortOrder, List ret, GraphTraversal baseQuery) { + if (sortOrder != null) { + Order order = sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc; + baseQuery = baseQuery.order().by(Constants.TERM_DISPLAY_TEXT_KEY, order); + } + + Set>> results = baseQuery + .valueMap(Constants.GUID_PROPERTY_KEY, Constants.TERM_DISPLAY_TEXT_KEY) + .range(offset, limit + offset).toSet(); + + constructTermsHeaders(ret, results); + } + + private void constructTermsHeaders(List ret, Set>> queryResult) { + for (Map> res : queryResult) { + AtlasRelatedTermHeader atlasRelatedTermHeader = new AtlasRelatedTermHeader(); + atlasRelatedTermHeader.setTermGuid(res.get(Constants.GUID_PROPERTY_KEY).get(0)); + atlasRelatedTermHeader.setDisplayText(res.get(Constants.TERM_DISPLAY_TEXT_KEY).get(0)); + ret.add(atlasRelatedTermHeader); + } + } + + @GraphTransaction + public List getGlossaryCategoriesHeadersOptimized(String glossaryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException { + if (Objects.isNull(glossaryGuid)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryGuid is null/empty"); + } + + if (DEBUG_ENABLED) { + LOG.debug("==> GlossaryService.getGlossaryCategoriesHeadersOptimized({}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder); + } + + List ret = new ArrayList<>(); + AtlasGraph graph = AtlasGraphProvider.getGraphInstance(); + + GraphTraversal query = graph.V() + .has(Constants.GUID_PROPERTY_KEY, glossaryGuid) + .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)) + .out(Constants.GLOSSARY_CATEGORY_EDGE_LABEL) + .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)); + + if (sortOrder != null) { + Order order = sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc; + query = query.order().by(Constants.CATEGORY_DISPLAY_TEXT_KEY, order); + } + + Set>> results = query + .valueMap(Constants.GUID_PROPERTY_KEY, Constants.CATEGORY_DISPLAY_TEXT_KEY) + .range(offset, limit + offset).toSet(); + + + for (Map> res : results) { + AtlasRelatedCategoryHeader atlasRelatedCategoryHeader = new AtlasRelatedCategoryHeader(); + atlasRelatedCategoryHeader.setCategoryGuid(res.get(Constants.GUID_PROPERTY_KEY).get(0)); + atlasRelatedCategoryHeader.setDisplayText(res.get(Constants.CATEGORY_DISPLAY_TEXT_KEY).get(0)); + + Set parentResults = graph.V() + .has(Constants.GUID_PROPERTY_KEY, atlasRelatedCategoryHeader.getCategoryGuid()) + .in(Constants.CATEGORY_PARENT_EDGE_LABEL) + .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)) + .values(Constants.GUID_PROPERTY_KEY).toSet(); + + if (!parentResults.isEmpty()) { + atlasRelatedCategoryHeader.setParentCategoryGuid(parentResults.iterator().next()); + } + + ret.add(atlasRelatedCategoryHeader); + } + return ret; + } + @GraphTransaction public List getGlossaryTerms(String glossaryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException { if (Objects.isNull(glossaryGuid)) { @@ -767,8 +888,8 @@ public List getGlossaryCategoriesHeaders(String glos List categories = new ArrayList<>(glossary.getCategories()); if (sortOrder != null) { categories.sort((o1, o2) -> sortOrder == SortOrder.ASCENDING ? - o1.getDisplayText().compareTo(o2.getDisplayText()) : - o2.getDisplayText().compareTo(o1.getDisplayText())); + o1.getDisplayText().compareTo(o2.getDisplayText()) : + o2.getDisplayText().compareTo(o1.getDisplayText())); } ret = new PaginationHelper<>(categories, offset, limit).getPaginatedList(); } else { @@ -824,8 +945,8 @@ public List getCategoryTerms(String categoryGuid, int of List terms = new ArrayList<>(glossaryCategory.getTerms()); if (sortOrder != null) { terms.sort((o1, o2) -> sortOrder == SortOrder.ASCENDING ? - o1.getDisplayText().compareTo(o2.getDisplayText()) : - o2.getDisplayText().compareTo(o1.getDisplayText())); + o1.getDisplayText().compareTo(o2.getDisplayText()) : + o2.getDisplayText().compareTo(o1.getDisplayText())); } ret = new PaginationHelper<>(terms, offset, limit).getPaginatedList(); } else { @@ -848,7 +969,7 @@ public Map> getRelatedTe LOG.debug("==> GlossaryService.getRelatedTerms({}, {}, {}, {})", termGuid, offset, limit, sortOrder); } - AtlasGlossaryTerm glossaryTerm = getTerm(termGuid); + AtlasGlossaryTerm glossaryTerm = getTerm(termGuid); Map> ret; if (glossaryTerm.hasTerms()) { ret = glossaryTerm.getRelatedTerms(); @@ -899,7 +1020,7 @@ public List getAssignedEntities(final String termGuid, int throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "termGuid is null/empty"); } - AtlasGlossaryTerm glossaryTerm = dataAccess.load(getAtlasGlossaryTermSkeleton(termGuid)); + AtlasGlossaryTerm glossaryTerm = dataAccess.load(getAtlasGlossaryTermSkeleton(termGuid)); Set assignedEntities = glossaryTerm.getAssignedEntities(); List ret; @@ -907,8 +1028,8 @@ public List getAssignedEntities(final String termGuid, int ret = new ArrayList<>(assignedEntities); if (sortOrder != null) { ret.sort((o1, o2) -> sortOrder == SortOrder.ASCENDING ? - o1.getDisplayText().compareTo(o2.getDisplayText()) : - o2.getDisplayText().compareTo(o1.getDisplayText())); + o1.getDisplayText().compareTo(o2.getDisplayText()) : + o2.getDisplayText().compareTo(o1.getDisplayText())); } ret = new PaginationHelper<>(assignedEntities, offset, limit).getPaginatedList(); } else { @@ -995,9 +1116,9 @@ private void setInfoForRelations(final AtlasGlossaryCategory glossaryCategory) t private void setDisplayNameForTermCategories(final Set categorizationHeaders) throws AtlasBaseException { List categories = categorizationHeaders - .stream() - .map(id -> getAtlasGlossaryCategorySkeleton(id.getCategoryGuid())) - .collect(Collectors.toList()); + .stream() + .map(id -> getAtlasGlossaryCategorySkeleton(id.getCategoryGuid())) + .collect(Collectors.toList()); Map categoryMap = new HashMap<>(); dataAccess.load(categories).forEach(c -> categoryMap.put(c.getGuid(), c)); categorizationHeaders.forEach(c -> c.setDisplayText(categoryMap.get(c.getCategoryGuid()).getName())); @@ -1005,9 +1126,9 @@ private void setDisplayNameForTermCategories(final Set categoryHeaders) throws AtlasBaseException { List categories = categoryHeaders - .stream() - .map(id -> getAtlasGlossaryCategorySkeleton(id.getCategoryGuid())) - .collect(Collectors.toList()); + .stream() + .map(id -> getAtlasGlossaryCategorySkeleton(id.getCategoryGuid())) + .collect(Collectors.toList()); Map categoryMap = new HashMap<>(); dataAccess.load(categories).forEach(c -> categoryMap.put(c.getGuid(), c)); for (AtlasRelatedCategoryHeader c : categoryHeaders) { @@ -1021,9 +1142,9 @@ private void setInfoForRelatedCategories(final Collection termHeaders) throws AtlasBaseException { List terms = termHeaders - .stream() - .map(id -> getAtlasGlossaryTermSkeleton(id.getTermGuid())) - .collect(Collectors.toList()); + .stream() + .map(id -> getAtlasGlossaryTermSkeleton(id.getTermGuid())) + .collect(Collectors.toList()); Map termMap = new HashMap<>(); dataAccess.load(terms).iterator().forEachRemaining(t -> termMap.put(t.getGuid(), t)); @@ -1039,9 +1160,9 @@ private String getDisplayText(AtlasGlossaryTerm term) { } static class PaginationHelper { - private int pageStart; - private int pageEnd; - private int maxSize; + private int pageStart; + private int pageEnd; + private int maxSize; private List items; PaginationHelper(Collection items, int offset, int limit) { @@ -1089,8 +1210,8 @@ public List importGlossaryData(InputStream inputStream, Strin throw new AtlasBaseException(AtlasErrorCode.INVALID_FILE_TYPE, fileName); } - List fileData = FileUtils.readFileData(fileName, inputStream); - List failedTermMsgs = new ArrayList<>(); + List fileData = FileUtils.readFileData(fileName, inputStream); + List failedTermMsgs = new ArrayList<>(); ret = glossaryTermUtils.getGlossaryTermDataList(fileData, failedTermMsgs); ret = createGlossaryTerms(ret); @@ -1109,11 +1230,11 @@ public List importGlossaryTermEntitiesData(InputStream inputS throw new AtlasBaseException(AtlasErrorCode.INVALID_FILE_TYPE, fileName); } - List fileData = FileUtils.readFileData(fileName, inputStream); - List failedTermMsgs = new ArrayList<>(); + List fileData = FileUtils.readFileData(fileName, inputStream); + List failedTermMsgs = new ArrayList<>(); ret = glossaryTermUtils.getGlossaryTermEntitiesDataList(fileData, failedTermMsgs); - for (AtlasGlossaryTerm term: ret) { + for (AtlasGlossaryTerm term : ret) { assignTermToEntities(term.getGuid(), new ArrayList<>(term.getAssignedEntities())); } } catch (IOException e) { @@ -1145,8 +1266,8 @@ public List importGlossaryCategoryData(InputStream inputS throw new AtlasBaseException(AtlasErrorCode.INVALID_FILE_TYPE, fileName); } - List fileData = FileUtils.readFileData(fileName, inputStream); - List failedCategoryMsgs = new ArrayList<>(); + List fileData = FileUtils.readFileData(fileName, inputStream); + List failedCategoryMsgs = new ArrayList<>(); ret = glossaryCategoryUtils.getGlossaryCategoryDataList(fileData, failedCategoryMsgs); ret = createGlossaryCategories(ret); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java index 26165a77502..2f12856815f 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java @@ -39,17 +39,8 @@ import org.springframework.stereotype.Service; import javax.inject.Inject; -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.WebApplicationException; +import javax.script.ScriptException; +import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.StreamingOutput; import java.io.IOException; @@ -647,6 +638,37 @@ public List getGlossaryTerms(@PathParam("glossaryGuid") Strin } } + /** + * Get root terms belonging to a specific glossary + * @param glossaryGuid unique identifier for glossary + * @param limit page size - by default there is no paging + * @param offset starting offset for loading terms + * @param sort ASC(default) or DESC + * @return List of terms associated with the glossary + * @throws AtlasBaseException + * @HTTP 200 List of glossary terms for the given glossary or an empty list + * @HTTP 404 If glossary guid in invalid + */ + @GET + @Path("/{glossaryGuid}/terms/headers/optimized") + public List getGlossaryTermsHeaderOptimized(@PathParam("glossaryGuid") String glossaryGuid, + @DefaultValue("-1") @QueryParam("limit") String limit, + @DefaultValue("0") @QueryParam("offset") String offset, + @DefaultValue("ASC") @QueryParam("sort") final String sort, + @DefaultValue("true") @QueryParam("isRoot") final boolean isRoot) throws AtlasBaseException, ScriptException { + Servlets.validateQueryParamLength("glossaryGuid", glossaryGuid); + AtlasPerfTracer perf = null; + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "GlossaryREST.getGlossaryRootTermsHeader(" + glossaryGuid + ")"); + } + + return glossaryService.getGlossaryTermsHeaders(glossaryGuid, Integer.parseInt(offset), Integer.parseInt(limit), toSortOrder(sort), isRoot); + } finally { + AtlasPerfTracer.log(perf); + } + } + /** * Get term headers belonging to a specific glossary * @param glossaryGuid unique identifier for glossary @@ -740,6 +762,37 @@ public List getGlossaryCategoriesHeaders(@PathParam( } } + /** + * Get the categories belonging to a specific glossary optimized + * @param glossaryGuid unique identifier for glossary term + * @param limit page size - by default there is no paging + * @param offset offset for pagination purpose + * @param sort ASC (default) or DESC + * @return List of associated categories + * @throws AtlasBaseException + * @HTTP 200 List of glossary categories for the given glossary or an empty list + * @HTTP 404 If glossary guid in invalid + */ + @GET + @Path("/{glossaryGuid}/categories/headers/optimized") + public List getGlossaryCategoriesHeadersOptimized(@PathParam("glossaryGuid") String glossaryGuid, + @DefaultValue("-1") @QueryParam("limit") String limit, + @DefaultValue("0") @QueryParam("offset") String offset, + @DefaultValue("ASC") @QueryParam("sort") final String sort) throws AtlasBaseException { + Servlets.validateQueryParamLength("glossaryGuid", glossaryGuid); + AtlasPerfTracer perf = null; + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "GlossaryREST.getGlossaryCategoriesHeadersOptimized(" + glossaryGuid + ")"); + } + + return glossaryService.getGlossaryCategoriesHeadersOptimized(glossaryGuid, Integer.parseInt(offset), Integer.parseInt(limit), toSortOrder(sort)); + + } finally { + AtlasPerfTracer.log(perf); + } + } + /** * Get all terms associated with the specific category * @param categoryGuid unique identifier for glossary category @@ -772,6 +825,38 @@ public List getCategoryTerms(@PathParam("categoryGuid") } } + /** + * Get all terms associated with the specific category + * @param categoryGuid unique identifier for glossary category + * @param limit page size - by default there is no paging + * @param offset offset for pagination purpose + * @param sort ASC (default) or DESC + * @return List of associated terms + * @throws AtlasBaseException + * @HTTP 200 List of terms for the given category or an empty list + * @HTTP 404 If glossary category guid in invalid + */ + @GET + @Path("/category/{categoryGuid}/terms/optimized") + public List getCategoryTermOptimized(@PathParam("categoryGuid") String categoryGuid, + @DefaultValue("-1") @QueryParam("limit") String limit, + @DefaultValue("0") @QueryParam("offset") String offset, + @DefaultValue("ASC") @QueryParam("sort") final String sort) throws AtlasBaseException { + Servlets.validateQueryParamLength("categoryGuid", categoryGuid); + + AtlasPerfTracer perf = null; + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "GlossaryREST.getCategoryTerms(" + categoryGuid + ")"); + } + + return glossaryService.getCategoryTermsHeaders(categoryGuid, Integer.parseInt(offset), Integer.parseInt(limit), toSortOrder(sort)); + + } finally { + AtlasPerfTracer.log(perf); + } + } + /** * Get all related terms for a specific term * @param termGuid unique identifier for glossary term From ed14159a56795cc33a2df83d095dde914adb67e6 Mon Sep 17 00:00:00 2001 From: Gaurav Sehgal Date: Wed, 15 Jul 2020 18:00:53 +0530 Subject: [PATCH 10/12] Rename methods --- .../main/java/org/apache/atlas/glossary/GlossaryService.java | 4 ++-- .../src/main/java/org/apache/atlas/web/rest/GlossaryREST.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java index 6f784167791..73a3d6a43db 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java @@ -726,7 +726,7 @@ public List getGlossaryTermsHeaders(String glossaryGuid, @GraphTransaction - public List getCategoryTermsHeaders(String categoryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException { + public List getCategoryTermsHeadersOptimized(String categoryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException { if (Objects.isNull(categoryGuid)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "categoryGuid is null/empty"); } @@ -749,7 +749,7 @@ public List getCategoryTermsHeaders(String categoryGuid, } @GraphTransaction - public List getGlossaryTermsHeaders(String glossaryGuid, int offset, int limit, SortOrder sortOrder, boolean isRoot) throws AtlasBaseException { + public List getGlossaryTermsHeadersOptimized(String glossaryGuid, int offset, int limit, SortOrder sortOrder, boolean isRoot) throws AtlasBaseException { if (Objects.isNull(glossaryGuid)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryGuid is null/empty"); } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java index 2f12856815f..980754fdbb5 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java @@ -663,7 +663,7 @@ public List getGlossaryTermsHeaderOptimized(@PathParam(" perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "GlossaryREST.getGlossaryRootTermsHeader(" + glossaryGuid + ")"); } - return glossaryService.getGlossaryTermsHeaders(glossaryGuid, Integer.parseInt(offset), Integer.parseInt(limit), toSortOrder(sort), isRoot); + return glossaryService.getGlossaryTermsHeadersOptimized(glossaryGuid, Integer.parseInt(offset), Integer.parseInt(limit), toSortOrder(sort), isRoot); } finally { AtlasPerfTracer.log(perf); } @@ -850,7 +850,7 @@ public List getCategoryTermOptimized(@PathParam("categor perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "GlossaryREST.getCategoryTerms(" + categoryGuid + ")"); } - return glossaryService.getCategoryTermsHeaders(categoryGuid, Integer.parseInt(offset), Integer.parseInt(limit), toSortOrder(sort)); + return glossaryService.getCategoryTermsHeadersOptimized(categoryGuid, Integer.parseInt(offset), Integer.parseInt(limit), toSortOrder(sort)); } finally { AtlasPerfTracer.log(perf); From 0e51efa05283f8b92f871ca4456f10044699b8f3 Mon Sep 17 00:00:00 2001 From: Gaurav Sehgal Date: Wed, 15 Jul 2020 18:46:31 +0530 Subject: [PATCH 11/12] Added glossary and category authorization in optimized apis --- .../org/apache/atlas/glossary/GlossaryService.java | 13 +++++++++++++ .../org/apache/atlas/glossary/GlossaryUtils.java | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java index 73a3d6a43db..1e5a3e34c63 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java @@ -20,6 +20,9 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.SortOrder; import org.apache.atlas.annotation.GraphTransaction; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.glossary.AtlasGlossary; import org.apache.atlas.model.glossary.AtlasGlossaryCategory; @@ -27,6 +30,7 @@ import org.apache.atlas.model.glossary.relations.AtlasRelatedCategoryHeader; import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader; import org.apache.atlas.model.glossary.relations.AtlasTermCategorizationHeader; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.AtlasGraphProvider; @@ -735,6 +739,9 @@ public List getCategoryTermsHeadersOptimized(String cate LOG.debug("==> GlossaryService.getCategoryTermsHeaders({}, {}, {}, {})", categoryGuid, offset, limit, sortOrder); } + AtlasAuthorizationUtils.verifyAccess( + new AtlasEntityAccessRequest(atlasTypeRegistry, AtlasPrivilege.ENTITY_READ, GlossaryUtils.getAtlasCategoryEntityHeaderSkeleton(categoryGuid))); + List ret = new ArrayList<>(); AtlasGraph graph = AtlasGraphProvider.getGraphInstance(); @@ -758,6 +765,9 @@ public List getGlossaryTermsHeadersOptimized(String glos LOG.debug("==> GlossaryService.getGlossaryTermsHeaders({}, {}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder, isRoot); } + AtlasAuthorizationUtils.verifyAccess( + new AtlasEntityAccessRequest(atlasTypeRegistry, AtlasPrivilege.ENTITY_READ, GlossaryUtils.getAtlasGlossaryEntityHeaderSkeleton(glossaryGuid))); + List ret = new ArrayList<>(); AtlasGraph graph = AtlasGraphProvider.getGraphInstance(); @@ -807,6 +817,9 @@ public List getGlossaryCategoriesHeadersOptimized(St LOG.debug("==> GlossaryService.getGlossaryCategoriesHeadersOptimized({}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder); } + AtlasAuthorizationUtils.verifyAccess( + new AtlasEntityAccessRequest(atlasTypeRegistry, AtlasPrivilege.ENTITY_READ, GlossaryUtils.getAtlasGlossaryEntityHeaderSkeleton(glossaryGuid))); + List ret = new ArrayList<>(); AtlasGraph graph = AtlasGraphProvider.getGraphInstance(); diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java index 2a2cebb43d6..3e7f7d46524 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java @@ -22,6 +22,7 @@ import org.apache.atlas.model.glossary.AtlasGlossaryCategory; import org.apache.atlas.model.glossary.AtlasGlossaryTerm; import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.repository.ogm.DataAccess; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; @@ -88,7 +89,13 @@ public static AtlasGlossaryCategory getAtlasGlossaryCategorySkeleton(final Strin return glossaryCategory; } + public static AtlasEntityHeader getAtlasGlossaryEntityHeaderSkeleton(final String glossaryGuid) { + return new AtlasEntityHeader(ATLAS_GLOSSARY_TYPENAME, glossaryGuid, null); + } + public static AtlasEntityHeader getAtlasCategoryEntityHeaderSkeleton(final String categoryGuid) { + return new AtlasEntityHeader(ATLAS_GLOSSARY_CATEGORY_TYPENAME, categoryGuid, null); + } protected void createRelationship(AtlasRelationship relationship) throws AtlasBaseException { relationshipStore.getOrCreate(relationship); From c4e48d120c76c481703977776eb742fd76c3ab82 Mon Sep 17 00:00:00 2001 From: Gaurav Sehgal Date: Fri, 17 Jul 2020 15:37:24 +0530 Subject: [PATCH 12/12] Added qualified name in terms and category optimized headers endpoint --- .../apache/atlas/repository/Constants.java | 10 ---- .../relations/AtlasRelatedCategoryHeader.java | 19 ++++-- .../relations/AtlasRelatedTermHeader.java | 25 +++++--- .../atlas/glossary/GlossaryService.java | 25 ++++---- .../apache/atlas/glossary/GlossaryUtils.java | 59 +++++++++++-------- .../org/apache/atlas/web/rest/TypesREST.java | 19 +----- 6 files changed, 83 insertions(+), 74 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 63657c03449..6bf43f139e5 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -110,16 +110,6 @@ public final class Constants { public static final String PATCH_ACTION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.action"); public static final String PATCH_STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.state"); - /** - * Glossary property keys. - */ - public static final String TERM_DISPLAY_TEXT_KEY = "AtlasGlossaryTerm.name"; - public static final String CATEGORY_DISPLAY_TEXT_KEY = "AtlasGlossaryCategory.name"; - public static final String CATEGORY_PARENT_EDGE_LABEL = "r:AtlasGlossaryCategoryHierarchyLink"; - public static final String CATEGORY_TERMS_EDGE_LABEL = "r:AtlasGlossaryTermCategorization"; - public static final String GLOSSARY_TERMS_EDGE_LABEL = "r:AtlasGlossaryTermAnchor"; - public static final String GLOSSARY_CATEGORY_EDGE_LABEL = "r:AtlasGlossaryCategoryAnchor"; - /** * The homeId field is used when saving into Atlas a copy of an object that is being imported from another * repository. The homeId will be set to a String that identifies the other repository. The specific format diff --git a/intg/src/main/java/org/apache/atlas/model/glossary/relations/AtlasRelatedCategoryHeader.java b/intg/src/main/java/org/apache/atlas/model/glossary/relations/AtlasRelatedCategoryHeader.java index 596481a4af0..58a3827fac8 100644 --- a/intg/src/main/java/org/apache/atlas/model/glossary/relations/AtlasRelatedCategoryHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/glossary/relations/AtlasRelatedCategoryHeader.java @@ -28,6 +28,7 @@ public class AtlasRelatedCategoryHeader { private String relationGuid; private String displayText; private String description; + private String qualifiedName; public AtlasRelatedCategoryHeader() { } @@ -56,21 +57,30 @@ public void setDisplayText(final String displayText) { this.displayText = displayText; } + public String getQualifiedName() { + return qualifiedName; + } + + public void setQualifiedName(final String qualifiedName) { + this.qualifiedName = qualifiedName; + } + @Override public boolean equals(final Object o) { if (this == o) return true; if (!(o instanceof AtlasRelatedCategoryHeader)) return false; final AtlasRelatedCategoryHeader that = (AtlasRelatedCategoryHeader) o; return Objects.equals(categoryGuid, that.categoryGuid) && - Objects.equals(parentCategoryGuid, that.parentCategoryGuid) && - Objects.equals(relationGuid, that.relationGuid) && - Objects.equals(description, that.description); + Objects.equals(parentCategoryGuid, that.parentCategoryGuid) && + Objects.equals(relationGuid, that.relationGuid) && + Objects.equals(description, that.description) && + Objects.equals(qualifiedName, that.qualifiedName); } @Override public int hashCode() { - return Objects.hash(categoryGuid, parentCategoryGuid, relationGuid, description); + return Objects.hash(categoryGuid, parentCategoryGuid, relationGuid, description, qualifiedName); } @Override @@ -81,6 +91,7 @@ public String toString() { sb.append(", relationGuid='").append(relationGuid).append('\''); sb.append(", displayText='").append(displayText).append('\''); sb.append(", description='").append(description).append('\''); + sb.append(", qualifiedName='").append(qualifiedName).append('\''); sb.append('}'); return sb.toString(); } diff --git a/intg/src/main/java/org/apache/atlas/model/glossary/relations/AtlasRelatedTermHeader.java b/intg/src/main/java/org/apache/atlas/model/glossary/relations/AtlasRelatedTermHeader.java index f16cf25ed22..5e9fc2febc3 100644 --- a/intg/src/main/java/org/apache/atlas/model/glossary/relations/AtlasRelatedTermHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/glossary/relations/AtlasRelatedTermHeader.java @@ -31,6 +31,7 @@ public class AtlasRelatedTermHeader { private String expression; private String steward; private String source; + private String qualifiedName; private AtlasTermRelationshipStatus status; @@ -69,6 +70,14 @@ public void setSteward(final String steward) { this.steward = steward; } + public String getQualifiedName() { + return qualifiedName; + } + + public void setQualifiedName(final String qualifiedName) { + this.qualifiedName = qualifiedName; + } + public String getSource() { return source; } @@ -91,18 +100,19 @@ public boolean equals(final Object o) { if (!(o instanceof AtlasRelatedTermHeader)) return false; final AtlasRelatedTermHeader that = (AtlasRelatedTermHeader) o; return Objects.equals(termGuid, that.termGuid) && - Objects.equals(relationGuid, that.relationGuid) && - Objects.equals(description, that.description) && - Objects.equals(expression, that.expression) && - Objects.equals(steward, that.steward) && - Objects.equals(source, that.source) && - status == that.status; + Objects.equals(relationGuid, that.relationGuid) && + Objects.equals(description, that.description) && + Objects.equals(expression, that.expression) && + Objects.equals(steward, that.steward) && + Objects.equals(source, that.source) && + Objects.equals(qualifiedName, that.qualifiedName) && + status == that.status; } @Override public int hashCode() { - return Objects.hash(termGuid, relationGuid, description, expression, steward, source, status); + return Objects.hash(termGuid, relationGuid, description, expression, steward, source, qualifiedName, status); } @Override @@ -115,6 +125,7 @@ public String toString() { sb.append(", expression='").append(expression).append('\''); sb.append(", steward='").append(steward).append('\''); sb.append(", source='").append(source).append('\''); + sb.append(", qualifiedName='").append(qualifiedName).append('\''); sb.append(", status=").append(status); sb.append('}'); return sb.toString(); diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java index 1e5a3e34c63..e1c0c5330eb 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java @@ -30,7 +30,6 @@ import org.apache.atlas.model.glossary.relations.AtlasRelatedCategoryHeader; import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader; import org.apache.atlas.model.glossary.relations.AtlasTermCategorizationHeader; -import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.AtlasGraphProvider; @@ -747,7 +746,7 @@ public List getCategoryTermsHeadersOptimized(String cate GraphTraversal query = graph.V().has(Constants.GUID_PROPERTY_KEY, categoryGuid) .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)) - .out(Constants.CATEGORY_TERMS_EDGE_LABEL) + .out(CATEGORY_TERMS_EDGE_LABEL) .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)); runPaginatedTermsQuery(offset, limit, sortOrder, ret, query); @@ -773,11 +772,11 @@ public List getGlossaryTermsHeadersOptimized(String glos GraphTraversal query = graph.V().has(Constants.GUID_PROPERTY_KEY, glossaryGuid) .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)) - .out(Constants.GLOSSARY_TERMS_EDGE_LABEL) + .out(GLOSSARY_TERMS_EDGE_LABEL) .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)); if (isRoot) { - query = query.where(__.inE(Constants.CATEGORY_TERMS_EDGE_LABEL).count().is(P.eq(0))); + query = query.where(__.inE(CATEGORY_TERMS_EDGE_LABEL).count().is(P.eq(0))); } runPaginatedTermsQuery(offset, limit, sortOrder, ret, query); @@ -788,11 +787,11 @@ public List getGlossaryTermsHeadersOptimized(String glos private void runPaginatedTermsQuery(int offset, int limit, SortOrder sortOrder, List ret, GraphTraversal baseQuery) { if (sortOrder != null) { Order order = sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc; - baseQuery = baseQuery.order().by(Constants.TERM_DISPLAY_TEXT_KEY, order); + baseQuery = baseQuery.order().by(TERM_DISPLAY_TEXT_KEY, order); } Set>> results = baseQuery - .valueMap(Constants.GUID_PROPERTY_KEY, Constants.TERM_DISPLAY_TEXT_KEY) + .valueMap(Constants.GUID_PROPERTY_KEY, TERM_DISPLAY_TEXT_KEY, TERM_QUALIFIED_NAME_KEY) .range(offset, limit + offset).toSet(); constructTermsHeaders(ret, results); @@ -802,7 +801,8 @@ private void constructTermsHeaders(List ret, Set> res : queryResult) { AtlasRelatedTermHeader atlasRelatedTermHeader = new AtlasRelatedTermHeader(); atlasRelatedTermHeader.setTermGuid(res.get(Constants.GUID_PROPERTY_KEY).get(0)); - atlasRelatedTermHeader.setDisplayText(res.get(Constants.TERM_DISPLAY_TEXT_KEY).get(0)); + atlasRelatedTermHeader.setDisplayText(res.get(TERM_DISPLAY_TEXT_KEY).get(0)); + atlasRelatedTermHeader.setQualifiedName(res.get(TERM_QUALIFIED_NAME_KEY).get(0)); ret.add(atlasRelatedTermHeader); } } @@ -826,27 +826,28 @@ public List getGlossaryCategoriesHeadersOptimized(St GraphTraversal query = graph.V() .has(Constants.GUID_PROPERTY_KEY, glossaryGuid) .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)) - .out(Constants.GLOSSARY_CATEGORY_EDGE_LABEL) + .out(GLOSSARY_CATEGORY_EDGE_LABEL) .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)); if (sortOrder != null) { Order order = sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc; - query = query.order().by(Constants.CATEGORY_DISPLAY_TEXT_KEY, order); + query = query.order().by(CATEGORY_DISPLAY_TEXT_KEY, order); } Set>> results = query - .valueMap(Constants.GUID_PROPERTY_KEY, Constants.CATEGORY_DISPLAY_TEXT_KEY) + .valueMap(Constants.GUID_PROPERTY_KEY, CATEGORY_DISPLAY_TEXT_KEY, CATEGORY_QUALIFIED_NAME_KEY) .range(offset, limit + offset).toSet(); for (Map> res : results) { AtlasRelatedCategoryHeader atlasRelatedCategoryHeader = new AtlasRelatedCategoryHeader(); atlasRelatedCategoryHeader.setCategoryGuid(res.get(Constants.GUID_PROPERTY_KEY).get(0)); - atlasRelatedCategoryHeader.setDisplayText(res.get(Constants.CATEGORY_DISPLAY_TEXT_KEY).get(0)); + atlasRelatedCategoryHeader.setDisplayText(res.get(CATEGORY_DISPLAY_TEXT_KEY).get(0)); + atlasRelatedCategoryHeader.setQualifiedName(res.get(CATEGORY_QUALIFIED_NAME_KEY).get(0)); Set parentResults = graph.V() .has(Constants.GUID_PROPERTY_KEY, atlasRelatedCategoryHeader.getCategoryGuid()) - .in(Constants.CATEGORY_PARENT_EDGE_LABEL) + .in(CATEGORY_PARENT_EDGE_LABEL) .has(Constants.STATE_PROPERTY_KEY, P.within(Constants.ACTIVE_STATE_VALUE)) .values(Constants.GUID_PROPERTY_KEY).toSet(); diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java index 3e7f7d46524..7fed09fd3ca 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java @@ -33,37 +33,48 @@ public abstract class GlossaryUtils { public static final String TERM_ASSIGNMENT_ATTR_DESCRIPTION = "description"; - public static final String TERM_ASSIGNMENT_ATTR_EXPRESSION = "expression"; - public static final String TERM_ASSIGNMENT_ATTR_STATUS = "status"; - public static final String TERM_ASSIGNMENT_ATTR_CONFIDENCE = "confidence"; - public static final String TERM_ASSIGNMENT_ATTR_CREATED_BY = "createdBy"; - public static final String TERM_ASSIGNMENT_ATTR_STEWARD = "steward"; - public static final String TERM_ASSIGNMENT_ATTR_SOURCE = "source"; - - static final String ATLAS_GLOSSARY_TYPENAME = "AtlasGlossary"; - static final String ATLAS_GLOSSARY_TERM_TYPENAME = "AtlasGlossaryTerm"; + public static final String TERM_ASSIGNMENT_ATTR_EXPRESSION = "expression"; + public static final String TERM_ASSIGNMENT_ATTR_STATUS = "status"; + public static final String TERM_ASSIGNMENT_ATTR_CONFIDENCE = "confidence"; + public static final String TERM_ASSIGNMENT_ATTR_CREATED_BY = "createdBy"; + public static final String TERM_ASSIGNMENT_ATTR_STEWARD = "steward"; + public static final String TERM_ASSIGNMENT_ATTR_SOURCE = "source"; + + static final String ATLAS_GLOSSARY_TYPENAME = "AtlasGlossary"; + static final String ATLAS_GLOSSARY_TERM_TYPENAME = "AtlasGlossaryTerm"; static final String ATLAS_GLOSSARY_CATEGORY_TYPENAME = "AtlasGlossaryCategory"; - public static final String NAME = "name"; - public static final String QUALIFIED_NAME_ATTR = "qualifiedName"; - public static final char[] invalidNameChars = {'@', '.'}; + public static final String NAME = "name"; + public static final String QUALIFIED_NAME_ATTR = "qualifiedName"; + public static final char[] invalidNameChars = {'@', '.'}; // Relation name constants - protected static final String ATLAS_GLOSSARY_PREFIX = ATLAS_GLOSSARY_TYPENAME; - protected static final String TERM_ANCHOR = ATLAS_GLOSSARY_PREFIX + "TermAnchor"; - protected static final String CATEGORY_ANCHOR = ATLAS_GLOSSARY_PREFIX + "CategoryAnchor"; - protected static final String CATEGORY_HIERARCHY = ATLAS_GLOSSARY_PREFIX + "CategoryHierarchyLink"; - protected static final String TERM_CATEGORIZATION = ATLAS_GLOSSARY_PREFIX + "TermCategorization"; - protected static final String TERM_ASSIGNMENT = ATLAS_GLOSSARY_PREFIX + "SemanticAssignment"; - protected static final String TERM_RELATION_ATTR_EXPRESSION = "expression"; + protected static final String ATLAS_GLOSSARY_PREFIX = ATLAS_GLOSSARY_TYPENAME; + protected static final String TERM_ANCHOR = ATLAS_GLOSSARY_PREFIX + "TermAnchor"; + protected static final String CATEGORY_ANCHOR = ATLAS_GLOSSARY_PREFIX + "CategoryAnchor"; + protected static final String CATEGORY_HIERARCHY = ATLAS_GLOSSARY_PREFIX + "CategoryHierarchyLink"; + protected static final String TERM_CATEGORIZATION = ATLAS_GLOSSARY_PREFIX + "TermCategorization"; + protected static final String TERM_ASSIGNMENT = ATLAS_GLOSSARY_PREFIX + "SemanticAssignment"; + protected static final String TERM_RELATION_ATTR_EXPRESSION = "expression"; protected static final String TERM_RELATION_ATTR_DESCRIPTION = "description"; - protected static final String TERM_RELATION_ATTR_STEWARD = "steward"; - protected static final String TERM_RELATION_ATTR_SOURCE = "source"; - protected static final String TERM_RELATION_ATTR_STATUS = "status"; + protected static final String TERM_RELATION_ATTR_STEWARD = "steward"; + protected static final String TERM_RELATION_ATTR_SOURCE = "source"; + protected static final String TERM_RELATION_ATTR_STATUS = "status"; + + + protected static final String TERM_DISPLAY_TEXT_KEY = ATLAS_GLOSSARY_TERM_TYPENAME + ".name"; + protected static final String CATEGORY_DISPLAY_TEXT_KEY = ATLAS_GLOSSARY_CATEGORY_TYPENAME + ".name"; + protected static final String CATEGORY_PARENT_EDGE_LABEL = "r:AtlasGlossaryCategoryHierarchyLink"; + protected static final String CATEGORY_TERMS_EDGE_LABEL = "r:AtlasGlossaryTermCategorization"; + protected static final String GLOSSARY_TERMS_EDGE_LABEL = "r:AtlasGlossaryTermAnchor"; + protected static final String GLOSSARY_CATEGORY_EDGE_LABEL = "r:AtlasGlossaryCategoryAnchor"; + protected static final String TERM_QUALIFIED_NAME_KEY = ATLAS_GLOSSARY_TERM_TYPENAME + ".qualifiedName"; + protected static final String CATEGORY_QUALIFIED_NAME_KEY = ATLAS_GLOSSARY_CATEGORY_TYPENAME + ".qualifiedName"; + protected final AtlasRelationshipStore relationshipStore; - protected final AtlasTypeRegistry typeRegistry; - protected final DataAccess dataAccess; + protected final AtlasTypeRegistry typeRegistry; + protected final DataAccess dataAccess; protected GlossaryUtils(final AtlasRelationshipStore relationshipStore, final AtlasTypeRegistry typeRegistry, final DataAccess dataAccess) { this.relationshipStore = relationshipStore; diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java index d879e57f6dd..15469c79d6f 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java @@ -19,15 +19,7 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.SearchFilter; -import org.apache.atlas.model.typedef.AtlasBaseTypeDef; -import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef; -import org.apache.atlas.model.typedef.AtlasClassificationDef; -import org.apache.atlas.model.typedef.AtlasEntityDef; -import org.apache.atlas.model.typedef.AtlasEnumDef; -import org.apache.atlas.model.typedef.AtlasRelationshipDef; -import org.apache.atlas.model.typedef.AtlasStructDef; -import org.apache.atlas.model.typedef.AtlasTypeDefHeader; -import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.model.typedef.*; import org.apache.atlas.repository.util.FilterUtil; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeUtil; @@ -40,14 +32,7 @@ import javax.inject.Inject; import javax.inject.Singleton; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.util.List;