From 1f6ae8952e611ecc1fc64685299b6954d0637cc0 Mon Sep 17 00:00:00 2001 From: Vinayak Marraiya Date: Tue, 2 Jun 2026 16:51:04 +0530 Subject: [PATCH 1/4] ATLAS-5220 : Add Maven profiles to build slim server by selecting only required storage/index backend --- audit/audit-cassandra/pom.xml | 110 +++++ .../audit/CassandraBasedAuditRepository.java | 0 .../audit/CassandraAuditRepositoryTest.java | 0 audit/audit-hbase/pom.xml | 47 ++ .../audit/HBaseBasedAuditRepository.java | 0 audit/audit-rdbms/pom.xml | 62 +++ .../rdbms/RdbmsBasedAuditRepository.java | 0 build-tools/pom.xml | 38 +- dashboard/pom.xml | 4 + distro/pom.xml | 103 +++- distro/src/bin/atlas_config.py | 9 +- distro/src/conf/atlas-application.properties | 18 +- .../main/assemblies/atlas-server-package.xml | 9 + .../main/assemblies/standalone-package.xml | 7 + distro/src/main/hbase-conf/hbase-site.xml | 31 ++ graphdb/janus-elasticsearch-shim/pom.xml | 52 +++ .../diskstorage/es/ElasticSearch7Index.java | 0 graphdb/janus/pom.xml | 274 ++++++----- .../graphdb/janus/AtlasJanusGraph.java | 16 +- .../janus/AtlasJanusGraphDatabase.java | 441 +++++++++++++++--- .../janus/AtlasJanusGraphIndexClient.java | 28 +- .../janus/AtlasBuildBackendAlignmentTest.java | 48 ++ graphdb/janusgraph-rdbms/pom.xml | 3 +- .../AtlasJanusRdbmsUniqueKeyHandler.java | 0 graphdb/pom.xml | 3 +- hbase-shaded-client-fixed/pom.xml | 23 + pom.xml | 187 +++++++- repository/pom.xml | 90 +--- .../AbstractStorageBasedAuditRepository.java | 4 +- .../util/AtlasRepositoryConfiguration.java | 5 +- .../v2/ClassificationAssociatorTest.java | 7 +- tools/atlas-index-repair/pom.xml | 20 + webapp/pom.xml | 202 +++++++- .../service/AtlasDebugMetricsSinkTest.java | 6 +- 34 files changed, 1497 insertions(+), 350 deletions(-) create mode 100644 audit/audit-cassandra/pom.xml rename {repository => audit/audit-cassandra}/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java (100%) rename {repository => audit/audit-cassandra}/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java (100%) create mode 100644 audit/audit-hbase/pom.xml rename {repository => audit/audit-hbase}/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java (100%) create mode 100644 audit/audit-rdbms/pom.xml rename {repository => audit/audit-rdbms}/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java (100%) create mode 100644 distro/src/main/hbase-conf/hbase-site.xml create mode 100644 graphdb/janus-elasticsearch-shim/pom.xml rename graphdb/{janus => janus-elasticsearch-shim}/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java (100%) create mode 100644 graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasBuildBackendAlignmentTest.java rename graphdb/{janus => janusgraph-rdbms}/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java (100%) diff --git a/audit/audit-cassandra/pom.xml b/audit/audit-cassandra/pom.xml new file mode 100644 index 00000000000..5591a3e3468 --- /dev/null +++ b/audit/audit-cassandra/pom.xml @@ -0,0 +1,110 @@ + + + + 4.0.0 + + + org.apache.atlas + apache-atlas + 3.0.0-SNAPSHOT + ../../pom.xml + + + atlas-audit-cassandra + jar + + Apache Atlas Audit (Cassandra) + ATLAS-5220: Cassandra entity audit repository; include when atlas.audit.backend=cassandra. + + + + + com.datastax.cassandra + cassandra-driver-core + 3.11.5 + + + ch.qos.logback + * + + + io.dropwizard.metrics + metrics-core + + + io.netty + netty-handler + + + + + org.apache.atlas + atlas-repository + ${project.version} + + + org.apache.cassandra + cassandra-all + 3.11.19 + + + ch.qos.logback + * + + + io.dropwizard.metrics + metrics-core + + + io.netty + netty-handler + + + org.jctools + jctools-core + + + + + + org.apache.atlas + atlas-repository + ${project.version} + test-jar + test + + + org.cassandraunit + cassandra-unit + 2.0.2.2 + test + + + ch.qos.logback + * + + + + + org.testng + testng + test + + + diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java b/audit/audit-cassandra/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java similarity index 100% rename from repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java rename to audit/audit-cassandra/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java b/audit/audit-cassandra/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java similarity index 100% rename from repository/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java rename to audit/audit-cassandra/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java diff --git a/audit/audit-hbase/pom.xml b/audit/audit-hbase/pom.xml new file mode 100644 index 00000000000..8f5a2a2c66d --- /dev/null +++ b/audit/audit-hbase/pom.xml @@ -0,0 +1,47 @@ + + + + 4.0.0 + + + org.apache.atlas + apache-atlas + 3.0.0-SNAPSHOT + ../../pom.xml + + + atlas-audit-hbase + jar + + Apache Atlas Audit (HBase) + ATLAS-5220: HBase entity audit repository; include on classpath when atlas.audit.backend=hbase. + + + + org.apache.atlas + atlas-repository + ${project.version} + + + org.apache.atlas + hbase-shaded-client-fixed + ${project.version} + + + diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java b/audit/audit-hbase/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java similarity index 100% rename from repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java rename to audit/audit-hbase/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java diff --git a/audit/audit-rdbms/pom.xml b/audit/audit-rdbms/pom.xml new file mode 100644 index 00000000000..c86056d5d8d --- /dev/null +++ b/audit/audit-rdbms/pom.xml @@ -0,0 +1,62 @@ + + + + 4.0.0 + + + org.apache.atlas + apache-atlas + 3.0.0-SNAPSHOT + ../../pom.xml + + + atlas-audit-rdbms + jar + + Apache Atlas Audit (RDBMS) + ATLAS-5220: JPA/RDBMS entity audit repository; include when atlas.audit.backend=rdbms. + + + + org.apache.atlas + atlas-repository + ${project.version} + + + org.apache.atlas + janusgraph-rdbms + ${project.version} + + + org.eclipse.persistence + eclipselink + ${eclipse.jpa.version} + + + org.eclipse.persistence + javax.persistence + ${javax.persistence.version} + + + org.postgresql + postgresql + ${postgresql.version} + + + diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java b/audit/audit-rdbms/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java similarity index 100% rename from repository/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java rename to audit/audit-rdbms/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java diff --git a/build-tools/pom.xml b/build-tools/pom.xml index 6f5dc4d0de0..283601ea965 100644 --- a/build-tools/pom.xml +++ b/build-tools/pom.xml @@ -17,27 +17,27 @@ ~ limitations under the License. --> - 4.0.0 + 4.0.0 - org.apache.atlas - atlas-buildtools - 3.0.0-SNAPSHOT - jar + org.apache.atlas + atlas-buildtools + 3.0.0-SNAPSHOT + jar - Apache Atlas Server Build Tools - Apache Atlas Build Tools like Checkstyle + Apache Atlas Server Build Tools + Apache Atlas Build Tools like Checkstyle - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - - + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + diff --git a/dashboard/pom.xml b/dashboard/pom.xml index 17bcf14f1e5..4d50c985adc 100644 --- a/dashboard/pom.xml +++ b/dashboard/pom.xml @@ -72,6 +72,10 @@ ${project.basedir}/ + + **/node_modules/** + **/.vscode/** + false diff --git a/distro/pom.xml b/distro/pom.xml index d7e95888949..cde50ff75e7 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -35,7 +35,7 @@ false false - atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository + solr #Solr #Solr cloud mode properties @@ -60,11 +60,11 @@ atlas.graph.index.search.solr.wait-searcher=false # plugins: https://docs.janusgraph.org/latest/elasticsearch.html #atlas.graph.index.search.hostname=localhost #atlas.graph.index.search.elasticsearch.client-only=true - hbase2 + #Hbase #For standalone mode , specify localhost -#for distributed mode, specify zookeeper quorum here -atlas.graph.storage.hostname= +#for distributed mode, specify zookeeper quorum here (comma-separated); empty values break Janus STRING options. +atlas.graph.storage.hostname=localhost atlas.graph.storage.hbase.regions-per-server=1 #In order to use Cassandra as a backend, comment out the hbase specific properties above, and uncomment the @@ -588,6 +588,101 @@ atlas.graph.storage.hbase.regions-per-server=1 + + + storage-berkeleyje + + false + + + # BerkeleyJE — local graph under ATLAS_HOME; edit paths if needed +atlas.graph.storage.directory=${sys:atlas.home}/data/berkeley +atlas.graph.storage.lock.clean-expired=true +atlas.graph.storage.lock.expiry-time=500 +atlas.graph.storage.lock.wait-time=300 + + + + storage-rdbms + + false + + + # RDBMS — PostgreSQL example; edit host, db, user, password before starting Atlas +atlas.graph.storage.rdbms.jpa.hikari.connectionTestQuery=select 1 +atlas.graph.storage.rdbms.jpa.hikari.connectionTimeout=30000 +atlas.graph.storage.rdbms.jpa.hikari.driverClassName=org.postgresql.Driver +atlas.graph.storage.rdbms.jpa.hikari.idleTimeout=300000 +atlas.graph.storage.rdbms.jpa.hikari.jdbcUrl=jdbc:postgresql://localhost:5432/atlas +atlas.graph.storage.rdbms.jpa.hikari.maxLifetime=1800000 +atlas.graph.storage.rdbms.jpa.hikari.maximumPoolSize=40 +atlas.graph.storage.rdbms.jpa.hikari.minimumIdle=5 +atlas.graph.storage.rdbms.jpa.hikari.password=atlas +atlas.graph.storage.rdbms.jpa.hikari.username=atlas +atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.dialect=org.eclipse.persistence.platform.database.PostgreSQLPlatform +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-database-schemas=true +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-script-source=META-INF/postgres/create_schema.sql +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-source=script +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.database.action=create + + + + index-elasticsearch + + false + + + elasticsearch + # Elasticsearch — set hostname to your cluster or load balancer +atlas.graph.index.search.elasticsearch.client-only=true +atlas.graph.index.search.hostname=localhost + + + + storage-cassandra + + false + + + # Janus CQL — point at your Cassandra nodes (comma-separated). Keyspace is created/used by Janus as configured. +atlas.graph.storage.cql.keyspace=janusgraph +atlas.graph.storage.hostname=127.0.0.1 + + + + audit-cassandra + + false + + + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.CassandraBasedAuditRepository + + + + index-solr + + false + + + # Solr Cloud (set zookeeper-url to your Solr/ZK ensemble) +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=localhost:2181 +atlas.graph.index.search.solr.zookeeper-connect-timeout=60000 +atlas.graph.index.search.solr.zookeeper-session-timeout=60000 +atlas.graph.index.search.solr.wait-searcher=false + +# Solr http mode (alternative) +#atlas.graph.index.search.solr.mode=http +#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr + + + embedded-cassandra-solr diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py index 70030ac5cce..7e78593cf70 100755 --- a/distro/src/bin/atlas_config.py +++ b/distro/src/bin/atlas_config.py @@ -104,7 +104,14 @@ def hbaseBinDir(dir): return os.path.join(dir, "hbase", BIN) def hbaseConfDir(dir): - return os.environ.get(HBASE_CONF_DIR, os.path.join(dir, "hbase", CONF)) + """Resolve HBase client conf dir. Relative HBASE_CONF_DIR is resolved against Atlas home so + atlas_start.py works when run from bin/ (cwd is not the install root).""" + conf = os.environ.get(HBASE_CONF_DIR) + if conf: + if not os.path.isabs(conf): + conf = os.path.normpath(os.path.join(dir, conf)) + return conf + return os.path.join(dir, "hbase", CONF) def zookeeperBinDir(dir): return os.environ.get(SOLR_BIN, os.path.join(dir, "zk", BIN)) diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties index b5734d7a8de..b0ef83f2643 100755 --- a/distro/src/conf/atlas-application.properties +++ b/distro/src/conf/atlas-application.properties @@ -18,21 +18,27 @@ ######### Graph Database Configs ######### +# ATLAS-5220: Maven build-time backend selection (set via -Pstorage-* / -Pindex-* / -Pdist). Do not point runtime +# config at a different storage/index backend unless you rebuild with matching profiles. +atlas.build.storage.backend=${atlas.storage.backend} +atlas.build.index.backend=${atlas.index.backend} + # Graph Database #Configures the graph database to use. Defaults to JanusGraph #atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase # Graph Storage -# Set atlas.graph.storage.backend to the correct value for your desired storage -# backend. Possible values: +# Set atlas.graph.storage.backend to the JanusGraph storage short name (see Janus docs). +# Common values: # -# hbase -# cassandra -# embeddedcassandra - Should only be set by building Atlas with -Pdist,embedded-cassandra-solr +# hbase2 (HBase 2.x; Atlas builds often set this via Maven graph.storage.backend) +# cql (Cassandra via Janus CQL backend — use this, not the word "cassandra", or Janus will look for a Java class) +# embeddedcassandra - Only when building with -Pdist,embedded-cassandra-solr (Atlas embedded stack) # berkeleyje +# rdbms # -# See the configuration documentation for more information about configuring the various storage backends. +# See the configuration documentation for more information about configuring the various storage backends. # atlas.graph.storage.backend=${graph.storage.backend} atlas.graph.storage.hbase.table=apache_atlas_janus diff --git a/distro/src/main/assemblies/atlas-server-package.xml b/distro/src/main/assemblies/atlas-server-package.xml index ba2e2665207..4bf1170ad84 100755 --- a/distro/src/main/assemblies/atlas-server-package.xml +++ b/distro/src/main/assemblies/atlas-server-package.xml @@ -52,6 +52,15 @@ 0755 + + + src/main/hbase-conf + hbase/conf + 0644 + 0755 + + target/hbase hbase diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml index 56b338232eb..a747060cf18 100755 --- a/distro/src/main/assemblies/standalone-package.xml +++ b/distro/src/main/assemblies/standalone-package.xml @@ -64,6 +64,13 @@ 0755 + + src/main/hbase-conf + hbase/conf + 0644 + 0755 + + target/hbase hbase diff --git a/distro/src/main/hbase-conf/hbase-site.xml b/distro/src/main/hbase-conf/hbase-site.xml new file mode 100644 index 00000000000..efebdc6a9f4 --- /dev/null +++ b/distro/src/main/hbase-conf/hbase-site.xml @@ -0,0 +1,31 @@ + + + + + + hbase.zookeeper.quorum + localhost + + + hbase.zookeeper.property.clientPort + 2181 + + diff --git a/graphdb/janus-elasticsearch-shim/pom.xml b/graphdb/janus-elasticsearch-shim/pom.xml new file mode 100644 index 00000000000..30080cebabd --- /dev/null +++ b/graphdb/janus-elasticsearch-shim/pom.xml @@ -0,0 +1,52 @@ + + + + 4.0.0 + + + org.apache.atlas + atlas-graphdb + 3.0.0-SNAPSHOT + + + atlas-graphdb-janus-es-shim + jar + + Apache Atlas JanusGraph Elasticsearch Shim + ATLAS-5220: optional Janus ES index accessor; add to WAR/tools only when atlas.index.backend=elasticsearch. + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + org.janusgraph + janusgraph-es + ${janusgraph.version} + + + ch.qos.logback + * + + + + + diff --git a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java b/graphdb/janus-elasticsearch-shim/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java similarity index 100% rename from graphdb/janus/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java rename to graphdb/janus-elasticsearch-shim/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index cd07c909800..47712eff58b 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -35,6 +35,8 @@ Some dependencies, like slf4j are excluded from the jar because they are included in Atlas --> + + hbase true false @@ -51,23 +53,10 @@ atlas-graphdb-common ${project.version} - - org.apache.atlas - hbase-shaded-client-fixed - ${project.version} - - - org.apache.atlas - janusgraph-rdbms - ${project.version} - - - ch.qos.logback - * - - - + org.apache.commons commons-configuration2 @@ -84,22 +73,7 @@ commons-text ${commons-text.version} - - org.apache.hadoop - hadoop-distcp - ${hadoop.version} - - - org.apache.hbase - hbase-shaded-mapreduce - ${hbase.version}-hadoop3 - - - org.apache.hadoop - hadoop-distcp - - - + org.apache.lucene lucene-analyzers-common @@ -167,89 +141,6 @@ tinkergraph-gremlin ${tinkerpop.version} - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${elasticsearch.version} - - - - org.janusgraph - janusgraph-berkeleyje - ${janusgraph.version} - - - ch.qos.logback - * - - - org.noggit - noggit - - - - - - org.janusgraph - janusgraph-cassandra - ${janusgraph.cassandra.version} - - - ch.qos.logback - * - - - com.codahale.metrics - metrics-core - - - - - - org.janusgraph - janusgraph-cql - ${janusgraph.version} - - - ch.qos.logback - * - - - - - - org.janusgraph - janusgraph-es - ${janusgraph.version} - - - ch.qos.logback - * - - - - - - org.janusgraph - janusgraph-hbase - ${janusgraph.version} - - - ch.qos.logback - * - - - org.apache.hbase - hbase-shaded-client - - - org.apache.hbase - hbase-shaded-mapreduce - - - - org.janusgraph janusgraph-lucene @@ -267,6 +158,7 @@ janusgraph-server ${janusgraph.version} + com.codahale.metrics * @@ -287,6 +179,26 @@ org.apache.tinkerpop gremlin-shaded + + org.janusgraph + janusgraph-berkeleyje + + + org.janusgraph + janusgraph-cassandra + + + org.janusgraph + janusgraph-cql + + + org.janusgraph + janusgraph-es + + + org.janusgraph + janusgraph-hbase + org.noggit noggit @@ -334,6 +246,13 @@ test + + org.apache.atlas + atlas-graphdb-janus-es-shim + ${project.version} + test + + org.apache.atlas atlas-testtools @@ -384,4 +303,129 @@ + + + + atlas-storage-hbase + + + atlas.storage.backend + hbase + + + + + org.apache.atlas + hbase-shaded-client-fixed + ${project.version} + + + org.apache.hadoop + hadoop-distcp + ${hadoop.version} + + + org.apache.hbase + hbase-shaded-mapreduce + ${hbase.version}-hadoop3 + + + org.apache.hadoop + hadoop-distcp + + + + + org.janusgraph + janusgraph-hbase + ${janusgraph.version} + + + ch.qos.logback + * + + + org.apache.hbase + hbase-shaded-client + + + org.apache.hbase + hbase-shaded-mapreduce + + + + + + + atlas-storage-cassandra + + + atlas.storage.backend + cassandra + + + + + org.janusgraph + janusgraph-cql + ${janusgraph.version} + + + ch.qos.logback + * + + + + + + + atlas-storage-berkeleyje + + + atlas.storage.backend + berkeleyje + + + + + org.janusgraph + janusgraph-berkeleyje + ${janusgraph.version} + + + ch.qos.logback + * + + + org.noggit + noggit + + + + + + + atlas-storage-rdbms + + + atlas.storage.backend + rdbms + + + + + org.apache.atlas + janusgraph-rdbms + ${project.version} + + + ch.qos.logback + * + + + + + + + diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java index e0d4857a1ca..2ec2132854c 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java @@ -137,7 +137,7 @@ public AtlasJanusGraph(JanusGraph graphInstance) { } if (StringUtils.equalsIgnoreCase(AtlasConfiguration.STORAGE_BACKEND_TYPE.getString(), "rdbms")) { - uniqueKeyHandler = new AtlasJanusRdbmsUniqueKeyHandler(); + uniqueKeyHandler = newRdbmsUniqueKeyHandlerOrNull(); } else { uniqueKeyHandler = null; } @@ -150,6 +150,20 @@ public AtlasJanusGraph(JanusGraph graphInstance) { janusGraph = (StandardJanusGraph) graphInstance; } + private static AtlasUniqueKeyHandler newRdbmsUniqueKeyHandlerOrNull() { + final String handlerClass = "org.apache.atlas.repository.graphdb.janus.AtlasJanusRdbmsUniqueKeyHandler"; + + try { + Class clz = Class.forName(handlerClass); + + return (AtlasUniqueKeyHandler) clz.getDeclaredConstructor().newInstance(); + } catch (Throwable t) { + LOG.warn("RDBMS storage is configured but {} is not available on the classpath", handlerClass, t); + + return null; + } + } + @Override public AtlasEdge addEdge(AtlasVertex outVertex, AtlasVertex inVertex, String edgeLabel) { try { diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java index 8648fb6c137..df297946c1d 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java @@ -31,6 +31,7 @@ import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationConverter; +import org.apache.commons.lang3.StringUtils; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.JanusGraphException; @@ -38,10 +39,6 @@ import org.janusgraph.core.schema.JanusGraphManagement; import org.janusgraph.diskstorage.StandardIndexProvider; import org.janusgraph.diskstorage.StandardStoreManager; -import org.janusgraph.diskstorage.es.ElasticSearch7Index; -import org.janusgraph.diskstorage.hbase.HBaseStoreManager; -import org.janusgraph.diskstorage.rdbms.RdbmsStoreManager; -import org.janusgraph.diskstorage.solr.Solr6Index; import org.janusgraph.graphdb.database.serialize.attribute.SerializableSerializer; import org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry; import org.slf4j.Logger; @@ -54,7 +51,10 @@ import java.math.BigInteger; import java.time.Duration; import java.util.ArrayList; +import java.util.Enumeration; import java.util.HashMap; +import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Properties; @@ -71,17 +71,30 @@ public class AtlasJanusGraphDatabase implements GraphDatabase FQCN in Janus {@code StandardStoreManager} without loading the class first. + * Pre-loading with {@link Class#forName(String)} caused missing map entries when the class was not yet + * visible to the caller's loader at static init time, leading Janus to treat {@code hbase2} as a class name. + * Implementation classes are loaded when the graph opens, as in the pre-ATLAS-5220 behavior. + */ + private static void safeRegisterStoreManager(String shortName, String managerClassName) { try { - Field field = StandardStoreManager.class.getDeclaredField("ALL_MANAGER_CLASSES"); + injectStoreManager(shortName, managerClassName); + } catch (Throwable t) { + LOG.warn("Failed to register Janus storage backend '{}' -> {}", shortName, managerClassName, t); + } + } - field.setAccessible(true); + private static void injectStoreManager(String shortName, String managerClassName) throws Exception { + Field field = StandardStoreManager.class.getDeclaredField("ALL_MANAGER_CLASSES"); - Field modifiersField = Field.class.getDeclaredField("modifiers"); + field.setAccessible(true); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + Field modifiersField = Field.class.getDeclaredField("modifiers"); - Map customMap = new HashMap<>(StandardStoreManager.getAllManagerClasses()); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - customMap.put("hbase2", HBaseStoreManager.class.getName()); + Map customMap = new HashMap<>(StandardStoreManager.getAllManagerClasses()); - ImmutableMap immap = ImmutableMap.copyOf(customMap); + customMap.put(shortName, managerClassName); - field.set(null, immap); + field.set(null, ImmutableMap.copyOf(customMap)); - LOG.debug("Injected HBase2 support - {}", HBaseStoreManager.class.getName()); - } catch (Exception e) { - throw new RuntimeException(e); - } + LOG.debug("Registered Janus storage backend {} -> {}", shortName, managerClassName); } - private static void addRdbmsSupport() { + private static void safeRegisterIndexProvider(String shortName, String providerClassName) { try { - Field field = StandardStoreManager.class.getDeclaredField("ALL_MANAGER_CLASSES"); + injectIndexProvider(shortName, providerClassName); + } catch (Throwable t) { + LOG.warn("Failed to register Janus index provider '{}' -> {}", shortName, providerClassName, t); + } + } - field.setAccessible(true); + private static void injectIndexProvider(String shortName, String providerClassName) throws Exception { + Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES"); - Field modifiersField = Field.class.getDeclaredField("modifiers"); + field.setAccessible(true); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + Field modifiersField = Field.class.getDeclaredField("modifiers"); - Map customMap = new HashMap<>(StandardStoreManager.getAllManagerClasses()); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - customMap.put("rdbms", RdbmsStoreManager.class.getName()); + Map customMap = new HashMap<>(StandardIndexProvider.getAllProviderClasses()); - ImmutableMap immap = ImmutableMap.copyOf(customMap); + customMap.put(shortName, providerClassName); - field.set(null, immap); + field.set(null, ImmutableMap.copyOf(customMap)); - LOG.debug("Injected RDBMS support - {}", RdbmsStoreManager.class.getName()); - } catch (Exception e) { - throw new RuntimeException(e); - } + LOG.debug("Registered Janus index provider {} -> {}", shortName, providerClassName); } - private static void addSolr6Index() { - try { - Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES"); + /** + * Register Janus shortName → implementation FQCN only when the class is loadable (ATLAS-5220 slim builds may omit JARs). + */ + private static void registerJanusOptionalBackends() { + registerStoreIfLoadable("hbase2", HBASE2_STORE_MANAGER_CLASS); + registerStoreIfLoadable("cql", CQL_STORE_MANAGER_CLASS); + registerStoreIfLoadable("rdbms", RDBMS_STORE_MANAGER_CLASS); + registerIndexIfLoadable("solr", SOLR6_INDEX_CLASS); + registerIndexIfLoadable("elasticsearch", ELASTICSEARCH7_INDEX_CLASS); + } - field.setAccessible(true); + private static void registerStoreIfLoadable(String shortName, String managerClassName) { + if (!isClassLoadable(managerClassName)) { + LOG.debug("Skipping Janus storage backend {}: {} not on classpath", shortName, managerClassName); - Field modifiersField = Field.class.getDeclaredField("modifiers"); + return; + } - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + safeRegisterStoreManager(shortName, managerClassName); + } - Map customMap = new HashMap<>(StandardIndexProvider.getAllProviderClasses()); + private static void registerIndexIfLoadable(String shortName, String providerClassName) { + if (!isClassLoadable(providerClassName)) { + LOG.debug("Skipping Janus index provider {}: {} not on classpath", shortName, providerClassName); - customMap.put("solr", Solr6Index.class.getName()); + return; + } - ImmutableMap immap = ImmutableMap.copyOf(customMap); + safeRegisterIndexProvider(shortName, providerClassName); + } - field.set(null, immap); + private static boolean isClassLoadable(String className) { + ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); + ClassLoader atlasLoader = AtlasJanusGraphDatabase.class.getClassLoader(); - LOG.debug("Injected solr6 index - {}", Solr6Index.class.getName()); - } catch (Exception e) { - throw new RuntimeException(e); + if (contextLoader != null && isClassLoadable(className, contextLoader)) { + return true; } - } - private static void addElasticSearch7Index() { - try { - Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES"); - - field.setAccessible(true); + if (atlasLoader != null && isClassLoadable(className, atlasLoader)) { + return true; + } - Field modifiersField = Field.class.getDeclaredField("modifiers"); + return isClassLoadable(className, ClassLoader.getSystemClassLoader()); + } - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + private static boolean isClassLoadable(String className, ClassLoader loader) { + if (loader == null) { + return false; + } - Map customMap = new HashMap<>(StandardIndexProvider.getAllProviderClasses()); + try { + Class.forName(className, false, loader); - customMap.put("elasticsearch", ElasticSearch7Index.class.getName()); + return true; + } catch (ClassNotFoundException e) { + return false; + } catch (LinkageError e) { + LOG.debug("Class {} not loadable with loader {}: {}", className, loader, e.toString()); - ImmutableMap immap = ImmutableMap.copyOf(customMap); + return false; + } + } - field.set(null, immap); + private static boolean isClasspathResourcePresent(String resourcePath) { + ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); + ClassLoader atlasLoader = AtlasJanusGraphDatabase.class.getClassLoader(); - LOG.debug("Injected es7 index - {}", ElasticSearch7Index.class.getName()); - } catch (Exception e) { - throw new RuntimeException(e); - } + return (contextLoader != null && contextLoader.getResource(resourcePath) != null) + || (atlasLoader != null && atlasLoader.getResource(resourcePath) != null); } private static void updateGlobalConfiguration(Map map) { @@ -438,10 +594,160 @@ private static void updateGlobalConfiguration(Map map) { } } + /** + * Spring/YAML list properties and {@code setProperty(key, String[])} (e.g. Solr ZK URLs) become {@code String[]}. + * JanusGraph {@code ConfigOption} values typed as STRING reject arrays and fail with + * {@code Invalid configuration value for [root.storage.hostname]: [Ljava.lang.String;@...}. + */ + private static void normalizeStringArrayValuesForJanus(Configuration janusConfig) { + ArrayList keys = new ArrayList<>(); + + for (Iterator it = janusConfig.getKeys(); it.hasNext(); ) { + keys.add(it.next()); + } + + for (String key : keys) { + Object val = janusConfig.getProperty(key); + + if (val instanceof String[]) { + String[] arr = (String[]) val; + + if (arr.length == 0) { + janusConfig.clearProperty(key); + } else { + janusConfig.setProperty(key, String.join(",", arr)); + } + } else if (val instanceof List) { + List list = (List) val; + + if (list.isEmpty()) { + janusConfig.clearProperty(key); + } else { + janusConfig.setProperty(key, joinCommaList(list)); + } + } + } + } + + /** + * {@link org.apache.commons.configuration.Configuration#getKeys()} on subset configs can omit keys that are only + * present as multi-valued / list types, so {@link #normalizeStringArrayValuesForJanus} may skip them. + * Janus {@code storage.hostname} is a STRING for HBase and CQL ({@link org.janusgraph.diskstorage.common.DistributedStoreManager}). + */ + private static void coerceCriticalJanusStringOptions(Configuration janusConfig) { + coerceSingleJanusString(janusConfig, "storage.hostname"); + } + + private static void coerceSingleJanusString(Configuration cfg, String key) { + Object val = cfg.getProperty(key); + + if (val == null || val instanceof String) { + return; + } + + if (val instanceof String[]) { + String[] arr = (String[]) val; + + cfg.setProperty(key, arr.length == 0 ? "" : String.join(",", arr)); + } else if (val instanceof List) { + cfg.setProperty(key, joinCommaList((List) val)); + } else { + cfg.setProperty(key, val.toString()); + } + } + + private static String joinCommaList(List list) { + StringBuilder sb = new StringBuilder(); + + for (Object o : list) { + if (sb.length() > 0) { + sb.append(','); + } + + sb.append(o != null ? o.toString() : ""); + } + + return sb.toString(); + } + + /** + * {@link ConfigurationConverter#getProperties} can leave {@code String[]} / {@link List} values inside + * {@link Properties}; {@link org.apache.commons.configuration2.ConfigurationConverter#getConfiguration(Properties)} + * then passes those through to JanusGraph unchanged. + */ + private static void sanitizePropertiesForJanus(Properties properties) { + synchronized (properties) { + for (Enumeration e = properties.keys(); e.hasMoreElements(); ) { + String key = String.valueOf(e.nextElement()); + Object val = properties.get(key); + + if (val instanceof String[]) { + String[] arr = (String[]) val; + + properties.put(key, arr.length == 0 ? "" : String.join(",", arr)); + } else if (val instanceof List) { + properties.put(key, joinCommaList((List) val)); + } + } + + // Hashtable-backed Properties may still omit non-String values from keys(); force known Janus STRING keys. + Object hostnameVal = properties.get("storage.hostname"); + + if (hostnameVal instanceof String[]) { + String[] arr = (String[]) hostnameVal; + + properties.put("storage.hostname", arr.length == 0 ? "" : String.join(",", arr)); + } else if (hostnameVal instanceof List) { + properties.put("storage.hostname", joinCommaList((List) hostnameVal)); + } + } + } + private static org.apache.commons.configuration2.Configuration createConfiguration2(Configuration conf) { + normalizeStringArrayValuesForJanus(conf); + coerceCriticalJanusStringOptions(conf); + Properties properties = ConfigurationConverter.getProperties(conf); - return org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties); + sanitizePropertiesForJanus(properties); + + org.apache.commons.configuration2.Configuration cfg2 = + org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties); + + // getConfiguration(Properties) can still surface String[] for keys Janus expects as STRING (e.g. storage.hostname). + coerceStringArraysInConfiguration2(cfg2); + + return cfg2; + } + + /** + * Final pass: Janus {@code storage.hostname} (and similar STRING options) must not be {@code String[]} after + * {@link org.apache.commons.configuration2.ConfigurationConverter#getConfiguration(Properties)}. + */ + private static void coerceStringArraysInConfiguration2(org.apache.commons.configuration2.Configuration cfg) { + if (cfg == null) { + return; + } + + ArrayList keys = new ArrayList<>(); + + for (Iterator it = cfg.getKeys(); it.hasNext(); ) { + keys.add(it.next()); + } + + for (String key : keys) { + Object val = cfg.getProperty(key); + + if (val instanceof String[]) { + String[] arr = (String[]) val; + + cfg.setProperty(key, arr.length == 0 ? "" : String.join(",", arr)); + } else if (val instanceof List) { + List list = (List) val; + + cfg.setProperty(key, list.isEmpty() ? "" : joinCommaList(list)); + } + } } private static void startEmbeddedSolr() throws AtlasException { @@ -479,11 +785,6 @@ private static void stopEmbeddedSolr() throws AtlasException { } static { - addHBase2Support(); - addRdbmsSupport(); - - addSolr6Index(); - - addElasticSearch7Index(); + registerJanusOptionalBackends(); } } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java index 0c73fa5b2af..ba45162428e 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java @@ -42,13 +42,12 @@ import org.apache.solr.client.solrj.response.TermsResponse; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.NamedList; -import org.janusgraph.diskstorage.es.ElasticSearch7Index; -import org.janusgraph.diskstorage.es.ElasticSearchClient; import org.janusgraph.diskstorage.solr.Solr6Index; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -476,11 +475,28 @@ private boolean isSolrHealthy() throws SolrServerException, IOException { return client != null && client.ping(Constants.VERTEX_INDEX).getStatus() == SOLR_HEALTHY_STATUS; } - private boolean isElasticsearchHealthy() throws IOException { - ElasticSearchClient client = ElasticSearch7Index.getElasticSearchClient(); - String janusVertexIndex = ApplicationProperties.DEFAULT_INDEX_NAME + "_" + Constants.VERTEX_INDEX; + private boolean isElasticsearchHealthy() { + try { + Class esIndexClass = Class.forName("org.janusgraph.diskstorage.es.ElasticSearch7Index"); + Method getClient = esIndexClass.getMethod("getElasticSearchClient"); + Object client = getClient.invoke(null); + + if (client == null) { + return false; + } - return client != null && client.indexExists(janusVertexIndex); + Method indexExists = client.getClass().getMethod("indexExists", String.class); + String janusVertexIndex = ApplicationProperties.DEFAULT_INDEX_NAME + "_" + Constants.VERTEX_INDEX; + Object exists = indexExists.invoke(client, janusVertexIndex); + + return Boolean.TRUE.equals(exists); + } catch (ClassNotFoundException e) { + LOG.debug("Elasticsearch index implementation not on classpath; skipping health check"); + return false; + } catch (ReflectiveOperationException e) { + LOG.debug("Elasticsearch health check failed: {}", e.getMessage()); + return false; + } } private SolrResponse updateFreeTextRequestHandler(SolrClient solrClient, String collectionName, Map indexFieldName2SearchWeightMap, Solr6Index.Mode mode) throws IOException, SolrServerException, AtlasBaseException { diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasBuildBackendAlignmentTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasBuildBackendAlignmentTest.java new file mode 100644 index 00000000000..3482e1c0d97 --- /dev/null +++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasBuildBackendAlignmentTest.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graphdb.janus; + +import org.testng.annotations.Test; + +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class AtlasBuildBackendAlignmentTest { + @Test + public void testStorageBackendCompatibility() { + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("hbase", "hbase2")); + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("hbase", "hbase")); + assertFalse(AtlasJanusGraphDatabase.isStorageBackendCompatible("hbase", "rdbms")); + + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("cassandra", "cql")); + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("cassandra", "embeddedcassandra")); + assertFalse(AtlasJanusGraphDatabase.isStorageBackendCompatible("cassandra", "hbase2")); + + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("rdbms", "rdbms")); + assertFalse(AtlasJanusGraphDatabase.isStorageBackendCompatible("rdbms", "cql")); + + assertTrue(AtlasJanusGraphDatabase.isStorageBackendCompatible("berkeleyje", "berkeleyje")); + } + + @Test + public void testIndexBackendCompatibility() { + assertTrue(AtlasJanusGraphDatabase.isIndexBackendCompatible("solr", "solr")); + assertFalse(AtlasJanusGraphDatabase.isIndexBackendCompatible("solr", "elasticsearch")); + assertTrue(AtlasJanusGraphDatabase.isIndexBackendCompatible("elasticsearch", "elasticsearch")); + } +} diff --git a/graphdb/janusgraph-rdbms/pom.xml b/graphdb/janusgraph-rdbms/pom.xml index 80301dac09a..a6f55d8cccc 100644 --- a/graphdb/janusgraph-rdbms/pom.xml +++ b/graphdb/janusgraph-rdbms/pom.xml @@ -40,7 +40,8 @@ org.apache.atlas - atlas-intg + atlas-graphdb-api + ${project.version} org.eclipse.persistence diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java b/graphdb/janusgraph-rdbms/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java similarity index 100% rename from graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java rename to graphdb/janusgraph-rdbms/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java diff --git a/graphdb/pom.xml b/graphdb/pom.xml index c4c07948ccd..5ad23e9cef3 100644 --- a/graphdb/pom.xml +++ b/graphdb/pom.xml @@ -31,12 +31,13 @@ Apache Atlas Graph Database Projects Apache Atlas Graph Database Projects - + api common graphdb-impls janus + janus-elasticsearch-shim janusgraph-rdbms diff --git a/hbase-shaded-client-fixed/pom.xml b/hbase-shaded-client-fixed/pom.xml index 09e75bae4aa..2d89578da33 100644 --- a/hbase-shaded-client-fixed/pom.xml +++ b/hbase-shaded-client-fixed/pom.xml @@ -73,6 +73,29 @@ + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-unpacked-hbase-to-classes + + copy-resources + + process-resources + + ${project.build.directory}/classes + + + ${project.build.directory}/unpacked + + + + + + + org.apache.maven.plugins diff --git a/pom.xml b/pom.xml index 08fbd2b8593..deb983e4609 100644 --- a/pom.xml +++ b/pom.xml @@ -54,6 +54,9 @@ addons/storm-bridge-shim addons/trino-extractor atlas-examples + audit/audit-cassandra + audit/audit-hbase + audit/audit-rdbms authorization build-tools client @@ -86,7 +89,13 @@ 4.7 1.0 1.8.7 + + hbase + solr + hbase + + 1.7.5 1.16.0 false @@ -112,12 +121,14 @@ 2.7.15 7.17.8 org.apache.atlas.repository.audit.InMemoryEntityAuditRepository + atlas.EntityAuditRepository.impl=${entity.repository.impl} 2.13.2 2.18.1 0.8 6.5.16 solr - berkeleyje + + hbase2 2.9.0 25.1-jre 4.1.0 @@ -2282,13 +2293,103 @@ + - embedded-cassandra-solr + dist + + false + - ${guava.version} + hbase + solr + hbase + + + org.apache.atlas.repository.audit.HBaseBasedAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository + solr + hbase2 + false + + + + + audit-cassandra + + + atlas.audit.backend + cassandra + + + + cassandra + WEB-INF/lib/atlas-audit-hbase-*.jar, + org.apache.atlas.repository.audit.CassandraBasedAuditRepository + + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.CassandraBasedAuditRepository + + + + audit-hbase + + + atlas.audit.backend + hbase + + + + hbase + + org.apache.atlas.repository.audit.HBaseBasedAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository + + + + audit-inmemory + + + atlas.audit.backend + inmemory + + + + inmemory + WEB-INF/lib/atlas-audit-hbase-*.jar, + org.apache.atlas.repository.audit.InMemoryEntityAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.InMemoryEntityAuditRepository + + + + audit-noop + + + atlas.audit.backend + noop + + + + noop + WEB-INF/lib/atlas-audit-hbase-*.jar, + org.apache.atlas.repository.audit.NoopEntityAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.NoopEntityAuditRepository + + + + audit-rdbms + + + atlas.audit.backend + rdbms + + + + rdbms + WEB-INF/lib/atlas-audit-hbase-*.jar, + org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository - distributed @@ -2296,24 +2397,23 @@ false + hbase + solr + hbase org.apache.atlas.repository.audit.HBaseBasedAuditRepository + atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository solr - hbase + hbase2 localhost localhost:9983 - - dist - - false - + embedded-cassandra-solr - false + ${guava.version} - - graph-provider-default @@ -2335,8 +2434,10 @@ + solr WEB-INF/lib/je-*.jar,WEB-INF/lib/solr-test-framework-*.jar, WEB-INF/lib/jts-*.jar,WEB-INF/lib/dom4j-*.jar,WEB-INF/lib/ant-*.jar solr + hbase2 atlas-graphdb-janus org.apache.atlas org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase @@ -2344,7 +2445,6 @@ true - graph-provider-janus @@ -2354,8 +2454,10 @@ + solr WEB-INF/lib/je-*.jar,WEB-INF/lib/solr-test-framework-*.jar, WEB-INF/lib/jts-*.jar,WEB-INF/lib/dom4j-*.jar,WEB-INF/lib/ant-*.jar solr + hbase2 atlas-graphdb-janus org.apache.atlas org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase @@ -2363,13 +2465,66 @@ true - + + + index-elasticsearch + + + atlas.index.backend + elasticsearch + + + + elasticsearch + + elasticsearch + + + + index-solr + + solr + + skipMinify build - + + + storage-berkeleyje + + berkeleyje + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + berkeleyje + + + + storage-cassandra + + + cassandra + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + cql + + + + storage-hbase + + hbase + + hbase2 + + + + storage-rdbms + + rdbms + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + rdbms + + diff --git a/repository/pom.xml b/repository/pom.xml index 000c5197ac2..1491dd3fbb5 100644 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -31,6 +31,10 @@ Apache Atlas Repository Apache Atlas Repository Module + + true false @@ -42,26 +46,6 @@ hppc ${hppc.version} - - com.datastax.cassandra - cassandra-driver-core - 3.11.5 - - - ch.qos.logback - * - - - io.dropwizard.metrics - metrics-core - - - io.netty - netty-handler - - - - com.opencsv opencsv @@ -123,41 +107,6 @@ org.apache.atlas atlas-server-api - - org.apache.atlas - hbase-shaded-client-fixed - ${project.version} - - - org.apache.atlas - janusgraph-rdbms - ${project.version} - - - - org.apache.cassandra - cassandra-all - 3.11.19 - - - ch.qos.logback - * - - - io.dropwizard.metrics - metrics-core - - - io.netty - netty-handler - - - org.jctools - jctools-core - - - - org.apache.commons commons-lang3 @@ -198,29 +147,11 @@ - - org.eclipse.persistence - eclipselink - ${eclipse.jpa.version} - - - - org.eclipse.persistence - javax.persistence - ${javax.persistence.version} - - org.mockito mockito-all - - org.postgresql - postgresql - ${postgresql.version} - - org.springframework spring-aop @@ -282,19 +213,6 @@ test-jar test - - org.cassandraunit - cassandra-unit - 2.0.2.2 - test - - - ch.qos.logback - * - - - - diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java index 06bc461e5e3..8aa51f9a834 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java @@ -28,12 +28,12 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.HashMap; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -158,7 +158,7 @@ protected String getKeyStr(String id, Long ts, int index) { } protected byte[] getKey(String id, Long ts, int index) { - return Bytes.toBytes(getKeyStr(id, ts, index)); + return getKeyStr(id, ts, index).getBytes(StandardCharsets.UTF_8); } protected long getTimestampFromKey(String key) { diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java b/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java index 0f940620032..7a4b2d0cb9f 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java @@ -20,7 +20,6 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.repository.audit.EntityAuditRepository; -import org.apache.atlas.repository.audit.HBaseBasedAuditRepository; import org.apache.atlas.repository.graphdb.GraphDatabase; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1; import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1; @@ -58,6 +57,8 @@ public class AtlasRepositoryConfiguration { private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = JANUS_GRAPH_DATABASE_IMPLEMENTATION_CLASS; private static final String ENTITY_NOTIFICATION_VERSION_PROPERTY = "atlas.notification.entity.version"; private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl"; + /** Default when property unset; class may be supplied by atlas-audit-hbase on the runtime classpath. */ + private static final String DEFAULT_AUDIT_REPOSITORY_IMPL_CLASS = "org.apache.atlas.repository.audit.HBaseBasedAuditRepository"; private static final String DELETE_HANDLER_V1_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandlerV1.impl"; private static final boolean DEFAULT_GREMLIN_OPTIMZER_ENABLED = true; @@ -103,7 +104,7 @@ public static Class getAuditRepositoryImpl() { try { Configuration config = ApplicationProperties.get(); - return ApplicationProperties.getClass(config, AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class); + return ApplicationProperties.getClass(config, AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, DEFAULT_AUDIT_REPOSITORY_IMPL_CLASS, EntityAuditRepository.class); } catch (AtlasException e) { throw new RuntimeException(e); } diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java index 0e787d78970..09a48997613 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java @@ -31,11 +31,12 @@ import org.apache.atlas.utils.AtlasJson; import org.apache.atlas.utils.TestResourceFileUtils; import org.apache.commons.lang3.StringUtils; -import org.elasticsearch.common.util.CollectionUtils; import org.testng.annotations.Test; import java.io.IOException; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -125,7 +126,9 @@ public void updaterEntityWithUniqueName() throws IOException, AtlasBaseException AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry(); AtlasTypeRegistry.AtlasTransientTypeRegistry ttr = typeRegistry.lockTypeRegistryForUpdate(); - ttr.addTypes(CollectionUtils.newSingletonArrayList(ed)); + List entityDefs = new ArrayList<>(1); + entityDefs.add(ed); + ttr.addTypes(entityDefs); AtlasGraph atlasGraph = mock(AtlasGraph.class); ClassificationAssociatorUpdaterForSpy updater = new ClassificationAssociatorUpdaterForSpy(atlasGraph, ttr, entitiesStore, "col-entity-PII"); diff --git a/tools/atlas-index-repair/pom.xml b/tools/atlas-index-repair/pom.xml index 0b12730d771..2f0c54d7729 100644 --- a/tools/atlas-index-repair/pom.xml +++ b/tools/atlas-index-repair/pom.xml @@ -65,4 +65,24 @@ + + + + + index-elasticsearch + + + atlas.index.backend + elasticsearch + + + + + org.apache.atlas + atlas-graphdb-janus-es-shim + ${project.version} + + + + diff --git a/webapp/pom.xml b/webapp/pom.xml index 9f5c5bb4e6f..bda02d0ac02 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -213,11 +213,24 @@ atlas-server-api + + + org.apache.atlas + hbase-shaded-client-fixed + ${project.version} + + org.apache.commons commons-collections4 + + org.apache.commons + commons-configuration2 + ${commons-conf2.version} + + org.apache.commons commons-lang3 @@ -266,6 +279,13 @@ + + + org.apache.hadoop + hadoop-distcp + ${hadoop.version} + + org.apache.hadoop hadoop-hdfs @@ -304,6 +324,17 @@ + + org.apache.hbase + hbase-shaded-mapreduce + ${hbase.version}-hadoop3 + + + org.apache.hadoop + hadoop-distcp + + + org.apache.httpcomponents @@ -341,18 +372,6 @@ jetty-webapp - - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${elasticsearch.version} - - - org.elasticsearch - elasticsearch - - - - org.glassfish javax.el @@ -532,7 +551,7 @@ true - WEB-INF/lib/hbase-shaded-client-${hbase.version}-hadoop3.jar,WEB-INF/lib/dom4j-*.jar,WEB-INF/lib/jsp-api*.jar,${packages.to.exclude} + WEB-INF/lib/hbase-shaded-client-*-hadoop3.jar,WEB-INF/lib/dom4j-*.jar,WEB-INF/lib/jsp-api*.jar,${atlas.war.audit.excludes}${atlas.war.storage.excludes}${packages.to.exclude} @@ -815,6 +834,163 @@ + + + audit-hbase + + + atlas.audit.backend + hbase + + + + + org.apache.atlas + atlas-audit-hbase + ${project.version} + + + + + audit-cassandra + + + org.apache.atlas + atlas-audit-cassandra + ${project.version} + + + + + audit-rdbms + + + org.apache.atlas + atlas-audit-rdbms + ${project.version} + + + + + + storage-berkeleyje + + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + + + + org.janusgraph + janusgraph-berkeleyje + ${janusgraph.version} + + + ch.qos.logback + * + + + org.noggit + noggit + + + + + + + storage-cassandra + + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + + + + org.janusgraph + janusgraph-cql + ${janusgraph.version} + + + ch.qos.logback + * + + + + + + + storage-hbase + + + atlas.storage.backend + hbase + + + + + org.janusgraph + janusgraph-hbase + ${janusgraph.version} + + + ch.qos.logback + * + + + org.apache.hbase + hbase-shaded-client + + + org.apache.hbase + hbase-shaded-mapreduce + + + + + + + storage-rdbms + + WEB-INF/lib/janusgraph-hbase-*.jar,WEB-INF/lib/hadoop-distcp-*.jar,WEB-INF/lib/hbase-shaded-mapreduce-*.jar, + + + + org.apache.atlas + janusgraph-rdbms + ${project.version} + + + ch.qos.logback + * + + + + + + + index-elasticsearch + + + atlas.index.backend + elasticsearch + + + + + org.apache.atlas + atlas-graphdb-janus-es-shim + ${project.version} + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + org.elasticsearch + elasticsearch + + + + + + dist diff --git a/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java b/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java index 5fb150b0903..64e053b5d12 100644 --- a/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/service/AtlasDebugMetricsSinkTest.java @@ -19,7 +19,6 @@ package org.apache.atlas.web.service; import org.apache.atlas.web.model.DebugMetrics; -import org.apache.hadoop.hbase.shaded.org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.mockito.Mock; @@ -45,9 +44,6 @@ public class AtlasDebugMetricsSinkTest { @Mock private AbstractMetric metric; - @Mock - private SubsetConfiguration subsetConfiguration; - private AtlasDebugMetricsSink debugMetricsSink; @BeforeMethod @@ -316,7 +312,7 @@ public void testFlush() { @Test public void testInit() { // Test that init method doesn't throw any exception - debugMetricsSink.init(subsetConfiguration); + debugMetricsSink.init(null); } @Test From 8cd04f2db43082a672b57ff11c1ce8aed81d7e9c Mon Sep 17 00:00:00 2001 From: Vinayak Marraiya Date: Wed, 3 Jun 2026 11:39:56 +0530 Subject: [PATCH 2/4] ATLAS-5220: Add audit aggregator and clean up pom noise --- audit/audit-cassandra/pom.xml | 4 ++-- audit/audit-hbase/pom.xml | 4 ++-- audit/audit-rdbms/pom.xml | 4 ++-- audit/pom.xml | 39 +++++++++++++++++++++++++++++++++++ build-tools/pom.xml | 38 +++++++++++++++++----------------- dashboard/pom.xml | 4 ---- graphdb/pom.xml | 6 +++--- pom.xml | 4 +--- 8 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 audit/pom.xml diff --git a/audit/audit-cassandra/pom.xml b/audit/audit-cassandra/pom.xml index 5591a3e3468..19bea805800 100644 --- a/audit/audit-cassandra/pom.xml +++ b/audit/audit-cassandra/pom.xml @@ -21,9 +21,9 @@ org.apache.atlas - apache-atlas + atlas-audit-modules 3.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml atlas-audit-cassandra diff --git a/audit/audit-hbase/pom.xml b/audit/audit-hbase/pom.xml index 8f5a2a2c66d..0fb6bcbc35a 100644 --- a/audit/audit-hbase/pom.xml +++ b/audit/audit-hbase/pom.xml @@ -21,9 +21,9 @@ org.apache.atlas - apache-atlas + atlas-audit-modules 3.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml atlas-audit-hbase diff --git a/audit/audit-rdbms/pom.xml b/audit/audit-rdbms/pom.xml index c86056d5d8d..a9b1a9a512b 100644 --- a/audit/audit-rdbms/pom.xml +++ b/audit/audit-rdbms/pom.xml @@ -21,9 +21,9 @@ org.apache.atlas - apache-atlas + atlas-audit-modules 3.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml atlas-audit-rdbms diff --git a/audit/pom.xml b/audit/pom.xml new file mode 100644 index 00000000000..dd3a933b26f --- /dev/null +++ b/audit/pom.xml @@ -0,0 +1,39 @@ + + + + 4.0.0 + + + org.apache.atlas + apache-atlas + 3.0.0-SNAPSHOT + + + atlas-audit-modules + pom + + Apache Atlas Audit Backend Modules + ATLAS-5220: optional entity-audit repository implementations (profile-selected at package time). + + + audit-cassandra + audit-hbase + audit-rdbms + + diff --git a/build-tools/pom.xml b/build-tools/pom.xml index 283601ea965..6f5dc4d0de0 100644 --- a/build-tools/pom.xml +++ b/build-tools/pom.xml @@ -17,27 +17,27 @@ ~ limitations under the License. --> - 4.0.0 + 4.0.0 - org.apache.atlas - atlas-buildtools - 3.0.0-SNAPSHOT - jar + org.apache.atlas + atlas-buildtools + 3.0.0-SNAPSHOT + jar - Apache Atlas Server Build Tools - Apache Atlas Build Tools like Checkstyle + Apache Atlas Server Build Tools + Apache Atlas Build Tools like Checkstyle - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - - + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + diff --git a/dashboard/pom.xml b/dashboard/pom.xml index 4d50c985adc..17bcf14f1e5 100644 --- a/dashboard/pom.xml +++ b/dashboard/pom.xml @@ -72,10 +72,6 @@ ${project.basedir}/ - - **/node_modules/** - **/.vscode/** - false diff --git a/graphdb/pom.xml b/graphdb/pom.xml index 5ad23e9cef3..e01268c2098 100644 --- a/graphdb/pom.xml +++ b/graphdb/pom.xml @@ -31,14 +31,14 @@ Apache Atlas Graph Database Projects Apache Atlas Graph Database Projects - + api common graphdb-impls - janus - janus-elasticsearch-shim janusgraph-rdbms + janus-elasticsearch-shim + janus diff --git a/pom.xml b/pom.xml index 34a28ce227c..f91bcd9068b 100644 --- a/pom.xml +++ b/pom.xml @@ -54,9 +54,7 @@ addons/storm-bridge-shim addons/trino-extractor atlas-examples - audit/audit-cassandra - audit/audit-hbase - audit/audit-rdbms + audit authorization build-tools client From 439cc94e450b72cb3cf831481a16ce02e88e54ae Mon Sep 17 00:00:00 2001 From: Vinayak Marraiya Date: Wed, 3 Jun 2026 11:44:49 +0530 Subject: [PATCH 3/4] ATLAS-5220: Keep audit and graph classes in original modules Remove audit/* and janus-elasticsearch-shim module splits so PR no longer shows rename-only Java files. Restore implementations under repository and graphdb/janus; keep Maven profile-based packaging. Co-authored-by: Cursor --- audit/audit-cassandra/pom.xml | 110 ------------------ audit/audit-hbase/pom.xml | 47 -------- audit/audit-rdbms/pom.xml | 62 ---------- audit/pom.xml | 39 ------- graphdb/janus-elasticsearch-shim/pom.xml | 52 --------- graphdb/janus/pom.xml | 27 +++-- .../AtlasJanusRdbmsUniqueKeyHandler.java | 0 .../diskstorage/es/ElasticSearch7Index.java | 0 graphdb/pom.xml | 5 +- pom.xml | 9 +- repository/pom.xml | 88 +++++++++++++- .../audit/CassandraBasedAuditRepository.java | 0 .../audit/HBaseBasedAuditRepository.java | 0 .../rdbms/RdbmsBasedAuditRepository.java | 0 .../util/AtlasRepositoryConfiguration.java | 2 +- .../audit/CassandraAuditRepositoryTest.java | 0 tools/atlas-index-repair/pom.xml | 7 +- webapp/pom.xml | 42 ------- 18 files changed, 112 insertions(+), 378 deletions(-) delete mode 100644 audit/audit-cassandra/pom.xml delete mode 100644 audit/audit-hbase/pom.xml delete mode 100644 audit/audit-rdbms/pom.xml delete mode 100644 audit/pom.xml delete mode 100644 graphdb/janus-elasticsearch-shim/pom.xml rename graphdb/{janusgraph-rdbms => janus}/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java (100%) rename graphdb/{janus-elasticsearch-shim => janus}/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java (100%) rename {audit/audit-cassandra => repository}/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java (100%) rename {audit/audit-hbase => repository}/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java (100%) rename {audit/audit-rdbms => repository}/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java (100%) rename {audit/audit-cassandra => repository}/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java (100%) diff --git a/audit/audit-cassandra/pom.xml b/audit/audit-cassandra/pom.xml deleted file mode 100644 index 19bea805800..00000000000 --- a/audit/audit-cassandra/pom.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - 4.0.0 - - - org.apache.atlas - atlas-audit-modules - 3.0.0-SNAPSHOT - ../pom.xml - - - atlas-audit-cassandra - jar - - Apache Atlas Audit (Cassandra) - ATLAS-5220: Cassandra entity audit repository; include when atlas.audit.backend=cassandra. - - - - - com.datastax.cassandra - cassandra-driver-core - 3.11.5 - - - ch.qos.logback - * - - - io.dropwizard.metrics - metrics-core - - - io.netty - netty-handler - - - - - org.apache.atlas - atlas-repository - ${project.version} - - - org.apache.cassandra - cassandra-all - 3.11.19 - - - ch.qos.logback - * - - - io.dropwizard.metrics - metrics-core - - - io.netty - netty-handler - - - org.jctools - jctools-core - - - - - - org.apache.atlas - atlas-repository - ${project.version} - test-jar - test - - - org.cassandraunit - cassandra-unit - 2.0.2.2 - test - - - ch.qos.logback - * - - - - - org.testng - testng - test - - - diff --git a/audit/audit-hbase/pom.xml b/audit/audit-hbase/pom.xml deleted file mode 100644 index 0fb6bcbc35a..00000000000 --- a/audit/audit-hbase/pom.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - 4.0.0 - - - org.apache.atlas - atlas-audit-modules - 3.0.0-SNAPSHOT - ../pom.xml - - - atlas-audit-hbase - jar - - Apache Atlas Audit (HBase) - ATLAS-5220: HBase entity audit repository; include on classpath when atlas.audit.backend=hbase. - - - - org.apache.atlas - atlas-repository - ${project.version} - - - org.apache.atlas - hbase-shaded-client-fixed - ${project.version} - - - diff --git a/audit/audit-rdbms/pom.xml b/audit/audit-rdbms/pom.xml deleted file mode 100644 index a9b1a9a512b..00000000000 --- a/audit/audit-rdbms/pom.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - 4.0.0 - - - org.apache.atlas - atlas-audit-modules - 3.0.0-SNAPSHOT - ../pom.xml - - - atlas-audit-rdbms - jar - - Apache Atlas Audit (RDBMS) - ATLAS-5220: JPA/RDBMS entity audit repository; include when atlas.audit.backend=rdbms. - - - - org.apache.atlas - atlas-repository - ${project.version} - - - org.apache.atlas - janusgraph-rdbms - ${project.version} - - - org.eclipse.persistence - eclipselink - ${eclipse.jpa.version} - - - org.eclipse.persistence - javax.persistence - ${javax.persistence.version} - - - org.postgresql - postgresql - ${postgresql.version} - - - diff --git a/audit/pom.xml b/audit/pom.xml deleted file mode 100644 index dd3a933b26f..00000000000 --- a/audit/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - 4.0.0 - - - org.apache.atlas - apache-atlas - 3.0.0-SNAPSHOT - - - atlas-audit-modules - pom - - Apache Atlas Audit Backend Modules - ATLAS-5220: optional entity-audit repository implementations (profile-selected at package time). - - - audit-cassandra - audit-hbase - audit-rdbms - - diff --git a/graphdb/janus-elasticsearch-shim/pom.xml b/graphdb/janus-elasticsearch-shim/pom.xml deleted file mode 100644 index 30080cebabd..00000000000 --- a/graphdb/janus-elasticsearch-shim/pom.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - 4.0.0 - - - org.apache.atlas - atlas-graphdb - 3.0.0-SNAPSHOT - - - atlas-graphdb-janus-es-shim - jar - - Apache Atlas JanusGraph Elasticsearch Shim - ATLAS-5220: optional Janus ES index accessor; add to WAR/tools only when atlas.index.backend=elasticsearch. - - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${elasticsearch.version} - - - org.janusgraph - janusgraph-es - ${janusgraph.version} - - - ch.qos.logback - * - - - - - diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index 47712eff58b..55002c6706a 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -73,7 +73,7 @@ commons-text ${commons-text.version} - + org.apache.lucene lucene-analyzers-common @@ -246,13 +246,6 @@ test - - org.apache.atlas - atlas-graphdb-janus-es-shim - ${project.version} - test - - org.apache.atlas atlas-testtools @@ -303,7 +296,7 @@ - + atlas-storage-hbase @@ -426,6 +419,22 @@ + + atlas-index-elasticsearch + + + atlas.index.backend + elasticsearch + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + diff --git a/graphdb/janusgraph-rdbms/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java similarity index 100% rename from graphdb/janusgraph-rdbms/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java rename to graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusRdbmsUniqueKeyHandler.java diff --git a/graphdb/janus-elasticsearch-shim/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java similarity index 100% rename from graphdb/janus-elasticsearch-shim/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java rename to graphdb/janus/src/main/java/org/janusgraph/diskstorage/es/ElasticSearch7Index.java diff --git a/graphdb/pom.xml b/graphdb/pom.xml index e01268c2098..c4c07948ccd 100644 --- a/graphdb/pom.xml +++ b/graphdb/pom.xml @@ -31,14 +31,13 @@ Apache Atlas Graph Database Projects Apache Atlas Graph Database Projects - + api common graphdb-impls - janusgraph-rdbms - janus-elasticsearch-shim janus + janusgraph-rdbms diff --git a/pom.xml b/pom.xml index f91bcd9068b..1e46e15be6a 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,6 @@ addons/storm-bridge-shim addons/trino-extractor atlas-examples - audit authorization build-tools client @@ -2324,7 +2323,7 @@ cassandra - WEB-INF/lib/atlas-audit-hbase-*.jar, + WEB-INF/lib/hbase-shaded-client-fixed-*.jar, org.apache.atlas.repository.audit.CassandraBasedAuditRepository atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.CassandraBasedAuditRepository @@ -2355,7 +2354,7 @@ inmemory - WEB-INF/lib/atlas-audit-hbase-*.jar, + WEB-INF/lib/hbase-shaded-client-fixed-*.jar, org.apache.atlas.repository.audit.InMemoryEntityAuditRepository atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.InMemoryEntityAuditRepository @@ -2370,7 +2369,7 @@ noop - WEB-INF/lib/atlas-audit-hbase-*.jar, + WEB-INF/lib/hbase-shaded-client-fixed-*.jar, org.apache.atlas.repository.audit.NoopEntityAuditRepository atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.NoopEntityAuditRepository @@ -2385,7 +2384,7 @@ rdbms - WEB-INF/lib/atlas-audit-hbase-*.jar, + WEB-INF/lib/hbase-shaded-client-fixed-*.jar, org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository diff --git a/repository/pom.xml b/repository/pom.xml index 1491dd3fbb5..0f6874ebe9b 100644 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -31,10 +31,6 @@ Apache Atlas Repository Apache Atlas Repository Module - - true false @@ -46,6 +42,25 @@ hppc ${hppc.version} + + com.datastax.cassandra + cassandra-driver-core + 3.11.5 + + + ch.qos.logback + * + + + io.dropwizard.metrics + metrics-core + + + io.netty + netty-handler + + + com.opencsv opencsv @@ -107,6 +122,40 @@ org.apache.atlas atlas-server-api + + org.apache.atlas + hbase-shaded-client-fixed + ${project.version} + + + org.apache.atlas + janusgraph-rdbms + ${project.version} + + + + org.apache.cassandra + cassandra-all + 3.11.19 + + + ch.qos.logback + * + + + io.dropwizard.metrics + metrics-core + + + io.netty + netty-handler + + + org.jctools + jctools-core + + + org.apache.commons commons-lang3 @@ -147,6 +196,18 @@ + + org.eclipse.persistence + eclipselink + ${eclipse.jpa.version} + + + + org.eclipse.persistence + javax.persistence + ${javax.persistence.version} + + org.mockito mockito-all @@ -164,6 +225,12 @@ ${spring.version} + + org.postgresql + postgresql + ${postgresql.version} + + org.testng testng @@ -213,6 +280,19 @@ test-jar test + + + org.cassandraunit + cassandra-unit + 2.0.2.2 + test + + + ch.qos.logback + * + + + diff --git a/audit/audit-cassandra/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java similarity index 100% rename from audit/audit-cassandra/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java rename to repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java diff --git a/audit/audit-hbase/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java similarity index 100% rename from audit/audit-hbase/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java rename to repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java diff --git a/audit/audit-rdbms/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java similarity index 100% rename from audit/audit-rdbms/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java rename to repository/src/main/java/org/apache/atlas/repository/audit/rdbms/RdbmsBasedAuditRepository.java diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java b/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java index 5a0dbad2359..22cf7994245 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java @@ -57,7 +57,7 @@ public class AtlasRepositoryConfiguration { private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = JANUS_GRAPH_DATABASE_IMPLEMENTATION_CLASS; private static final String ENTITY_NOTIFICATION_VERSION_PROPERTY = "atlas.notification.entity.version"; private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl"; - /** Default when property unset; class may be supplied by atlas-audit-hbase on the runtime classpath. */ + /** Default when property unset; implementation class is in atlas-repository. */ private static final String DEFAULT_AUDIT_REPOSITORY_IMPL_CLASS = "org.apache.atlas.repository.audit.HBaseBasedAuditRepository"; private static final String DELETE_HANDLER_V1_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandlerV1.impl"; private static final boolean DEFAULT_GREMLIN_OPTIMZER_ENABLED = true; diff --git a/audit/audit-cassandra/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java similarity index 100% rename from audit/audit-cassandra/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java rename to repository/src/test/java/org/apache/atlas/repository/audit/CassandraAuditRepositoryTest.java diff --git a/tools/atlas-index-repair/pom.xml b/tools/atlas-index-repair/pom.xml index 1d06333871e..18a94c3c1e6 100644 --- a/tools/atlas-index-repair/pom.xml +++ b/tools/atlas-index-repair/pom.xml @@ -71,7 +71,6 @@ - index-elasticsearch @@ -82,9 +81,9 @@ - org.apache.atlas - atlas-graphdb-janus-es-shim - ${project.version} + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} diff --git a/webapp/pom.xml b/webapp/pom.xml index c9840499c48..3f177c1c84c 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -829,43 +829,6 @@ - - - audit-hbase - - - atlas.audit.backend - hbase - - - - - org.apache.atlas - atlas-audit-hbase - ${project.version} - - - - - audit-cassandra - - - org.apache.atlas - atlas-audit-cassandra - ${project.version} - - - - - audit-rdbms - - - org.apache.atlas - atlas-audit-rdbms - ${project.version} - - - storage-berkeleyje @@ -967,11 +930,6 @@ - - org.apache.atlas - atlas-graphdb-janus-es-shim - ${project.version} - org.elasticsearch.client elasticsearch-rest-high-level-client From d27a43673362e16ac841167ffdcd94c2a1f6a810 Mon Sep 17 00:00:00 2001 From: Vinayak Marraiya Date: Wed, 3 Jun 2026 11:49:20 +0530 Subject: [PATCH 4/4] ATLAS-5220: Removed unwanted comments --- distro/src/conf/atlas-application.properties | 2 -- graphdb/janus/pom.xml | 4 ---- .../graphdb/janus/AtlasJanusGraphDatabase.java | 10 +--------- pom.xml | 9 --------- webapp/pom.xml | 2 -- 5 files changed, 1 insertion(+), 26 deletions(-) diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties index b0ef83f2643..c1b6798a530 100755 --- a/distro/src/conf/atlas-application.properties +++ b/distro/src/conf/atlas-application.properties @@ -18,8 +18,6 @@ ######### Graph Database Configs ######### -# ATLAS-5220: Maven build-time backend selection (set via -Pstorage-* / -Pindex-* / -Pdist). Do not point runtime -# config at a different storage/index backend unless you rebuild with matching profiles. atlas.build.storage.backend=${atlas.storage.backend} atlas.build.index.backend=${atlas.index.backend} diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index 55002c6706a..228bf720eeb 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -35,7 +35,6 @@ Some dependencies, like slf4j are excluded from the jar because they are included in Atlas --> - hbase true false @@ -73,7 +72,6 @@ commons-text ${commons-text.version} - org.apache.lucene lucene-analyzers-common @@ -158,7 +156,6 @@ janusgraph-server ${janusgraph.version} - com.codahale.metrics * @@ -296,7 +293,6 @@ - atlas-storage-hbase diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java index 74d2e832a36..9e5f8c9d3ea 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java @@ -78,9 +78,7 @@ public class AtlasJanusGraphDatabase implements GraphDatabase FQCN in Janus {@code StandardStoreManager} without loading the class first. * Pre-loading with {@link Class#forName(String)} caused missing map entries when the class was not yet * visible to the caller's loader at static init time, leading Janus to treat {@code hbase2} as a class name. - * Implementation classes are loaded when the graph opens, as in the pre-ATLAS-5220 behavior. + * Implementation classes are loaded when the graph opens. */ private static void safeRegisterStoreManager(String shortName, String managerClassName) { try { @@ -498,9 +493,6 @@ private static void injectIndexProvider(String shortName, String providerClassNa LOG.debug("Registered Janus index provider {} -> {}", shortName, providerClassName); } - /** - * Register Janus shortName → implementation FQCN only when the class is loadable (ATLAS-5220 slim builds may omit JARs). - */ private static void registerJanusOptionalBackends() { registerStoreIfLoadable("hbase2", HBASE2_STORE_MANAGER_CLASS); registerStoreIfLoadable("cql", CQL_STORE_MANAGER_CLASS); diff --git a/pom.xml b/pom.xml index 1e46e15be6a..c29131dc8df 100644 --- a/pom.xml +++ b/pom.xml @@ -86,7 +86,6 @@ 4.7 1.0 1.8.7 - hbase solr hbase @@ -123,7 +122,6 @@ 0.8 6.5.16 solr - hbase2 2.9.0 25.1-jre @@ -2292,8 +2290,6 @@ - dist @@ -2312,7 +2308,6 @@ false - audit-cassandra @@ -2325,7 +2320,6 @@ cassandra WEB-INF/lib/hbase-shaded-client-fixed-*.jar, org.apache.atlas.repository.audit.CassandraBasedAuditRepository - atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.CassandraBasedAuditRepository @@ -2464,7 +2458,6 @@ true - index-elasticsearch @@ -2475,7 +2468,6 @@ elasticsearch - elasticsearch @@ -2491,7 +2483,6 @@ build - storage-berkeleyje diff --git a/webapp/pom.xml b/webapp/pom.xml index 3f177c1c84c..a8296521a78 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -274,7 +274,6 @@ - org.apache.hadoop hadoop-distcp @@ -829,7 +828,6 @@ - storage-berkeleyje