From 15f223b36362ca2650ea9168771ecd25d00da806 Mon Sep 17 00:00:00 2001 From: Sheetal Shah Date: Mon, 27 Jul 2026 15:53:08 +0530 Subject: [PATCH 1/9] ATLAS-5317: Resilient bulk purge with transactional batching, partial success handling, and purge audit enhancements --- addons/models/0000-Area0/0010-base_model.json | 18 +- .../009-base_model_add_audit_runid.json | 16 + .../010-base_model_add_audit_row_kind.json | 16 + .../org/apache/atlas/AtlasClientV2Test.java | 79 ++ distro/src/conf/atlas-application.properties | 11 + distro/src/conf/atlas-logback.xml | 17 + .../org/apache/atlas/AtlasConfiguration.java | 3 +- .../java/org/apache/atlas/AtlasErrorCode.java | 3 + .../atlas/model/audit/AtlasAuditEntry.java | 27 + .../instance/EntityMutationResponse.java | 102 ++- .../atlas/model/instance/FailedEntity.java | 105 +++ .../atlas/model/instance/MutationSummary.java | 37 + .../atlas/model/instance/PurgeSummary.java | 235 ++++++ intg/src/main/python/apache_atlas/utils.py | 17 +- intg/src/main/resources/atlas-logback.xml | 17 + .../model/audit/TestAtlasAuditEntry.java | 6 + .../instance/TestEntityMutationResponse.java | 64 ++ .../repository/audit/AtlasAuditService.java | 185 ++++- .../repository/ogm/AtlasAuditEntryDTO.java | 14 +- .../repository/purge/PurgeExecutionStats.java | 328 ++++++++ .../atlas/repository/purge/PurgeUtils.java | 351 +++++++++ .../store/graph/AtlasEntityStore.java | 14 +- .../store/graph/v1/DeleteHandlerV1.java | 18 +- .../store/graph/v2/AtlasEntityStoreV2.java | 133 ++-- .../atlas/services/PurgeAuditWriter.java | 120 +++ .../atlas/services/PurgeBatchExecutor.java | 120 +++ .../services/PurgeBatchOrchestrator.java | 479 ++++++++++++ .../atlas/services/PurgeBatchResult.java | 70 ++ .../apache/atlas/services/PurgeService.java | 244 +++--- .../repository/audit/AdminPurgeTest.java | 195 ----- .../audit/AtlasAuditServiceTest.java | 219 ++++++ .../store/graph/v1/DeleteHandlerV1Test.java | 140 +++- .../graph/v2/AtlasEntityStoreV2Test.java | 397 +++++++++- .../store/graph/v2/AtlasEntityTestBase.java | 3 +- .../services/PurgeBatchExecutorTest.java | 208 ++++++ .../services/PurgeBatchOrchestratorTest.java | 657 ++++++++++++++++ .../atlas/services/PurgeServiceTest.java | 700 ++++++++++++++++-- .../atlas/web/resources/AdminResource.java | 161 +++- .../errors/AtlasBaseExceptionMapperTest.java | 19 + .../web/resources/AdminResourceTest.java | 360 ++++++++- 40 files changed, 5373 insertions(+), 535 deletions(-) create mode 100644 addons/models/0000-Area0/patches/009-base_model_add_audit_runid.json create mode 100644 addons/models/0000-Area0/patches/010-base_model_add_audit_row_kind.json create mode 100644 intg/src/main/java/org/apache/atlas/model/instance/FailedEntity.java create mode 100644 intg/src/main/java/org/apache/atlas/model/instance/MutationSummary.java create mode 100644 intg/src/main/java/org/apache/atlas/model/instance/PurgeSummary.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/purge/PurgeExecutionStats.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/purge/PurgeUtils.java create mode 100644 repository/src/main/java/org/apache/atlas/services/PurgeAuditWriter.java create mode 100644 repository/src/main/java/org/apache/atlas/services/PurgeBatchExecutor.java create mode 100644 repository/src/main/java/org/apache/atlas/services/PurgeBatchOrchestrator.java create mode 100644 repository/src/main/java/org/apache/atlas/services/PurgeBatchResult.java delete mode 100644 repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java create mode 100644 repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java create mode 100644 repository/src/test/java/org/apache/atlas/services/PurgeBatchOrchestratorTest.java diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json index 843478ad70e..f2d34f03370 100644 --- a/addons/models/0000-Area0/0010-base_model.json +++ b/addons/models/0000-Area0/0010-base_model.json @@ -15,7 +15,19 @@ { "ordinal": 6, "value": "TYPE_DEF_UPDATE" }, { "ordinal": 7, "value": "TYPE_DEF_DELETE" }, { "ordinal": 8, "value": "SERVER_START" }, - { "ordinal": 9, "value": "SERVER_STATE_ACTIVE" } + { "ordinal": 9, "value": "SERVER_STATE_ACTIVE" }, + { "ordinal": 10, "value": "AUTO_PURGE" } + ] + }, + { + "name": "audit_row_kind", + "description": "Distinguishes single, summary, and batch rows in a correlated audit run", + "typeVersion": "1.0", + "serviceType": "atlas_core", + "elementDefs": [ + { "ordinal": 0, "value": "SINGLE" }, + { "ordinal": 1, "value": "SUMMARY" }, + { "ordinal": 2, "value": "BATCH" } ] } ], @@ -165,7 +177,9 @@ { "name": "clientId", "typeName": "string", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false }, { "name": "params", "typeName": "string", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false }, { "name": "result", "typeName": "string", "cardinality": "SINGLE", "isIndexable": false, "isOptional": true, "isUnique": false }, - { "name": "resultCount", "typeName": "long", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false } + { "name": "resultCount", "typeName": "long", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false }, + { "name": "runId", "typeName": "string", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false }, + { "name": "auditRowKind", "typeName": "audit_row_kind", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false } ] }, { diff --git a/addons/models/0000-Area0/patches/009-base_model_add_audit_runid.json b/addons/models/0000-Area0/patches/009-base_model_add_audit_runid.json new file mode 100644 index 00000000000..de140e91351 --- /dev/null +++ b/addons/models/0000-Area0/patches/009-base_model_add_audit_runid.json @@ -0,0 +1,16 @@ +{ + "patches": [ + { + "id": "TYPEDEF_PATCH_0009_001", + "description": "Add runId attribute to __AtlasAuditEntry for purge audit correlation", + "action": "ADD_ATTRIBUTE", + "typeName": "__AtlasAuditEntry", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "params": null, + "attributeDefs": [ + { "name": "runId", "typeName": "string", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false } + ] + } + ] +} diff --git a/addons/models/0000-Area0/patches/010-base_model_add_audit_row_kind.json b/addons/models/0000-Area0/patches/010-base_model_add_audit_row_kind.json new file mode 100644 index 00000000000..881b4a313f4 --- /dev/null +++ b/addons/models/0000-Area0/patches/010-base_model_add_audit_row_kind.json @@ -0,0 +1,16 @@ +{ + "patches": [ + { + "id": "TYPEDEF_PATCH_0010_001", + "description": "Add auditRowKind attribute to __AtlasAuditEntry for batch/summary audit correlation", + "action": "ADD_ATTRIBUTE", + "typeName": "__AtlasAuditEntry", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": null, + "attributeDefs": [ + { "name": "auditRowKind", "typeName": "audit_row_kind", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false } + ] + } + ] +} diff --git a/client/client-v2/src/test/java/org/apache/atlas/AtlasClientV2Test.java b/client/client-v2/src/test/java/org/apache/atlas/AtlasClientV2Test.java index 52a67707210..c2d2e24d1c3 100644 --- a/client/client-v2/src/test/java/org/apache/atlas/AtlasClientV2Test.java +++ b/client/client-v2/src/test/java/org/apache/atlas/AtlasClientV2Test.java @@ -43,6 +43,9 @@ import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.instance.ClassificationAssociateRequest; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.EntityMutations.EntityOperation; +import org.apache.atlas.model.instance.FailedEntity; +import org.apache.atlas.model.instance.PurgeSummary; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; import org.apache.atlas.model.profile.AtlasUserSavedSearch; import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef; @@ -52,8 +55,10 @@ import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.utils.AtlasJson; import org.apache.commons.configuration2.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.http.HttpStatus; import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.MockitoAnnotations; @@ -79,6 +84,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; @@ -2512,6 +2518,79 @@ public void testPurgeEntitiesByGuidsRealExecution() throws Exception { assertEquals(result, mockResponse); } + @Test + public void testPurgeEntitiesByGuidsAssignsToBaseType() throws Exception { + TestableAtlasClientV2 client = new TestableAtlasClientV2(); + EntityMutationResponse mockResponse = new EntityMutationResponse(); + client.setMockResponse(mockResponse, EntityMutationResponse.class); + + EntityMutationResponse result = client.purgeEntitiesByGuids(Collections.singleton("guid1")); + + assertNotNull(result); + assertEquals(result, mockResponse); + } + + @Test + public void testPurgeEntitiesByGuidsDeserializesPurgeFieldsOnHttp200() throws Exception { + EntityMutationResponse result = invokePurgeWithMockedHttpResponse(HttpStatus.SC_OK, buildSamplePurgeResponseJson()); + + assertPurgeResponseFields(result); + } + + @Test + public void testPurgeEntitiesByGuidsDeserializedResponseAssignsToBaseType() throws Exception { + EntityMutationResponse result = invokePurgeWithMockedHttpResponse(HttpStatus.SC_OK, buildSamplePurgeResponseJson()); + + assertNotNull(result.getEntitiesByOperation(EntityOperation.PURGE)); + assertEquals(result.getEntitiesByOperation(EntityOperation.PURGE).size(), 1); + assertEquals(result.getEntitiesByOperation(EntityOperation.PURGE).get(0).getGuid(), "purged-guid"); + } + + private EntityMutationResponse invokePurgeWithMockedHttpResponse(int httpStatus, String responseJson) throws Exception { + AtlasClientV2 atlasClient = new AtlasClientV2(service, configuration); + WebResource.Builder builder = setupBuilder(AtlasClientV2.API_V2.PURGE_ENTITIES_BY_GUIDS, service); + ClientResponse response = mock(ClientResponse.class); + + when(response.getStatus()).thenReturn(httpStatus); + when(response.getEntity(String.class)).thenReturn(responseJson); + when(builder.method(eq(javax.ws.rs.HttpMethod.PUT), eq(ClientResponse.class), any())).thenReturn(response); + + return atlasClient.purgeEntitiesByGuids(Collections.singleton("purged-guid")); + } + + private static String buildSamplePurgeResponseJson() { + EntityMutationResponse response = new EntityMutationResponse(); + + Map> mutatedEntities = new HashMap<>(); + List purged = new ArrayList<>(); + purged.add(new AtlasEntityHeader("Table", "purged-guid", null)); + mutatedEntities.put(EntityOperation.PURGE, purged); + response.setMutatedEntities(mutatedEntities); + + response.addFailedEntity(new FailedEntity("failed-guid", "ATLAS-404-00-006", "instance not found")); + response.setSummary(new PurgeSummary(2, 1, 0, 1, 0)); + + return AtlasJson.toJson(response); + } + + private static void assertPurgeResponseFields(EntityMutationResponse result) { + assertNotNull(result); + assertNotNull(result.getPurgedEntities()); + assertEquals(result.getPurgedEntities().size(), 1); + assertEquals(result.getPurgedEntities().get(0).getGuid(), "purged-guid"); + + assertNotNull(result.getFailedEntities()); + assertEquals(result.getFailedEntities().size(), 1); + assertEquals(result.getFailedEntities().get(0).getGuid(), "failed-guid"); + assertEquals(result.getFailedEntities().get(0).getErrorCode(), "ATLAS-404-00-006"); + assertEquals(result.getFailedEntities().get(0).getErrorMessage(), "instance not found"); + + assertNotNull(result.getPurgeSummary()); + assertEquals(result.getPurgeSummary().getRequestedCount(), 2); + assertEquals(result.getPurgeSummary().getPurgedCount(), 1); + assertEquals(result.getPurgeSummary().getFailedCount(), 1); + } + @Test public void testAddClassificationByEntityRequest() throws Exception { TestableAtlasClientV2 client = new TestableAtlasClientV2(); diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties index f3bb96875fb..735fb670967 100755 --- a/distro/src/conf/atlas-application.properties +++ b/distro/src/conf/atlas-application.properties @@ -277,6 +277,17 @@ atlas.rest.notification.enableTLS=false #atlas.headers.= +######### Purge Configuration ######## + +# Maximum number of GUIDs allowed per REST purge request (PUT /api/atlas/v2/admin/purge). +#atlas.purge.api.max.request.size=1000 + +# Number of worker threads for REST and scheduled bulk purge (WorkItemManager consumers). +#atlas.purge.workers.count=2 + +# Number of GUIDs committed per worker batch during bulk purge. +#atlas.purge.worker.batch.size=100 + ######### UI Configuration ######## #atlas.ui.default.version=v2 diff --git a/distro/src/conf/atlas-logback.xml b/distro/src/conf/atlas-logback.xml index e6f54b14158..11bb3184786 100755 --- a/distro/src/conf/atlas-logback.xml +++ b/distro/src/conf/atlas-logback.xml @@ -106,6 +106,19 @@ + + ${atlas.log.dir}/purgefailure.log + true + + %date [%thread] %level{5} [%file:%line] %msg%n + + + ${atlas.log.dir}/purgefailure-%d.log + 20 + false + + + ${atlas.log.dir}/notification_processor.log true @@ -180,6 +193,10 @@ + + + + diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 19d4de7e2c3..477235a00c8 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -121,7 +121,8 @@ public enum AtlasConfiguration { ATLAS_ASYNC_IMPORT_MIN_DURATION_OVERRIDE_TEST_AUTOMATION("atlas.async.import.min.duration.override.test.automation", false), ASYNC_IMPORT_TOPIC_PREFIX("atlas.async.import.topic.prefix", "ATLAS_IMPORT_"), ASYNC_IMPORT_REQUEST_ID_PREFIX("atlas.async.import.request_id.prefix", "async_import_"), - REPLACE_HUGE_SPARK_PROCESS_ATTRIBUTES_PATCH("atlas.process.spark.attributes.update.patch", false); + REPLACE_HUGE_SPARK_PROCESS_ATTRIBUTES_PATCH("atlas.process.spark.attributes.update.patch", false), + PURGE_API_MAX_REQUEST_SIZE("atlas.purge.api.max.request.size", 1000); private static final Configuration APPLICATION_PROPERTIES; private final String propertyName; diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index a5aa58a1ba4..9e82b12485b 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -184,6 +184,9 @@ public enum AtlasErrorCode { BLANK_VALUE_ATTRIBUTE(400, "ATLAS-400-00-105", "Value Attribute can't be empty!"), INVALID_RELATIONSHIP_LABEL(400, "ATLAS-400-00-106", "Invalid relationship label {0}. The referenced entity type {1} could not be resolved from the type registry."), NON_INDEXABLE_BM_DELETE_NOT_ALLOWED(400, "ATLAS-400-00-107", "Deletion not allowed for non-indexable Business Metadata ''{0}'' without force=true. Non-indexable attributes cannot be validated efficiently for references; use force=true to skip validation and delete (warning: orphaned references may remain)."), + INVALID_GUID(400, "ATLAS-400-00-108", "guid {0} is not a valid UUID"), + PURGE_REQUEST_SIZE_EXCEEDS_LIMIT(400, "ATLAS-400-00-109", "purge request size {0} exceeds maximum limit {1}"), + NOT_IN_DELETED_STATE(400, "ATLAS-400-00-10A", "entity {0} is not in DELETED state"), UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"), diff --git a/intg/src/main/java/org/apache/atlas/model/audit/AtlasAuditEntry.java b/intg/src/main/java/org/apache/atlas/model/audit/AtlasAuditEntry.java index 722c1a52e4a..5b454f0850c 100644 --- a/intg/src/main/java/org/apache/atlas/model/audit/AtlasAuditEntry.java +++ b/intg/src/main/java/org/apache/atlas/model/audit/AtlasAuditEntry.java @@ -46,6 +46,8 @@ public class AtlasAuditEntry extends AtlasBaseModelObject implements Serializabl private String clientId; private String result; private long resultCount; + private String runId; + private AuditRowKind auditRowKind; public AtlasAuditEntry() { } @@ -120,6 +122,22 @@ public void setResultCount(long resultCount) { this.resultCount = resultCount; } + public String getRunId() { + return runId; + } + + public void setRunId(String runId) { + this.runId = runId; + } + + public AuditRowKind getAuditRowKind() { + return auditRowKind; + } + + public void setAuditRowKind(AuditRowKind auditRowKind) { + this.auditRowKind = auditRowKind; + } + @Override public StringBuilder toString(StringBuilder sb) { sb.append(", userName: ").append(userName); @@ -130,10 +148,18 @@ public StringBuilder toString(StringBuilder sb) { sb.append(", endTime: ").append(endTime); sb.append(", result: ").append(result); sb.append(", resultCount: ").append(resultCount); + sb.append(", runId: ").append(runId); + sb.append(", auditRowKind: ").append(auditRowKind); return sb; } + public enum AuditRowKind { + SINGLE, + SUMMARY, + BATCH + } + public enum AuditOperation { PURGE("PURGE"), AUTO_PURGE("AUTO_PURGE"), @@ -155,6 +181,7 @@ public enum AuditOperation { public EntityAuditEventV2.EntityAuditActionV2 toEntityAuditActionV2() throws AtlasBaseException { switch (this.type) { case "PURGE": + case "AUTO_PURGE": return EntityAuditEventV2.EntityAuditActionV2.ENTITY_PURGE; default: try { diff --git a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java index 79833ab2e31..27f60065d73 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java @@ -21,10 +21,10 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import org.apache.atlas.model.instance.EntityMutations.EntityOperation; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -33,9 +33,11 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; @@ -50,13 +52,18 @@ public class EntityMutationResponse implements Serializable { private static final long serialVersionUID = 1L; private Map> mutatedEntities; + @JsonIgnore + private transient Map> entityGuidsPerOp; private Map guidAssignments; + private List failedEntities; + private MutationSummary summary; public EntityMutationResponse() { } public EntityMutationResponse(final Map> mutatedEntities) { this.mutatedEntities = mutatedEntities; + rebuildEntityGuidsPerOp(); } public Map> getMutatedEntities() { @@ -65,6 +72,7 @@ public Map> getMutatedEntities() { public void setMutatedEntities(final Map> mutatedEntities) { this.mutatedEntities = mutatedEntities; + rebuildEntityGuidsPerOp(); } public Map getGuidAssignments() { @@ -75,6 +83,40 @@ public void setGuidAssignments(Map guidAssignments) { this.guidAssignments = guidAssignments; } + public List getFailedEntities() { + return failedEntities; + } + + public void setFailedEntities(List failedEntities) { + this.failedEntities = failedEntities; + } + + public MutationSummary getSummary() { + return summary; + } + + @JsonDeserialize(as = PurgeSummary.class) + public void setSummary(MutationSummary summary) { + this.summary = summary; + } + + @JsonIgnore + public PurgeSummary getPurgeSummary() { + return summary instanceof PurgeSummary ? (PurgeSummary) summary : null; + } + + public void addFailedEntity(FailedEntity failedEntity) { + if (failedEntity == null) { + return; + } + + if (failedEntities == null) { + failedEntities = new ArrayList<>(); + } + + failedEntities.add(failedEntity); + } + @JsonIgnore public List getEntitiesByOperation(EntityOperation op) { if (mutatedEntities != null) { @@ -228,9 +270,14 @@ public AtlasEntityHeader getFirstPartialUpdatedEntityByTypeName(String typeName) @JsonIgnore public void addEntity(EntityOperation op, AtlasEntityHeader header) { - // if an entity is already included in CREATE, update the header, to capture propagated classifications + if (header == null) { + return; + } + + // Duplicate GUID under CREATE: keep the first header, ignore later additions + String guid = header.getGuid(); if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) { - if (entityHeaderExists(getCreatedEntities(), header.getGuid())) { + if (entityHeaderExists(EntityOperation.CREATE, guid)) { op = EntityOperation.CREATE; } } @@ -241,7 +288,12 @@ public void addEntity(EntityOperation op, AtlasEntityHeader header) { List opEntities = mutatedEntities.computeIfAbsent(op, k -> new ArrayList<>()); - if (!entityHeaderExists(opEntities, header.getGuid())) { + if (guid == null) { + opEntities.add(header); + return; + } + + if (getGuidsForOperation(op).add(guid)) { opEntities.add(header); } } @@ -258,7 +310,7 @@ public StringBuilder toString(StringBuilder sb) { @Override public int hashCode() { - return Objects.hash(mutatedEntities, guidAssignments); + return Objects.hash(mutatedEntities, guidAssignments, failedEntities, summary); } @Override @@ -272,7 +324,9 @@ public boolean equals(Object o) { EntityMutationResponse that = (EntityMutationResponse) o; return Objects.equals(mutatedEntities, that.mutatedEntities) && - Objects.equals(guidAssignments, that.guidAssignments); + Objects.equals(guidAssignments, that.guidAssignments) && + Objects.equals(failedEntities, that.failedEntities) && + Objects.equals(summary, that.summary); } @Override @@ -280,19 +334,39 @@ public String toString() { return toString(new StringBuilder()).toString(); } - private boolean entityHeaderExists(List entityHeaders, String guid) { - boolean ret = false; + private Set getGuidsForOperation(EntityOperation op) { + if (entityGuidsPerOp == null) { + rebuildEntityGuidsPerOp(); + } + + return entityGuidsPerOp.computeIfAbsent(op, k -> new HashSet<>()); + } + + private void rebuildEntityGuidsPerOp() { + entityGuidsPerOp = new HashMap<>(); - if (CollectionUtils.isNotEmpty(entityHeaders) && guid != null) { - for (AtlasEntityHeader entityHeader : entityHeaders) { - if (StringUtils.equals(entityHeader.getGuid(), guid)) { - ret = true; - break; + if (mutatedEntities == null) { + return; + } + + for (Map.Entry> entry : mutatedEntities.entrySet()) { + Set guids = new HashSet<>(); + List headers = entry.getValue(); + + if (headers != null) { + for (AtlasEntityHeader header : headers) { + if (header.getGuid() != null) { + guids.add(header.getGuid()); + } } } + + entityGuidsPerOp.put(entry.getKey(), guids); } + } - return ret; + private boolean entityHeaderExists(EntityOperation op, String guid) { + return guid != null && getGuidsForOperation(op).contains(guid); } private AtlasEntityHeader getFirstEntityByType(List entitiesByOperation, String typeName) { diff --git a/intg/src/main/java/org/apache/atlas/model/instance/FailedEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/FailedEntity.java new file mode 100644 index 00000000000..9bdcd9e0e79 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/FailedEntity.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.io.Serializable; +import java.util.Objects; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +/** + * Represents a single GUID that failed during a mutation operation, with error code and message. + */ +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class FailedEntity implements Serializable { + private static final long serialVersionUID = 1L; + + private String guid; + private String errorCode; + private String errorMessage; + + public FailedEntity() { + } + + public FailedEntity(String guid, String errorCode, String errorMessage) { + this.guid = guid; + this.errorCode = errorCode; + this.errorMessage = errorMessage; + } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public String getErrorCode() { + return errorCode; + } + + public void setErrorCode(String errorCode) { + this.errorCode = errorCode; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("FailedEntity{"); + sb.append("guid='").append(guid).append('\''); + sb.append(", errorCode='").append(errorCode).append('\''); + sb.append(", errorMessage='").append(errorMessage).append('\''); + sb.append('}'); + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FailedEntity that = (FailedEntity) o; + return Objects.equals(guid, that.guid) && + Objects.equals(errorCode, that.errorCode) && + Objects.equals(errorMessage, that.errorMessage); + } + + @Override + public int hashCode() { + return Objects.hash(guid, errorCode, errorMessage); + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/instance/MutationSummary.java b/intg/src/main/java/org/apache/atlas/model/instance/MutationSummary.java new file mode 100644 index 00000000000..4270aa30404 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/MutationSummary.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.io.Serializable; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +/** + * Base class for operation-specific mutation summary counts. + */ +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public abstract class MutationSummary implements Serializable { + private static final long serialVersionUID = 1L; +} diff --git a/intg/src/main/java/org/apache/atlas/model/instance/PurgeSummary.java b/intg/src/main/java/org/apache/atlas/model/instance/PurgeSummary.java new file mode 100644 index 00000000000..d4e9a53aab3 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/PurgeSummary.java @@ -0,0 +1,235 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.util.Objects; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +/** + * Summary counts for purge operations. + */ +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class PurgeSummary extends MutationSummary { + private static final long serialVersionUID = 1L; + + private long requestedCount; + private long purgedCount; + private long purgedDependenciesCount; + private long failedCount; + private long failedDependenciesCount; + private long skippedCount; + private long validGuidCount; + private boolean executionFailed; + private long expandedEntityCount; + private long skippedRequestedCount; + private long skippedDependenciesCount; + private long unprocessedCount; + private long batchCount; + private String runId; + + public PurgeSummary() { + } + + public PurgeSummary(long requestedCount, long purgedCount, long purgedDependenciesCount, long failedCount, long skippedCount) { + this(requestedCount, purgedCount, purgedDependenciesCount, failedCount, 0, skippedCount); + } + + public PurgeSummary(long requestedCount, long purgedCount, long purgedDependenciesCount, long failedCount, + long failedDependenciesCount, long skippedCount) { + this.requestedCount = requestedCount; + this.purgedCount = purgedCount; + this.purgedDependenciesCount = purgedDependenciesCount; + this.failedCount = failedCount; + this.failedDependenciesCount = failedDependenciesCount; + this.skippedCount = skippedCount; + } + + public long getRequestedCount() { + return requestedCount; + } + + public void setRequestedCount(long requestedCount) { + this.requestedCount = requestedCount; + } + + public long getPurgedCount() { + return purgedCount; + } + + public void setPurgedCount(long purgedCount) { + this.purgedCount = purgedCount; + } + + public long getPurgedDependenciesCount() { + return purgedDependenciesCount; + } + + public void setPurgedDependenciesCount(long purgedDependenciesCount) { + this.purgedDependenciesCount = purgedDependenciesCount; + } + + public long getFailedCount() { + return failedCount; + } + + public void setFailedCount(long failedCount) { + this.failedCount = failedCount; + } + + public long getFailedDependenciesCount() { + return failedDependenciesCount; + } + + public void setFailedDependenciesCount(long failedDependenciesCount) { + this.failedDependenciesCount = failedDependenciesCount; + } + + public long getSkippedCount() { + return skippedCount; + } + + public void setSkippedCount(long skippedCount) { + this.skippedCount = skippedCount; + } + + public long getValidGuidCount() { + return validGuidCount; + } + + public void setValidGuidCount(long validGuidCount) { + this.validGuidCount = validGuidCount; + } + + public boolean getExecutionFailed() { + return executionFailed; + } + + public void setExecutionFailed(boolean executionFailed) { + this.executionFailed = executionFailed; + } + + public long getExpandedEntityCount() { + return expandedEntityCount; + } + + public void setExpandedEntityCount(long expandedEntityCount) { + this.expandedEntityCount = expandedEntityCount; + } + + public long getSkippedRequestedCount() { + return skippedRequestedCount; + } + + public void setSkippedRequestedCount(long skippedRequestedCount) { + this.skippedRequestedCount = skippedRequestedCount; + } + + public long getSkippedDependenciesCount() { + return skippedDependenciesCount; + } + + public void setSkippedDependenciesCount(long skippedDependenciesCount) { + this.skippedDependenciesCount = skippedDependenciesCount; + } + + public long getUnprocessedCount() { + return unprocessedCount; + } + + public void setUnprocessedCount(long unprocessedCount) { + this.unprocessedCount = unprocessedCount; + } + + public long getBatchCount() { + return batchCount; + } + + public void setBatchCount(long batchCount) { + this.batchCount = batchCount; + } + + public String getRunId() { + return runId; + } + + public void setRunId(String runId) { + this.runId = runId; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("PurgeSummary{"); + sb.append("requestedCount=").append(requestedCount); + sb.append(", purgedCount=").append(purgedCount); + sb.append(", purgedDependenciesCount=").append(purgedDependenciesCount); + sb.append(", failedCount=").append(failedCount); + sb.append(", failedDependenciesCount=").append(failedDependenciesCount); + sb.append(", skippedCount=").append(skippedCount); + sb.append(", validGuidCount=").append(validGuidCount); + sb.append(", executionFailed=").append(executionFailed); + sb.append(", expandedEntityCount=").append(expandedEntityCount); + sb.append(", skippedRequestedCount=").append(skippedRequestedCount); + sb.append(", skippedDependenciesCount=").append(skippedDependenciesCount); + sb.append(", unprocessedCount=").append(unprocessedCount); + sb.append(", batchCount=").append(batchCount); + sb.append(", runId=").append(runId); + sb.append('}'); + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PurgeSummary that = (PurgeSummary) o; + return requestedCount == that.requestedCount && + purgedCount == that.purgedCount && + purgedDependenciesCount == that.purgedDependenciesCount && + failedCount == that.failedCount && + failedDependenciesCount == that.failedDependenciesCount && + skippedCount == that.skippedCount && + validGuidCount == that.validGuidCount && + executionFailed == that.executionFailed && + expandedEntityCount == that.expandedEntityCount && + skippedRequestedCount == that.skippedRequestedCount && + skippedDependenciesCount == that.skippedDependenciesCount && + unprocessedCount == that.unprocessedCount && + batchCount == that.batchCount && + Objects.equals(runId, that.runId); + } + + @Override + public int hashCode() { + return Objects.hash(requestedCount, purgedCount, purgedDependenciesCount, failedCount, failedDependenciesCount, + skippedCount, validGuidCount, executionFailed, expandedEntityCount, skippedRequestedCount, + skippedDependenciesCount, unprocessedCount, batchCount, runId); + } +} diff --git a/intg/src/main/python/apache_atlas/utils.py b/intg/src/main/python/apache_atlas/utils.py index b8540121220..6a3c709b5f4 100644 --- a/intg/src/main/python/apache_atlas/utils.py +++ b/intg/src/main/python/apache_atlas/utils.py @@ -112,12 +112,20 @@ def type_coerce_dict_list(obj, objType): class API: - def __init__(self, path, method, expected_status, consumes=APPLICATION_JSON, produces=APPLICATION_JSON): + def __init__(self, path, method, expected_status, consumes=APPLICATION_JSON, produces=APPLICATION_JSON, + alternate_expected_statuses=None): self.path = path self.method = method self.expected_status = expected_status self.consumes = consumes self.produces = produces + self.alternate_expected_statuses = alternate_expected_statuses or [] + + def matches_expected_status(self, status_code): + if status_code == self.expected_status: + return True + + return status_code in self.alternate_expected_statuses def multipart_urljoin(self, base_path, *path_elems): """Join a base path and multiple context path elements. Handle single @@ -136,11 +144,13 @@ def urljoin_pair(left, right): return reduce(urljoin_pair, path_elems, base_path) def format_path(self, params): - return API(self.path.format(**params), self.method, self.expected_status, self.consumes, self.produces) + return API(self.path.format(**params), self.method, self.expected_status, self.consumes, self.produces, + self.alternate_expected_statuses) def format_path_with_params(self, *params): request_path = self.multipart_urljoin(self.path, *params) - return API(request_path, self.method, self.expected_status, self.consumes, self.produces) + return API(request_path, self.method, self.expected_status, self.consumes, self.produces, + self.alternate_expected_statuses) class HTTPMethod(enum.Enum): @@ -152,5 +162,6 @@ class HTTPMethod(enum.Enum): class HTTPStatus: OK = 200 + MULTI_STATUS = 207 NO_CONTENT = 204 SERVICE_UNAVAILABLE = 503 diff --git a/intg/src/main/resources/atlas-logback.xml b/intg/src/main/resources/atlas-logback.xml index 74052748260..2534f6172a9 100755 --- a/intg/src/main/resources/atlas-logback.xml +++ b/intg/src/main/resources/atlas-logback.xml @@ -67,6 +67,19 @@ + + ${atlas.log.dir}/purgefailure.log + true + + %date [%thread] %level{5} [%file:%line] %msg%n + + + ${atlas.log.dir}/purgefailure-%d.log + 20 + false + + + @@ -91,6 +104,10 @@ + + + + diff --git a/intg/src/test/java/org/apache/atlas/model/audit/TestAtlasAuditEntry.java b/intg/src/test/java/org/apache/atlas/model/audit/TestAtlasAuditEntry.java index 973cb447bd9..f9214ed06ae 100644 --- a/intg/src/test/java/org/apache/atlas/model/audit/TestAtlasAuditEntry.java +++ b/intg/src/test/java/org/apache/atlas/model/audit/TestAtlasAuditEntry.java @@ -272,6 +272,12 @@ public void testAuditOperationToEntityAuditActionV2Purge() throws AtlasBaseExcep assertEquals(action, EntityAuditEventV2.EntityAuditActionV2.ENTITY_PURGE); } + @Test + public void testAuditOperationToEntityAuditActionV2AutoPurge() throws AtlasBaseException { + EntityAuditEventV2.EntityAuditActionV2 action = AtlasAuditEntry.AuditOperation.AUTO_PURGE.toEntityAuditActionV2(); + assertEquals(action, EntityAuditEventV2.EntityAuditActionV2.ENTITY_PURGE); + } + @Test(expectedExceptions = AtlasBaseException.class) public void testAuditOperationToEntityAuditActionV2InvalidOperation() throws AtlasBaseException { AtlasAuditEntry.AuditOperation.SERVER_START.toEntityAuditActionV2(); diff --git a/intg/src/test/java/org/apache/atlas/model/instance/TestEntityMutationResponse.java b/intg/src/test/java/org/apache/atlas/model/instance/TestEntityMutationResponse.java index 8b9bea6cdba..62689c68431 100644 --- a/intg/src/test/java/org/apache/atlas/model/instance/TestEntityMutationResponse.java +++ b/intg/src/test/java/org/apache/atlas/model/instance/TestEntityMutationResponse.java @@ -586,4 +586,68 @@ public void testComplexScenario() { assertEquals(guidAssignments, response.getGuidAssignments()); } + + @Test + public void testSetMutatedEntitiesThenAddEntity() { + EntityMutationResponse response = new EntityMutationResponse(); + + Map> mutatedEntities = new HashMap<>(); + List purgedEntities = new ArrayList<>(); + purgedEntities.add(new AtlasEntityHeader("type1", "guid1", null)); + mutatedEntities.put(EntityOperation.PURGE, purgedEntities); + + // Rebuilds the GUID index from the map; duplicate add must be ignored. + response.setMutatedEntities(mutatedEntities); + + // Add the same entity again — must not duplicate. + AtlasEntityHeader duplicateEntity = new AtlasEntityHeader("type1", "guid1", null); + response.addEntity(EntityOperation.PURGE, duplicateEntity); + + assertEquals(1, response.getPurgedEntities().size()); + assertEquals("guid1", response.getPurgedEntities().get(0).getGuid()); + } + + @Test + public void testAddEntityThenSetMutatedEntitiesThenAddEntity() { + EntityMutationResponse response = new EntityMutationResponse(); + + // 1. addEntity runs -> index has CREATE -> {guid1} + AtlasEntityHeader entity1 = new AtlasEntityHeader("type1", "guid1", null); + response.addEntity(EntityOperation.CREATE, entity1); + + assertEquals(1, response.getCreatedEntities().size()); + + // 2. setMutatedEntities replaces lists + Map> newMutatedEntities = new HashMap<>(); + List newCreatedEntities = new ArrayList<>(); + newCreatedEntities.add(new AtlasEntityHeader("type2", "guid2", null)); + newMutatedEntities.put(EntityOperation.CREATE, newCreatedEntities); + + // This must invalidate the cache! + response.setMutatedEntities(newMutatedEntities); + + // 3. addEntity runs again for guid2 already in the new list + AtlasEntityHeader entity2 = new AtlasEntityHeader("type2", "guid2", null); + response.addEntity(EntityOperation.CREATE, entity2); + // If cache wasn't invalidated, entityHeaderExists might use stale cache {guid1} + // and add guid2 again, resulting in 2 entries for guid2 (actually size 2 in the list). + assertEquals(1, response.getCreatedEntities().size()); + assertEquals("guid2", response.getCreatedEntities().get(0).getGuid()); + } + + @Test + public void testNullGuidBehavior() { + EntityMutationResponse response = new EntityMutationResponse(); + + AtlasEntityHeader nullGuidEntity1 = new AtlasEntityHeader("type1", null, null); + AtlasEntityHeader nullGuidEntity2 = new AtlasEntityHeader("type2", null, null); + + // Null GUIDs are not indexed, but they should still be added to the lists. + response.addEntity(EntityOperation.CREATE, nullGuidEntity1); + response.addEntity(EntityOperation.CREATE, nullGuidEntity2); + + assertEquals(2, response.getCreatedEntities().size()); + assertNull(response.getCreatedEntities().get(0).getGuid()); + assertNull(response.getCreatedEntities().get(1).getGuid()); + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/AtlasAuditService.java b/repository/src/main/java/org/apache/atlas/repository/audit/AtlasAuditService.java index a14aedfe385..12dde833ed7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/AtlasAuditService.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/AtlasAuditService.java @@ -18,21 +18,26 @@ package org.apache.atlas.repository.audit; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; +import org.apache.atlas.SortOrder; import org.apache.atlas.annotation.AtlasService; import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.discovery.AtlasDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AtlasAuditEntry; import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditRowKind; import org.apache.atlas.model.audit.AuditSearchParameters; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.PurgeSummary; import org.apache.atlas.repository.ogm.AtlasAuditEntryDTO; import org.apache.atlas.repository.ogm.DataAccess; +import org.apache.atlas.repository.purge.PurgeUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -45,7 +50,9 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; @AtlasService public class AtlasAuditService { @@ -68,13 +75,31 @@ public void save(AtlasAuditEntry entry) throws AtlasBaseException { } public void add(AuditOperation operation, String params, String result, long resultCount) throws AtlasBaseException { + add(operation, params, result, resultCount, null); + } + + public void add(AuditOperation operation, String params, String result, long resultCount, String runId) throws AtlasBaseException { + add(operation, params, result, resultCount, runId, AuditRowKind.SINGLE); + } + + public void add(AuditOperation operation, String params, String result, long resultCount, String runId, + AuditRowKind auditRowKind) throws AtlasBaseException { final Date startTime = new Date(RequestContext.get().getRequestTime()); final Date endTime = new Date(); - add(operation, startTime, endTime, params, result, resultCount); + add(operation, startTime, endTime, params, result, resultCount, runId, auditRowKind); } public void add(AuditOperation operation, Date startTime, Date endTime, String params, String result, long resultCount) throws AtlasBaseException { + add(operation, startTime, endTime, params, result, resultCount, null, AuditRowKind.SINGLE); + } + + public void add(AuditOperation operation, Date startTime, Date endTime, String params, String result, long resultCount, String runId) throws AtlasBaseException { + add(operation, startTime, endTime, params, result, resultCount, runId, AuditRowKind.SINGLE); + } + + public void add(AuditOperation operation, Date startTime, Date endTime, String params, String result, long resultCount, + String runId, AuditRowKind auditRowKind) throws AtlasBaseException { String userName = RequestContext.get().getCurrentUser(); String clientId = RequestContext.get().getClientIPAddress(); @@ -88,10 +113,19 @@ public void add(AuditOperation operation, Date startTime, Date endTime, String p } } - add(userName, operation, clientId, startTime, endTime, params, result, resultCount); + add(userName, operation, clientId, startTime, endTime, params, result, resultCount, runId, auditRowKind); } public void add(String userName, AuditOperation operation, String clientId, Date startTime, Date endTime, String params, String result, long resultCount) throws AtlasBaseException { + add(userName, operation, clientId, startTime, endTime, params, result, resultCount, null, AuditRowKind.SINGLE); + } + + public void add(String userName, AuditOperation operation, String clientId, Date startTime, Date endTime, String params, String result, long resultCount, String runId) throws AtlasBaseException { + add(userName, operation, clientId, startTime, endTime, params, result, resultCount, runId, AuditRowKind.SINGLE); + } + + public void add(String userName, AuditOperation operation, String clientId, Date startTime, Date endTime, String params, String result, long resultCount, + String runId, AuditRowKind auditRowKind) throws AtlasBaseException { LOG.debug("==> AtlasAuditService.add()"); AtlasAuditEntry entry = new AtlasAuditEntry(); @@ -104,6 +138,8 @@ public void add(String userName, AuditOperation operation, String clientId, Date entry.setParams(params); entry.setResult(result); entry.setResultCount(resultCount); + entry.setRunId(runId); + entry.setAuditRowKind(auditRowKind != null ? auditRowKind : AuditRowKind.SINGLE); save(entry); @@ -114,6 +150,100 @@ public void add(String userName, AuditOperation operation, String clientId, Date } } + public List getPurgedEntityGuidsForRun(AtlasAuditEntry summaryEntry) throws AtlasBaseException { + List batchRows = getPurgeBatchAuditEntriesForRun(summaryEntry); + List purgedGuids = PurgeUtils.collectPurgedGuidsFromBatchEntries(batchRows); + + PurgeSummary summary = PurgeUtils.parsePurgeSummary(summaryEntry); + if (summary != null) { + long expectedPurgedGuids = summary.getPurgedCount() + summary.getPurgedDependenciesCount(); + if (expectedPurgedGuids > 0 && purgedGuids.size() < expectedPurgedGuids) { + LOG.warn("Purged entity GUID count {} is less than summary expected {} for runId={}", + purgedGuids.size(), expectedPurgedGuids, PurgeUtils.resolveRunId(summaryEntry)); + } + } + + return purgedGuids; + } + + public List getPurgeBatchAuditGuidsForRun(AtlasAuditEntry summaryEntry) throws AtlasBaseException { + return getPurgeBatchAuditEntriesForRun(summaryEntry).stream() + .map(AtlasAuditEntry::getGuid) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private List getPurgeBatchAuditEntriesForRun(AtlasAuditEntry summaryEntry) throws AtlasBaseException { + String runId = PurgeUtils.resolveRunId(summaryEntry); + if (StringUtils.isBlank(runId)) { + return new ArrayList<>(); + } + + return getPurgeBatchAuditEntriesByRunId(runId); + } + + private List getPurgeBatchAuditEntriesByRunId(String runId) throws AtlasBaseException { + if (StringUtils.isBlank(runId)) { + return new ArrayList<>(); + } + + int pageSize = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt(); + int offset = 0; + List all = new ArrayList<>(); + + while (true) { + SearchParameters searchParameters = new SearchParameters(); + searchParameters.setTypeName(ENTITY_TYPE_AUDIT_ENTRY); + searchParameters.setEntityFilters(buildPurgeBatchRunIdFilter(runId)); + searchParameters.setLimit(pageSize); + searchParameters.setOffset(offset); + searchParameters.setSortBy(AtlasAuditEntryDTO.ATTRIBUTE_START_TIME); + searchParameters.setSortOrder(SortOrder.ASCENDING); + searchParameters.setAttributes(getAuditEntityAttributes()); + + List page = toAtlasAuditEntries(discoveryService.searchWithParameters(searchParameters)); + + if (CollectionUtils.isEmpty(page)) { + break; + } + + all.addAll(page); + + if (page.size() < pageSize) { + break; + } + + offset += pageSize; + } + + return all; + } + + private SearchParameters.FilterCriteria buildPurgeBatchRunIdFilter(String runId) throws AtlasBaseException { + SearchParameters.FilterCriteria root = new SearchParameters.FilterCriteria(); + root.setCondition(SearchParameters.FilterCriteria.Condition.AND); + root.setCriterion(new ArrayList<>()); + + addParameterIfValueNotEmpty(root, AtlasAuditEntryDTO.ATTRIBUTE_RUN_ID, SearchParameters.Operator.EQ, runId); + addParameterIfValueNotEmpty(root, AtlasAuditEntryDTO.ATTRIBUTE_AUDIT_ROW_KIND, SearchParameters.Operator.EQ, + AuditRowKind.BATCH.name()); + + return getNonEmptyFilter(root); + } + + private void addParameterIfValueNotEmpty(SearchParameters.FilterCriteria criteria, String attributeName, + SearchParameters.Operator operator, String value) { + if (StringUtils.isEmpty(value)) { + return; + } + + SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); + filterCriteria.setAttributeName(attributeName); + filterCriteria.setAttributeValue(value); + filterCriteria.setOperator(operator); + criteria.getCriterion().add(filterCriteria); + } + public AtlasAuditEntry get(AtlasAuditEntry entry) throws AtlasBaseException { if (entry.getGuid() == null) { throw new AtlasBaseException("Entity does not have GUID set. load cannot proceed."); @@ -195,36 +325,37 @@ private void validateSortByParameter(String sortBy) throws AtlasBaseException { } private SearchParameters.FilterCriteria getNonEmptyFilter(SearchParameters.FilterCriteria auditFilter) throws AtlasBaseException { - SearchParameters.FilterCriteria outCriteria = new SearchParameters.FilterCriteria(); - - outCriteria.setCriterion(new ArrayList<>()); - - if (auditFilter != null) { - outCriteria.setCondition(auditFilter.getCondition()); - - List givenFilterCriterion = auditFilter.getCriterion(); + if (auditFilter == null) { + return null; + } - for (SearchParameters.FilterCriteria each : givenFilterCriterion) { - if (StringUtils.isNotEmpty(each.getAttributeName()) && !AtlasAuditEntryDTO.getAttributes().contains(each.getAttributeName())) { - throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, each.getAttributeName(), "Atlas Audit Entry"); + SearchParameters.FilterCriteria outCriteria = new SearchParameters.FilterCriteria(); + outCriteria.setCondition(auditFilter.getCondition()); + + if (auditFilter.getCriterion() != null) { + outCriteria.setCriterion(new ArrayList<>()); + for (SearchParameters.FilterCriteria each : auditFilter.getCriterion()) { + if (each.getCondition() != null && CollectionUtils.isNotEmpty(each.getCriterion())) { + SearchParameters.FilterCriteria nested = getNonEmptyFilter(each); + if (nested != null && CollectionUtils.isNotEmpty(nested.getCriterion())) { + outCriteria.getCriterion().add(nested); + } + } else { + if (StringUtils.isNotEmpty(each.getAttributeName()) && !AtlasAuditEntryDTO.getAttributes().contains(each.getAttributeName())) { + throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, each.getAttributeName(), "Atlas Audit Entry"); + } + + if (StringUtils.isNotEmpty(each.getAttributeValue())) { + SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); + filterCriteria.setAttributeName(each.getAttributeName()); + filterCriteria.setAttributeValue(each.getAttributeValue()); + filterCriteria.setOperator(each.getOperator()); + outCriteria.getCriterion().add(filterCriteria); + } } - - addParameterIfValueNotEmpty(outCriteria, each.getAttributeName(), each.getOperator(), each.getAttributeValue()); } } return outCriteria; } - - private void addParameterIfValueNotEmpty(SearchParameters.FilterCriteria criteria, String attributeName, SearchParameters.Operator operator, String value) { - if (StringUtils.isNotEmpty(value)) { - SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); - - filterCriteria.setAttributeName(attributeName); - filterCriteria.setAttributeValue(value); - filterCriteria.setOperator(operator); - - criteria.getCriterion().add(filterCriteria); - } - } } diff --git a/repository/src/main/java/org/apache/atlas/repository/ogm/AtlasAuditEntryDTO.java b/repository/src/main/java/org/apache/atlas/repository/ogm/AtlasAuditEntryDTO.java index 8db49a6c05b..8f825b93399 100644 --- a/repository/src/main/java/org/apache/atlas/repository/ogm/AtlasAuditEntryDTO.java +++ b/repository/src/main/java/org/apache/atlas/repository/ogm/AtlasAuditEntryDTO.java @@ -43,11 +43,14 @@ public class AtlasAuditEntryDTO extends AbstractDataTransferObject ATTRIBUTE_NAMES = new HashSet<>(Arrays.asList(ATTRIBUTE_USER_NAME, ATTRIBUTE_OPERATION, ATTRIBUTE_PARAMS, ATTRIBUTE_START_TIME, ATTRIBUTE_END_TIME, - ATTRIBUTE_CLIENT_ID, ATTRIBUTE_RESULT, ATTRIBUTE_RESULT_COUNT)); + ATTRIBUTE_CLIENT_ID, ATTRIBUTE_RESULT, ATTRIBUTE_RESULT_COUNT, ATTRIBUTE_RUN_ID, + ATTRIBUTE_AUDIT_ROW_KIND)); @Inject public AtlasAuditEntryDTO(AtlasTypeRegistry typeRegistry) { @@ -70,6 +73,12 @@ public static AtlasAuditEntry from(String guid, Map attributes) entry.setClientId((String) attributes.get(ATTRIBUTE_CLIENT_ID)); entry.setResult((String) attributes.get(ATTRIBUTE_RESULT)); entry.setResultCount((long) attributes.get(ATTRIBUTE_RESULT_COUNT)); + entry.setRunId((String) attributes.get(ATTRIBUTE_RUN_ID)); + + Object auditRowKind = attributes.get(ATTRIBUTE_AUDIT_ROW_KIND); + if (auditRowKind != null) { + entry.setAuditRowKind(AtlasAuditEntry.AuditRowKind.valueOf(auditRowKind.toString())); + } return entry; } @@ -96,6 +105,9 @@ public AtlasEntity toEntity(AtlasAuditEntry obj) { entity.setAttribute(ATTRIBUTE_CLIENT_ID, obj.getClientId()); entity.setAttribute(ATTRIBUTE_RESULT, obj.getResult()); entity.setAttribute(ATTRIBUTE_RESULT_COUNT, obj.getResultCount()); + entity.setAttribute(ATTRIBUTE_RUN_ID, obj.getRunId()); + entity.setAttribute(ATTRIBUTE_AUDIT_ROW_KIND, + obj.getAuditRowKind() != null ? obj.getAuditRowKind().name() : null); return entity; } diff --git a/repository/src/main/java/org/apache/atlas/repository/purge/PurgeExecutionStats.java b/repository/src/main/java/org/apache/atlas/repository/purge/PurgeExecutionStats.java new file mode 100644 index 00000000000..0e90eb04cc4 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/purge/PurgeExecutionStats.java @@ -0,0 +1,328 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.purge; + +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.FailedEntity; +import org.apache.atlas.model.instance.PurgeSummary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Mutable run-context for purge execution accounting. Counters are updated sequentially + * during orchestration aggregation and reconciliation. + *

+ * This class is strictly single-thread-owned. Mutations must only happen on the main + * orchestrator thread after worker shutdown. + *

+ * Request-scope balance: {@code requestedCount = purgedCount + failedCount + skippedRequestedCount}. + * Expanded-scope balance uses {@link #getExpandedEntityCount()} and deduplicated outcome counters. + */ +public final class PurgeExecutionStats { + private static final Logger LOG = LoggerFactory.getLogger(PurgeExecutionStats.class); + + private final Set originallyRequestedGuids; + private final Set producedDeletionCandidates; + private final Set accountedPurgedGuids = new HashSet<>(); + private final Set accountedOutcomeGuids = new HashSet<>(); + + private long validGuidCount; + private long batchCount; + private long reconciledUnprocessedCount; + private long purgedCount; + private long purgedDependenciesCount; + private long failedCount; + private long failedDependenciesCount; + private long skippedRequestedCount; + private long skippedDependenciesCount; + private boolean executionFailed; + + public PurgeExecutionStats(Set originallyRequestedGuids, long validGuidCount) { + this.originallyRequestedGuids = originallyRequestedGuids; + this.validGuidCount = validGuidCount; + this.producedDeletionCandidates = new LinkedHashSet<>(); + } + + public Set getOriginallyRequestedGuids() { + return originallyRequestedGuids; + } + + public Set getProducedDeletionCandidates() { + return producedDeletionCandidates; + } + + public long getValidGuidCount() { + return validGuidCount; + } + + public long getExpandedEntityCount() { + return producedDeletionCandidates.size(); + } + + public long getSkippedCount() { + return skippedRequestedCount + skippedDependenciesCount; + } + + public long getReconciledUnprocessedCount() { + return reconciledUnprocessedCount; + } + + public long getPurgedCount() { + return purgedCount; + } + + public long getPurgedDependenciesCount() { + return purgedDependenciesCount; + } + + public long getFailedCount() { + return failedCount; + } + + public long getFailedDependenciesCount() { + return failedDependenciesCount; + } + + public long getSkippedRequestedCount() { + return skippedRequestedCount; + } + + public long getSkippedDependenciesCount() { + return skippedDependenciesCount; + } + + public long getBatchCount() { + return batchCount; + } + + public boolean isExecutionFailed() { + return executionFailed; + } + + public void markExecutionFailed() { + executionFailed = true; + } + + public void recordFailures(List failures) { + if (failures == null) { + return; + } + + for (FailedEntity failedEntity : failures) { + recordFailure(failedEntity); + } + } + + public void recordBatchOutcome(EntityMutationResponse batchResponse, Set batchGuids) { + if (batchGuids == null || batchGuids.isEmpty()) { + return; + } + + List purgedEntities = batchResponse != null ? batchResponse.getPurgedEntities() : null; + List failedEntities = batchResponse != null ? batchResponse.getFailedEntities() : null; + recordBatchOutcome(purgedEntities, failedEntities, batchGuids); + } + + public void recordBatchOutcome(List purgedEntities, List failedEntities, + Set batchGuids) { + if (batchGuids == null || batchGuids.isEmpty()) { + return; + } + + batchCount++; + + applyOutcomes(purgedEntities, failedEntities); + + validateBatchInvariant(batchGuids, purgedEntities, failedEntities); + } + + public void recordBatchThrowable(Set batchGuids, List batchFailures) { + if (batchGuids == null || batchGuids.isEmpty()) { + return; + } + + batchCount++; + + applyOutcomes(null, batchFailures); + + validateBatchInvariant(batchGuids, null, batchFailures); + } + + public void recordReconciledUnprocessed(long count) { + reconciledUnprocessedCount += count; + } + + void recordPurged(String guid) { + if (guid == null || !accountedPurgedGuids.add(guid)) { + return; + } + + if (accountedOutcomeGuids.remove(guid)) { + decrementOutcome(guid); + } + + accountedOutcomeGuids.add(guid); + + if (originallyRequestedGuids.contains(guid)) { + purgedCount++; + } else { + purgedDependenciesCount++; + } + } + + public void recordFailure(FailedEntity failedEntity) { + if (failedEntity == null) { + return; + } + + String guid = failedEntity.getGuid(); + String errorCode = failedEntity.getErrorCode(); + + if (guid == null || accountedPurgedGuids.contains(guid) || !accountedOutcomeGuids.add(guid)) { + return; + } + + boolean requested = originallyRequestedGuids.contains(guid); + + incrementFailureCounters(errorCode, requested); + + if (PurgeUtils.isExecutionFailureCode(errorCode)) { + executionFailed = true; + } + } + + private void incrementFailureCounters(String errorCode, boolean requested) { + if (PurgeUtils.isSkippablePurgeFailureCode(errorCode)) { + if (requested) { + skippedRequestedCount++; + } else { + skippedDependenciesCount++; + } + } else if (requested) { + failedCount++; + } else { + failedDependenciesCount++; + } + } + + /** + * Rebuilds stats by scanning a completed response. Used by tests and legacy callers that + * do not run through the worker pipeline. + */ + public static PurgeExecutionStats fromResponse(EntityMutationResponse response, + Set originallyRequestedGuids, + long validGuidCount) { + PurgeExecutionStats stats = new PurgeExecutionStats(originallyRequestedGuids, validGuidCount); + stats.applyOutcomes( + response != null ? response.getPurgedEntities() : null, + response != null ? response.getFailedEntities() : null); + + return stats; + } + + private void applyOutcomes(List purgedEntities, List failedEntities) { + if (purgedEntities != null) { + for (AtlasEntityHeader header : purgedEntities) { + recordPurged(header.getGuid()); + } + } + + if (failedEntities != null) { + for (FailedEntity failedEntity : failedEntities) { + recordFailure(failedEntity); + } + } + } + + public void validateSummaryBalances(PurgeSummary summary) { + long requestedBalance = summary.getPurgedCount() + summary.getFailedCount() + summary.getSkippedRequestedCount(); + if (summary.getRequestedCount() != requestedBalance) { + LOG.warn("Purge summary request-scope balance mismatch: requestedCount={}, purged+failed+skippedRequested={}", + summary.getRequestedCount(), requestedBalance); + } + + // unprocessedCount is a breakdown of reconciled shutdown failures already included in failedCount. + long expandedBalance = summary.getPurgedCount() + summary.getPurgedDependenciesCount() + + summary.getFailedCount() + summary.getFailedDependenciesCount() + + summary.getSkippedRequestedCount() + summary.getSkippedDependenciesCount(); + if (summary.getExpandedEntityCount() > 0 && summary.getExpandedEntityCount() != expandedBalance) { + LOG.warn("Purge summary expanded-scope balance mismatch: expandedEntityCount={}, outcomeTotal={}", + summary.getExpandedEntityCount(), expandedBalance); + } + } + + private void decrementOutcome(String guid) { + boolean requested = originallyRequestedGuids.contains(guid); + + if (requested) { + if (purgedCount > 0) { + purgedCount--; + return; + } + if (failedCount > 0) { + failedCount--; + return; + } + if (skippedRequestedCount > 0) { + skippedRequestedCount--; + } + } else { + if (purgedDependenciesCount > 0) { + purgedDependenciesCount--; + return; + } + if (failedDependenciesCount > 0) { + failedDependenciesCount--; + return; + } + if (skippedDependenciesCount > 0) { + skippedDependenciesCount--; + } + } + } + + private static void validateBatchInvariant(Set batchGuids, List purgedEntities, + List failedEntities) { + int batchInputCount = batchGuids.size(); + int batchPurged = purgedEntities != null ? purgedEntities.size() : 0; + int batchFailed = 0; + int batchSkipped = 0; + + if (failedEntities != null) { + for (FailedEntity failedEntity : failedEntities) { + if (PurgeUtils.isSkippablePurgeFailureCode(failedEntity.getErrorCode())) { + batchSkipped++; + } else { + batchFailed++; + } + } + } + + int batchAccounted = batchPurged + batchFailed + batchSkipped; + if (batchAccounted != batchInputCount) { + LOG.warn("Purge batch accounting mismatch: batchSize={}, purged={}, failed={}, skipped={}", + batchInputCount, batchPurged, batchFailed, batchSkipped); + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/purge/PurgeUtils.java b/repository/src/main/java/org/apache/atlas/repository/purge/PurgeUtils.java new file mode 100644 index 00000000000..0f27962b3c1 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/purge/PurgeUtils.java @@ -0,0 +1,351 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.purge; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.AtlasAuditEntry; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditRowKind; +import org.apache.atlas.model.audit.AuditSearchParameters; +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.instance.AtlasEntity.Status; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.FailedEntity; +import org.apache.atlas.model.instance.PurgeSummary; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.ogm.AtlasAuditEntryDTO; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasJson; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Shared purge policy helpers, summary attachment, and purge audit read helpers. + */ +public final class PurgeUtils { + private static final Logger LOG = LoggerFactory.getLogger(PurgeUtils.class); + + enum FailureCategory { + SKIPPABLE, + PRE_VALIDATION_FAILURE, + EXECUTION_FAILURE + } + + private PurgeUtils() { + } + + public static boolean isSkippablePurgeFailureCode(String errorCode) { + return classifyPurgeFailureCode(errorCode) == FailureCategory.SKIPPABLE; + } + + public static boolean isExecutionFailureCode(String errorCode) { + return classifyPurgeFailureCode(errorCode) == FailureCategory.EXECUTION_FAILURE; + } + + static FailureCategory classifyPurgeFailureCode(String errorCode) { + if (AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode().equals(errorCode) + || AtlasErrorCode.NOT_IN_DELETED_STATE.getErrorCode().equals(errorCode)) { + return FailureCategory.SKIPPABLE; + } + if (AtlasErrorCode.INVALID_GUID.getErrorCode().equals(errorCode) + || AtlasErrorCode.TYPE_NAME_NOT_FOUND.getErrorCode().equals(errorCode)) { + return FailureCategory.PRE_VALIDATION_FAILURE; + } + return FailureCategory.EXECUTION_FAILURE; + } + + public static PurgeSummary buildPurgeSummary(PurgeExecutionStats stats, String runId) { + PurgeSummary summary = new PurgeSummary( + stats.getOriginallyRequestedGuids().size(), + stats.getPurgedCount(), + stats.getPurgedDependenciesCount(), + stats.getFailedCount(), + stats.getFailedDependenciesCount(), + stats.getSkippedCount()); + summary.setValidGuidCount(stats.getValidGuidCount()); + summary.setExpandedEntityCount(stats.getExpandedEntityCount()); + summary.setBatchCount(stats.getBatchCount()); + summary.setSkippedRequestedCount(stats.getSkippedRequestedCount()); + summary.setSkippedDependenciesCount(stats.getSkippedDependenciesCount()); + summary.setUnprocessedCount(stats.getReconciledUnprocessedCount()); + summary.setExecutionFailed(stats.isExecutionFailed()); + summary.setRunId(runId); + return summary; + } + + public static void preScanGuids(Set guids, Set validGuids, List failedEntities, + AtlasGraph graph, AtlasTypeRegistry typeRegistry) { + if (guids == null) { + return; + } + + for (String guid : guids) { + FailedEntity failure = validatePurgePreconditions(guid, graph, typeRegistry); + if (failure != null) { + failedEntities.add(failure); + } else { + validGuids.add(guid); + } + } + } + + public static FailedEntity validatePurgePreconditions(String guid, AtlasGraph graph, + AtlasTypeRegistry typeRegistry) { + if (!isValidUuid(guid)) { + LOG.debug("Purge request ignored for invalid GUID format: {}", guid); + return createPreScanFailure(guid, AtlasErrorCode.INVALID_GUID, guid); + } + + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(graph, guid); + + if (vertex == null) { + LOG.debug("Purge request ignored for non-existent entity: guid={}", guid); + return createPreScanFailure(guid, AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); + } + + if (AtlasGraphUtilsV2.getState(vertex) != Status.DELETED) { + LOG.debug("Purge request ignored for entity not in DELETED state: guid={}", guid); + return createPreScanFailure(guid, AtlasErrorCode.NOT_IN_DELETED_STATE, guid); + } + + String typeName = AtlasGraphUtilsV2.getTypeName(vertex); + if (StringUtils.isBlank(typeName)) { + LOG.debug("Purge request ignored for entity with missing type name: guid={}", guid); + return createPreScanFailure(guid, AtlasErrorCode.TYPE_NAME_NOT_FOUND, "null"); + } + if (!typeRegistry.isRegisteredType(typeName)) { + LOG.debug("Purge request ignored for entity with unregistered type: guid={}, typeName={}", guid, typeName); + return createPreScanFailure(guid, AtlasErrorCode.TYPE_NAME_NOT_FOUND, typeName); + } + + return null; + } + + public static FailedEntity createPreScanFailure(String guid, AtlasErrorCode errorCode, String... messageArgs) { + return new FailedEntity(guid, + errorCode.getErrorCode(), + errorCode.getFormattedErrorMessage(messageArgs)); + } + + public static boolean isValidUuid(String guid) { + if (guid == null) { + return false; + } + try { + java.util.UUID.fromString(guid); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + public static FailedEntity classifyPurgeFailure(String guid, Throwable error, String defaultMessage) { + if (error instanceof AtlasBaseException) { + AtlasBaseException atlasError = (AtlasBaseException) error; + return new FailedEntity(guid, atlasError.getAtlasErrorCode().getErrorCode(), + error.getMessage()); + } + + String message = defaultMessage != null ? defaultMessage : (error != null ? error.getMessage() : null); + return new FailedEntity(guid, AtlasErrorCode.INTERNAL_ERROR.getErrorCode(), message); + } + + public static void attachPurgeSummary(EntityMutationResponse response, PurgeExecutionStats stats, String runId) { + if (response == null || stats == null) { + return; + } + + PurgeSummary summary = buildPurgeSummary(stats, runId); + stats.validateSummaryBalances(summary); + response.setSummary(summary); + } + + public static String buildGuidParams(Collection guids) { + return guids.stream() + .filter(Objects::nonNull) + .sorted() + .collect(Collectors.joining(",")); + } + + public static boolean isPurgeSummaryAudit(AtlasAuditEntry entry) { + return entry != null && AuditRowKind.SUMMARY == entry.getAuditRowKind(); + } + + public static boolean isPurgeBatchAudit(AtlasAuditEntry entry) { + return entry != null && AuditRowKind.BATCH == entry.getAuditRowKind(); + } + + public static PurgeSummary parsePurgeSummary(AtlasAuditEntry entry) { + if (entry == null || StringUtils.isBlank(entry.getResult())) { + return null; + } + + String result = entry.getResult().trim(); + if (!result.startsWith("{")) { + return null; + } + + try { + PurgeSummary summary = AtlasJson.fromJson(result, PurgeSummary.class); + return hasPurgeSummary(summary) ? summary : null; + } catch (Exception e) { + return null; + } + } + + private static boolean hasPurgeSummary(PurgeSummary summary) { + if (summary == null) { + return false; + } + + return StringUtils.isNotBlank(summary.getRunId()) + || summary.getRequestedCount() > 0 + || summary.getExpandedEntityCount() > 0 + || summary.getBatchCount() > 0 + || summary.getValidGuidCount() > 0; + } + + public static String resolveRunId(AtlasAuditEntry auditEntry) { + if (auditEntry == null || StringUtils.isBlank(auditEntry.getRunId())) { + return null; + } + + return auditEntry.getRunId(); + } + + private static boolean isLegacyPurgeAudit(AtlasAuditEntry entry) { + return entry != null + && entry.getAuditRowKind() == null + && (entry.getOperation() == AuditOperation.PURGE || entry.getOperation() == AuditOperation.AUTO_PURGE); + } + + public static boolean isCorrelatedPurgeAudit(AtlasAuditEntry entry) { + return isPurgeSummaryAudit(entry) || isPurgeBatchAudit(entry) || isLegacyPurgeAudit(entry); + } + + public static List excludeBatchRowsFromResults(List entries) { + List ret = new ArrayList<>(); + + if (entries == null) { + return ret; + } + + for (AtlasAuditEntry entry : entries) { + if (!isPurgeBatchAudit(entry)) { + ret.add(entry); + } + } + + return ret; + } + + public static void excludeBatchRowsFromAuditSearch(AuditSearchParameters auditSearchParameters) { + if (auditSearchParameters == null) { + return; + } + + SearchParameters.FilterCriteria excludeBatch = new SearchParameters.FilterCriteria(); + excludeBatch.setAttributeName(AtlasAuditEntryDTO.ATTRIBUTE_AUDIT_ROW_KIND); + excludeBatch.setOperator(SearchParameters.Operator.NEQ); + excludeBatch.setAttributeValue(AuditRowKind.BATCH.name()); + + SearchParameters.FilterCriteria originalFilters = auditSearchParameters.getAuditFilters(); + SearchParameters.FilterCriteria newFilters = new SearchParameters.FilterCriteria(); + newFilters.setCondition(SearchParameters.FilterCriteria.Condition.AND); + newFilters.setCriterion(new ArrayList<>()); + + if (originalFilters != null) { + newFilters.getCriterion().add(originalFilters); + } + + newFilters.getCriterion().add(excludeBatch); + auditSearchParameters.setAuditFilters(newFilters); + } + + public static List paginateStringList(List values, int limit, int offset) { + if (values == null || values.isEmpty()) { + return new ArrayList<>(); + } + + int from = Math.min(Math.max(offset, 0), values.size()); + int to = Math.min(from + Math.max(limit, 0), values.size()); + return new ArrayList<>(values.subList(from, to)); + } + + public static boolean hasRunIdFilter(SearchParameters.FilterCriteria auditFilters) { + if (auditFilters == null) { + return false; + } + + if (AtlasAuditEntryDTO.ATTRIBUTE_RUN_ID.equals(auditFilters.getAttributeName()) + && SearchParameters.Operator.EQ.equals(auditFilters.getOperator()) + && StringUtils.isNotEmpty(auditFilters.getAttributeValue())) { + return true; + } + + if (auditFilters.getCriterion() != null) { + for (SearchParameters.FilterCriteria each : auditFilters.getCriterion()) { + if (hasRunIdFilter(each)) { + return true; + } + } + } + + return false; + } + + public static List collectPurgedGuidsFromBatchEntries(List batchRows) { + List orderedEntityGuids = new ArrayList<>(); + Set seen = new LinkedHashSet<>(); + + if (batchRows == null) { + return orderedEntityGuids; + } + + for (AtlasAuditEntry batchRow : batchRows) { + appendPurgedGuidsFromBatchResult(batchRow.getResult(), orderedEntityGuids, seen); + } + + return orderedEntityGuids; + } + + private static void appendPurgedGuidsFromBatchResult(String result, List orderedEntityGuids, Set seen) { + if (StringUtils.isBlank(result)) { + return; + } + + for (String guid : result.split(",")) { + guid = guid.trim(); + if (StringUtils.isNotBlank(guid) && seen.add(guid)) { + orderedEntityGuids.add(guid); + } + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index 6f13175cee0..7454c83b26f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -28,7 +28,6 @@ import org.apache.atlas.model.instance.AtlasEntityHeaders; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.EntityMutationResponse; -import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.EntityStream; import org.apache.atlas.type.AtlasEntityType; @@ -225,14 +224,17 @@ EntityMutationResponse deleteByUniqueAttributes(AtlasEntityType entityType, Map< /* * Return list of purged entity guids */ - EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseException; + EntityMutationResponse purgeEntitiesInBatch(Set deletedVertices) throws AtlasBaseException; /* - * Returns set of auto-purged entity guids + * Resolves purge/delete dependency candidates for the given entity GUIDs. + * + * INCOMPATIBLE CHANGE (ATLAS-5317): return type changed from Set to + * Set. The method was added in ATLAS-4920; ATLAS-5317 returns GUIDs so + * worker-batch purge can enqueue candidates without leaking transaction-bound vertices. + * Custom AtlasEntityStore implementations must match this signature. */ - EntityMutationResponse purgeEntitiesInBatch(Set deletedVertices) throws AtlasBaseException; - - Set accumulateDeletionCandidates(Set vertices) throws AtlasBaseException; + Set accumulateDeletionCandidates(Set vertices) throws AtlasBaseException; /** * Add classification(s) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index 5041d287474..4477a3099bf 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -221,7 +221,8 @@ public Set accumulateDeletionCandidates(Collection ins /* actually delete traits and then the vertex along its references */ - public void deleteTraitsAndVertices(Collection deletionCandidateVertices) throws AtlasBaseException { + public Collection deleteTraitsAndVertices(Collection deletionCandidateVertices) throws AtlasBaseException { + Collection deletedVertices = new ArrayList<>(); for (AtlasVertex deletionCandidateVertex : deletionCandidateVertices) { if (deletionCandidateVertex == null) { continue; @@ -235,10 +236,12 @@ public void deleteTraitsAndVertices(Collection deletionCandidateVer try { deleteAllClassifications(deletionCandidateVertex); deleteTypeVertex(deletionCandidateVertex, isInternalType(deletionCandidateVertex)); + deletedVertices.add(deletionCandidateVertex); } catch (IllegalStateException e) { LOG.warn("deleteTraitsAndVertices(): skipping vertex - already removed", e); } } + return deletedVertices; } public void addUpstreamProcessEntities(AtlasVertex entityVertex, Set deletionCandidateVertices, Set instanceVertexGuids) throws AtlasBaseException { @@ -1157,7 +1160,7 @@ protected void deleteEdgeBetweenVertices(AtlasVertex outVertex, AtlasVertex inVe LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex), attribute.getName()); } - if (skipVertexForDelete(outVertex)) { + if (skipVertexForDelete(outVertex) || isDeletedEntity(outVertex)) { return; } @@ -1433,13 +1436,10 @@ private boolean skipVertexForDelete(AtlasVertex vertex) { final RequestContext reqContext = RequestContext.get(); final String guid = AtlasGraphUtilsV2.getIdFromVertex(vertex); - if (guid != null && !reqContext.isDeletedEntity(guid)) { - final AtlasEntity.Status vertexState = getState(vertex); - if (reqContext.isPurgeRequested()) { - ret = vertexState == ACTIVE; // skip purging ACTIVE vertices - } else { - ret = vertexState == DELETED; // skip deleting DELETED vertices - } + if (reqContext.isPurgeRequested()) { + ret = getState(vertex) == ACTIVE; // skip purging ACTIVE vertices + } else if (guid != null && !reqContext.isDeletedEntity(guid)) { + ret = getState(vertex) == DELETED; // skip deleting DELETED vertices } } catch (IllegalStateException excp) { LOG.warn("skipVertexForDelete(): failed guid/state for the vertex", excp); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 578c1163283..e7959c3b4b9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -23,7 +23,6 @@ import org.apache.atlas.GraphTransactionInterceptor; import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.GraphTransaction; -import org.apache.atlas.authorize.AtlasAdminAccessRequest; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder; @@ -43,6 +42,7 @@ import org.apache.atlas.model.instance.AtlasEntityHeaders; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.FailedEntity; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; @@ -81,6 +81,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -546,80 +548,85 @@ public EntityMutationResponse deleteByIds(final List guids) throws Atlas return ret; } - @Override - @GraphTransaction - public EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseException { - if (CollectionUtils.isEmpty(guids)) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Guid(s) not specified"); - } - - AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_PURGE), "purge entity: guids=", guids); - - Collection purgeCandidates = new ArrayList<>(); - - for (String guid : guids) { - AtlasVertex vertex = AtlasGraphUtilsV2.findDeletedByGuid(graph, guid); - - if (vertex == null) { - // Entity does not exist - treat as non-error, since the caller - // wanted to delete the entity and it's already gone. - LOG.warn("Purge request ignored for non-existent/active entity with guid {}", guid); - - continue; - } - - purgeCandidates.add(vertex); - } - - if (purgeCandidates.isEmpty()) { - LOG.info("No purge candidate entities were found for guids: {} which is already deleted", guids); - } - - EntityMutationResponse ret = purgeVertices(purgeCandidates); - - // Notify the change listeners - entityChangeNotifier.onEntitiesMutated(ret, false); - - return ret; - } - @Override @GraphTransaction public EntityMutationResponse purgeEntitiesInBatch(Set purgeCandidates) throws AtlasBaseException { - LOG.info("==> purgeEntitiesInBatch()"); + LOG.debug("purgeEntitiesInBatch: batchSize={}", purgeCandidates.size()); Collection purgeVertices = new ArrayList<>(); - EntityMutationResponse response = new EntityMutationResponse(); + EntityMutationResponse response = new EntityMutationResponse(); + Map prePurgeHeaders = new IdentityHashMap<>(); RequestContext requestContext = RequestContext.get(); requestContext.setDeleteType(DeleteType.HARD); // hard deleter requestContext.setPurgeRequested(true); + if (CollectionUtils.isNotEmpty(purgeCandidates)) { + GraphTransactionInterceptor.lockObjectAndReleasePostCommit(new ArrayList<>(purgeCandidates)); + GraphTransactionInterceptor.clearCache(); + } for (String guid : purgeCandidates) { AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(graph, guid); - if (vertex != null) { + if (vertex == null || !vertex.exists()) { + LOG.debug("Purge batch skipped guid={} as vertex was already removed (likely by concurrent batch)", guid); + addPurgeBatchFailure(response, guid, AtlasErrorCode.INSTANCE_GUID_NOT_FOUND); + continue; + } + + try { + if (AtlasGraphUtilsV2.getState(vertex) != Status.DELETED) { + LOG.warn("Purge batch skipped guid={} as it is no longer in DELETED state", guid); + addPurgeBatchFailure(response, guid, AtlasErrorCode.NOT_IN_DELETED_STATE); + continue; + } + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(vertex); purgeVertices.add(vertex); - response.addEntity(PURGE, entityHeader); + prePurgeHeaders.put(vertex, entityHeader); + } catch (IllegalStateException e) { + LOG.debug("Purge batch skipped guid={} as vertex was already removed (likely by concurrent batch)", guid, e); + addPurgeBatchFailure(response, guid, AtlasErrorCode.INSTANCE_GUID_NOT_FOUND); } } - deleteDelegate.getHandler().deleteTraitsAndVertices(purgeVertices); + Collection deletedVertices = deleteDelegate.getHandler().deleteTraitsAndVertices(purgeVertices); + Set deletedVertexSet = Collections.newSetFromMap(new IdentityHashMap<>()); + deletedVertexSet.addAll(deletedVertices); - entityChangeNotifier.onEntitiesMutated(response, false); + EntityMutationResponse notificationResponse = new EntityMutationResponse(); - for (AtlasEntityHeader entity : response.getPurgedEntities()) { - LOG.info("Auto purged entity with guid {}", entity.getGuid()); + for (Map.Entry entry : prePurgeHeaders.entrySet()) { + if (deletedVertexSet.contains(entry.getKey())) { + response.addEntity(PURGE, entry.getValue()); + notificationResponse.addEntity(PURGE, entry.getValue()); + } else { + String guid = entry.getValue().getGuid(); + LOG.debug("Purge batch skipped guid={} as vertex was not deleted by this batch, assuming concurrently removed", guid); + addPurgeBatchFailure(response, guid, AtlasErrorCode.INSTANCE_GUID_NOT_FOUND); + } } - LOG.info("<== purgeEntitiesInBatch()"); + // Notify listeners only for vertices hard-deleted in this batch. GUIDs removed by a concurrent + // batch (during pre-check or delete) are already purged, so do not trigger notifications again. + entityChangeNotifier.onEntitiesMutated(notificationResponse, false); + + if (CollectionUtils.isNotEmpty(response.getPurgedEntities())) { + LOG.debug("purgeEntitiesInBatch: purged {} entity(ies)", response.getPurgedEntities().size()); + } return response; } + private void addPurgeBatchFailure(EntityMutationResponse response, String guid, AtlasErrorCode errorCode) { + if (response != null) { + response.addFailedEntity(new FailedEntity(guid, errorCode.getErrorCode(), errorCode.getFormattedErrorMessage(guid))); + } + } + @Override - public Set accumulateDeletionCandidates(Set guids) throws AtlasBaseException { - LOG.info("==> accumulateDeletionCandidates() !"); + @GraphTransaction + public Set accumulateDeletionCandidates(Set guids) throws AtlasBaseException { + LOG.debug("accumulateDeletionCandidates: guidCount={}", guids.size()); Set vertices = new HashSet<>(); for (String guid : guids) { @@ -627,7 +634,18 @@ public Set accumulateDeletionCandidates(Set guids) throws A vertices.add(vertex); } - return deleteDelegate.getHandler().accumulateDeletionCandidates(vertices); + Set deletionCandidates = deleteDelegate.getHandler().accumulateDeletionCandidates(vertices); + Set candidateGuids = new LinkedHashSet<>(); + + for (AtlasVertex vertex : deletionCandidates) { + String candidateGuid = AtlasGraphUtilsV2.getIdFromVertex(vertex); + + if (candidateGuid != null) { + candidateGuids.add(candidateGuid); + } + } + + return candidateGuids; } @Override @@ -1404,21 +1422,6 @@ private EntityMutationResponse deleteVertices(Collection deletionCa return response; } - private EntityMutationResponse purgeVertices(Collection purgeCandidates) throws AtlasBaseException { - EntityMutationResponse response = new EntityMutationResponse(); - RequestContext req = RequestContext.get(); - - req.setDeleteType(DeleteType.HARD); - req.setPurgeRequested(true); - deleteDelegate.getHandler().deleteEntities(purgeCandidates); // this will update req with list of purged entities - - for (AtlasEntityHeader entity : req.getDeletedEntities()) { - response.addEntity(PURGE, entity); - } - - return response; - } - private void validateAndNormalize(AtlasClassification classification) throws AtlasBaseException { AtlasClassificationType type = typeRegistry.getClassificationTypeByName(classification.getTypeName()); diff --git a/repository/src/main/java/org/apache/atlas/services/PurgeAuditWriter.java b/repository/src/main/java/org/apache/atlas/services/PurgeAuditWriter.java new file mode 100644 index 00000000000..238598f2f3f --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/services/PurgeAuditWriter.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.services; + +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditRowKind; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.FailedEntity; +import org.apache.atlas.model.instance.PurgeSummary; +import org.apache.atlas.repository.audit.AtlasAuditService; +import org.apache.atlas.repository.purge.PurgeExecutionStats; +import org.apache.atlas.repository.purge.PurgeUtils; +import org.apache.atlas.utils.AtlasJson; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Purge audit write path: batch/summary graph writes and purgefailure.log emission. + */ +public final class PurgeAuditWriter { + private static final Logger LOG = LoggerFactory.getLogger(PurgeAuditWriter.class); + private static final Logger PURGE_FAILURE_LOG = LoggerFactory.getLogger("PURGE_FAILURE"); + + private PurgeAuditWriter() { + } + + public static void logFailures(String runId, AuditOperation operation, List failedEntities) { + if (failedEntities == null) { + return; + } + + for (FailedEntity failedEntity : failedEntities) { + PURGE_FAILURE_LOG.error("[PURGE_FAILURE] runId={} op={} guid={} code={} msg={}", + runId, operation, failedEntity.getGuid(), failedEntity.getErrorCode(), failedEntity.getErrorMessage()); + } + } + + public static void writeBatch(AtlasAuditService auditService, AuditOperation operation, String runId, + Set batchInputGuids, EntityMutationResponse batchResponse) { + if (batchResponse == null || CollectionUtils.isEmpty(batchInputGuids)) { + return; + } + + logFailures(runId, operation, batchResponse.getFailedEntities()); + + if (auditService == null || operation == null) { + return; + } + + String params = PurgeUtils.buildGuidParams(batchInputGuids); + + List purgedEntities = batchResponse.getPurgedEntities(); + String result; + long resultCount; + + if (CollectionUtils.isEmpty(purgedEntities)) { + result = ""; + resultCount = 0; + } else { + result = PurgeUtils.buildGuidParams(purgedEntities.stream() + .map(AtlasEntityHeader::getGuid) + .collect(Collectors.toList())); + resultCount = purgedEntities.size(); + } + + try { + auditService.add(operation, params, result, resultCount, runId, AuditRowKind.BATCH); + } catch (Exception e) { + LOG.warn("Failed to write purge batch audit entry", e); + } + } + + public static void finishRun(AtlasAuditService auditService, AuditOperation operation, String runId, + Set originallyRequestedGuids, PurgeExecutionStats stats) { + if (stats == null || CollectionUtils.isEmpty(originallyRequestedGuids)) { + return; + } + + writeSummary(auditService, operation, runId, originallyRequestedGuids, stats); + } + + private static void writeSummary(AtlasAuditService auditService, AuditOperation operation, String runId, + Set originallyRequestedGuids, PurgeExecutionStats stats) { + if (auditService == null || operation == null) { + return; + } + + PurgeSummary summary = PurgeUtils.buildPurgeSummary(stats, runId); + String params = PurgeUtils.buildGuidParams(originallyRequestedGuids); + String result = AtlasJson.toJson(summary); + long resultCount = summary.getPurgedCount(); + + try { + auditService.add(operation, params, result, resultCount, runId, AuditRowKind.SUMMARY); + } catch (Exception e) { + LOG.warn("Failed to write purge summary audit entry", e); + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/services/PurgeBatchExecutor.java b/repository/src/main/java/org/apache/atlas/services/PurgeBatchExecutor.java new file mode 100644 index 00000000000..706fd413d96 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/services/PurgeBatchExecutor.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.services; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.GraphTransactionInterceptor; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.Callable; + +public class PurgeBatchExecutor { + private static final Logger LOG = LoggerFactory.getLogger(PurgeBatchExecutor.class); + + private static final int MAX_RETRIES = 3; + private static final int BASE_BACKOFF_MS = 500; + + /** + * Fully-qualified class names treated as retryable lock or backend conflicts during purge batch + * execution. Names are matched against the throwable cause chain to avoid a compile-time dependency + * on JanusGraph or Berkeley JE types in the service layer. + *

+ * Design default: {@code PermanentLockingException}. Berkeley JE lock timeouts/deadlocks and + * {@code PermanentBackendException} are included for the embedded Berkeley backend. + */ + static final Set RETRYABLE_LOCK_CONFLICT_EXCEPTION_CLASS_NAMES = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList( + "org.janusgraph.diskstorage.locking.PermanentLockingException", + "com.sleepycat.je.LockTimeoutException", + "com.sleepycat.je.DeadlockException", + "org.janusgraph.diskstorage.PermanentBackendException"))); + + private final AtlasEntityStore entityStore; + + public PurgeBatchExecutor(AtlasEntityStore entityStore) { + this.entityStore = entityStore; + } + + public AtlasEntityStore getEntityStore() { + return entityStore; + } + + public EntityMutationResponse executeBatch(Set batch) throws AtlasBaseException { + return withRetry(() -> entityStore.purgeEntitiesInBatch(batch)); + } + + /** + * Returns {@code true} when {@code throwable} or any of its causes matches a known retryable + * lock or backend conflict type. + */ + static boolean isRetryableLockConflict(Throwable throwable) { + if (throwable == null) { + return false; + } + + for (Throwable c = throwable; c != null; c = c.getCause()) { + if (RETRYABLE_LOCK_CONFLICT_EXCEPTION_CLASS_NAMES.contains(c.getClass().getName())) { + return true; + } + } + + return false; + } + + private T withRetry(Callable action) throws AtlasBaseException { + int attempt = 0; + + while (true) { + try { + return action.call(); + } catch (Throwable e) { + boolean canRetry = isRetryableLockConflict(e) && attempt < (MAX_RETRIES - 1); + if (canRetry) { + GraphTransactionInterceptor.clearCache(); + RequestContext.get().clearCache(); + + long backoff = (long) BASE_BACKOFF_MS * (attempt + 1); + LOG.warn("Lock conflict for purge batch on attempt {}/{}, backing off {} ms", + attempt + 1, MAX_RETRIES, backoff); + try { + Thread.sleep(backoff); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + attempt++; + continue; + } + + LOG.error("Failed to process purge batch on attempt {}/{}", attempt + 1, MAX_RETRIES, e); + if (e instanceof AtlasBaseException) { + throw (AtlasBaseException) e; + } + throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e); + } + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/services/PurgeBatchOrchestrator.java b/repository/src/main/java/org/apache/atlas/services/PurgeBatchOrchestrator.java new file mode 100644 index 00000000000..7e02a75b63a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/services/PurgeBatchOrchestrator.java @@ -0,0 +1,479 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.services; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.DeleteType; +import org.apache.atlas.GraphTransactionInterceptor; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.EntityMutations.EntityOperation; +import org.apache.atlas.model.instance.FailedEntity; +import org.apache.atlas.pc.WorkItemBuilder; +import org.apache.atlas.pc.WorkItemConsumer; +import org.apache.atlas.pc.WorkItemManager; +import org.apache.atlas.repository.audit.AtlasAuditService; +import org.apache.atlas.repository.purge.PurgeExecutionStats; +import org.apache.atlas.repository.purge.PurgeUtils; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.commons.configuration2.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.BlockingQueue; + +public class PurgeBatchOrchestrator { + private static final Logger LOG = LoggerFactory.getLogger(PurgeBatchOrchestrator.class); + public static final String PURGE_WORKERS_NAME = "Entity-Purge-Worker"; + private static final String PURGE_WORKER_BATCH_SIZE_KEY = "atlas.purge.worker.batch.size"; + private static final String PURGE_WORKERS_COUNT_KEY = "atlas.purge.workers.count"; + private static final int DEFAULT_PURGE_WORKER_BATCH_SIZE = 100; + private static final int DEFAULT_PURGE_WORKERS_COUNT = 2; + static final String UNPROCESSED_PURGE_GUID_MESSAGE = "Not processed during purge shutdown"; + + private final PurgeBatchExecutor purgeBatchExecutor; + private final AtlasAuditService auditService; + private final AuditOperation auditOperation; + + public PurgeBatchOrchestrator(PurgeBatchExecutor purgeBatchExecutor, AtlasAuditService auditService, + AuditOperation auditOperation) { + this.purgeBatchExecutor = purgeBatchExecutor; + this.auditService = auditService; + this.auditOperation = auditOperation; + } + + public EntityMutationResponse executePurge(Set validGuids, List failedEntities, + PurgeExecutionStats stats, String runId) throws AtlasBaseException { + if (failedEntities == null) { + failedEntities = new ArrayList<>(); + } + + Configuration configuration = null; + try { + configuration = ApplicationProperties.get(); + } catch (Exception e) { + LOG.warn("executePurge: failed to load application properties, using defaults", e); + } + int batchSize = getWorkerBatchSize(configuration); + int numWorkers = getWorkersCount(configuration); + + Set producedDeletionCandidates = stats != null + ? stats.getProducedDeletionCandidates() : new LinkedHashSet<>(); + + WorkItemManager manager = createManager(batchSize, numWorkers, runId); + + LOG.info("executePurge: starting iterative expand+WIM purge with batchSize={}, numWorkers={}, validGuids={}", + batchSize, numWorkers, validGuids.size()); + + EntityMutationResponse response = new EntityMutationResponse(); + try { + for (String validGuid : validGuids) { + if (producedDeletionCandidates.contains(validGuid)) { + continue; + } + + try { + expandDeletionCandidatesAndProduce(validGuid, producedDeletionCandidates, manager, purgeBatchExecutor.getEntityStore()); + } catch (Exception e) { + LOG.warn("executePurge: failed to accumulate deletion candidates for guid={}", validGuid, e); + FailedEntity failedEntity = PurgeUtils.classifyPurgeFailure(validGuid, e, null); + failedEntities.add(failedEntity); + } finally { + clearPurgeCandidateExpansionState(); + } + } + + LOG.info("executePurge: produced {} guid(s) from {} valid request guid(s)", + producedDeletionCandidates.size(), validGuids.size()); + } finally { + try { + manager.shutdown(); + } catch (InterruptedException e) { + LOG.warn("executePurge: purge worker shutdown interrupted; collecting partial results", e); + Thread.currentThread().interrupt(); + } + + aggregateBatchResults(manager.getResults(), response, stats, failedEntities); + reconcileUnprocessedGuids(producedDeletionCandidates, response, null, stats); + } + + return response; + } + + public static int getWorkerBatchSize(Configuration configuration) { + return configuration != null + ? configuration.getInt(PURGE_WORKER_BATCH_SIZE_KEY, DEFAULT_PURGE_WORKER_BATCH_SIZE) + : DEFAULT_PURGE_WORKER_BATCH_SIZE; + } + + public static int getWorkersCount(Configuration configuration) { + return configuration != null + ? configuration.getInt(PURGE_WORKERS_COUNT_KEY, DEFAULT_PURGE_WORKERS_COUNT) + : DEFAULT_PURGE_WORKERS_COUNT; + } + + public static void expandDeletionCandidatesAndProduce(String rootGuid, Set producedDeletionCandidates, + WorkItemManager manager, + AtlasEntityStore entityStore) throws AtlasBaseException { + Set deletionCandidateGuids = entityStore.accumulateDeletionCandidates(Collections.singleton(rootGuid)); + + for (String guid : deletionCandidateGuids) { + if (guid != null && producedDeletionCandidates.add(guid)) { + manager.checkProduce(guid); + } + } + + if (producedDeletionCandidates.add(rootGuid)) { + manager.checkProduce(rootGuid); + } + } + + /** + * Clears graph and RequestContext caches after one root GUID is expanded and enqueued, + * so the next expansion does not reuse stale vertex handles. + */ + public static void clearPurgeCandidateExpansionState() { + GraphTransactionInterceptor.clearCache(); + RequestContext.get().clearCache(); + initializePurgeContext(); + } + + public static void initializePurgeContext() { + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + } + + public static void resetPurgeContext() { + GraphTransactionInterceptor.clearCache(); + RequestContext.get().clearCache(); + RequestContext.get().setDeleteType(DeleteType.DEFAULT); + RequestContext.get().setPurgeRequested(false); + } + + public static int aggregateBatchResults(Queue results, EntityMutationResponse target, + PurgeExecutionStats stats, List expansionFailures) { + if (expansionFailures != null) { + for (FailedEntity failedEntity : expansionFailures) { + if (stats != null) { + stats.recordFailure(failedEntity); + } + target.addFailedEntity(failedEntity); + } + } + + if (results == null) { + LOG.warn("Purge worker results queue was not initialized"); + return expansionFailures != null ? expansionFailures.size() : 0; + } + + int resultCount = results.size(); + + while (!results.isEmpty()) { + Object res = results.poll(); + + if (res instanceof PurgeBatchResult) { + PurgeBatchResult batchResult = (PurgeBatchResult) res; + + if (stats != null) { + if (batchResult.hasBatchException()) { + stats.recordBatchThrowable(batchResult.getBatchGuids(), batchResult.getFailedEntities()); + } else { + stats.recordBatchOutcome(batchResult.getPurgedEntities(), batchResult.getFailedEntities(), + batchResult.getBatchGuids()); + } + } + + for (AtlasEntityHeader header : batchResult.getPurgedEntities()) { + target.addEntity(EntityOperation.PURGE, header); + } + + for (FailedEntity failedEntity : batchResult.getFailedEntities()) { + target.addFailedEntity(failedEntity); + } + } else { + LOG.warn("Unexpected result type in purge worker queue: {}", res != null ? res.getClass().getName() : "null"); + } + } + + return resultCount + (expansionFailures != null ? expansionFailures.size() : 0); + } + + public static int reconcileUnprocessedGuids(Collection submittedGuids, EntityMutationResponse response, + List pendingFailures, PurgeExecutionStats stats) { + if (submittedGuids == null || submittedGuids.isEmpty()) { + return 0; + } + + Set accountedGuids = collectAccountedGuids(response, pendingFailures); + int reconciledCount = 0; + + for (String guid : submittedGuids) { + if (guid == null || accountedGuids.contains(guid)) { + continue; + } + + FailedEntity failedEntity = PurgeUtils.classifyPurgeFailure(guid, null, UNPROCESSED_PURGE_GUID_MESSAGE); + + if (pendingFailures != null) { + pendingFailures.add(failedEntity); + } else if (response != null) { + response.addFailedEntity(failedEntity); + } + + if (stats != null) { + stats.recordReconciledUnprocessed(1); + stats.recordFailure(failedEntity); + } + + accountedGuids.add(guid); + reconciledCount++; + } + + if (reconciledCount > 0) { + LOG.warn("Purge reconciliation marked {} unprocessed guid(s) as failed", reconciledCount); + } + + return reconciledCount; + } + + private static Set collectAccountedGuids(EntityMutationResponse response, List pendingFailures) { + Set accountedGuids = new HashSet<>(); + + if (response != null) { + if (response.getPurgedEntities() != null) { + for (AtlasEntityHeader entityHeader : response.getPurgedEntities()) { + if (entityHeader.getGuid() != null) { + accountedGuids.add(entityHeader.getGuid()); + } + } + } + + List failedEntities = response.getFailedEntities(); + if (failedEntities != null) { + for (FailedEntity failedEntity : failedEntities) { + if (failedEntity.getGuid() != null) { + accountedGuids.add(failedEntity.getGuid()); + } + } + } + } + + if (pendingFailures != null) { + for (FailedEntity failedEntity : pendingFailures) { + if (failedEntity.getGuid() != null) { + accountedGuids.add(failedEntity.getGuid()); + } + } + } + + return accountedGuids; + } + + public PurgeBatchManager createManager(int batchSize, int numWorkers, String runId) { + PurgeWorkerContext workerContext = PurgeWorkerContext.capture(RequestContext.get(), runId); + + PurgeBatchConsumerBuilder builder = new PurgeBatchConsumerBuilder(purgeBatchExecutor, auditService, + auditOperation, batchSize, workerContext); + return new PurgeBatchManager(builder, batchSize, numWorkers); + } + + /** + * Immutable snapshot of request audit context propagated to purge worker threads. + */ + static final class PurgeWorkerContext { + private final String user; + private final Set userGroups; + private final String clientIPAddress; + private final List forwardedAddresses; + private final String runId; + + private PurgeWorkerContext(String user, Set userGroups, String clientIPAddress, + List forwardedAddresses, String runId) { + this.user = user; + this.userGroups = userGroups; + this.clientIPAddress = clientIPAddress; + this.forwardedAddresses = forwardedAddresses; + this.runId = runId; + } + + static PurgeWorkerContext capture(RequestContext source, String runId) { + Set groups = source.getUserGroups(); + List forwarded = source.getForwardedAddresses(); + + return new PurgeWorkerContext( + source.getUser(), + groups != null ? new HashSet<>(groups) : null, + source.getClientIPAddress(), + forwarded != null ? new ArrayList<>(forwarded) : null, + runId); + } + + String getRunId() { + return runId; + } + + void applyTo(RequestContext target) { + target.setUser(user, userGroups); + target.setClientIPAddress(clientIPAddress); + target.setForwardedAddresses(forwardedAddresses); + } + } + + public static class PurgeBatchConsumer extends WorkItemConsumer { + private final Set batch = new LinkedHashSet<>(); + private final PurgeBatchExecutor purgeBatchExecutor; + private final AtlasAuditService auditService; + private final AuditOperation auditOperation; + private final int batchSize; + private final PurgeWorkerContext workerContext; + private int batchesProcessed; + + public PurgeBatchConsumer(BlockingQueue queue, PurgeBatchExecutor purgeBatchExecutor, + AtlasAuditService auditService, AuditOperation auditOperation, + int batchSize, PurgeWorkerContext workerContext) { + super(queue); + this.purgeBatchExecutor = purgeBatchExecutor; + this.auditService = auditService; + this.auditOperation = auditOperation; + this.batchSize = batchSize; + this.workerContext = workerContext; + this.batchesProcessed = 0; + LOG.debug("Purge worker consumer started with batchSize={}", batchSize); + } + + @Override + protected void processItem(String guid) { + LOG.debug("==> processing the entity {}", guid); + batch.add(guid); + commit(); + } + + @Override + protected void doCommit() { + if (batch.size() == batchSize) { + attemptCommit(); + } + } + + @Override + protected void commitDirty() { + if (!batch.isEmpty()) { + attemptCommit(); + } + super.commitDirty(); + } + + protected void attemptCommit() { + if (batch.isEmpty()) { + return; + } + + Set batchGuids = new LinkedHashSet<>(batch); + + RequestContext context = RequestContext.get(); + context.clearCache(); + try { + workerContext.applyTo(context); + initializePurgeContext(); + + EntityMutationResponse res = purgeBatchExecutor.executeBatch(batchGuids); + + if (auditService != null && auditOperation != null) { + PurgeAuditWriter.writeBatch(auditService, auditOperation, + workerContext.getRunId(), batchGuids, res); + } + + addResult(new PurgeBatchResult(batchGuids, + res != null ? res.getPurgedEntities() : null, + res != null ? res.getFailedEntities() : null, + false, null)); + } catch (Throwable e) { + LOG.error("==> Exception in purge batch commit: {}", e.getMessage(), e); + List batchFailures = new ArrayList<>(); + for (String guid : batch) { + LOG.warn("Purge batch failure for guid={}: {}", guid, e.getMessage()); + FailedEntity failedEntity = PurgeUtils.classifyPurgeFailure(guid, e, null); + batchFailures.add(failedEntity); + } + + if (auditService != null && auditOperation != null) { + EntityMutationResponse response = new EntityMutationResponse(); + for (FailedEntity fe : batchFailures) { + response.addFailedEntity(fe); + } + PurgeAuditWriter.writeBatch(auditService, auditOperation, + workerContext.getRunId(), batchGuids, response); + } + + addResult(new PurgeBatchResult(batchGuids, null, batchFailures, true, e)); + } finally { + RequestContext.clear(); + batchesProcessed++; + batch.clear(); + LOG.debug("Purge worker processed batch {}", batchesProcessed); + } + } + } + + public static class PurgeBatchConsumerBuilder implements WorkItemBuilder { + private final PurgeBatchExecutor purgeBatchExecutor; + private final AtlasAuditService auditService; + private final AuditOperation auditOperation; + private final int batchSize; + private final PurgeWorkerContext workerContext; + + public PurgeBatchConsumerBuilder(PurgeBatchExecutor purgeBatchExecutor, AtlasAuditService auditService, + AuditOperation auditOperation, int batchSize, PurgeWorkerContext workerContext) { + this.purgeBatchExecutor = purgeBatchExecutor; + this.auditService = auditService; + this.auditOperation = auditOperation; + this.batchSize = batchSize; + this.workerContext = workerContext; + } + + @Override + public PurgeBatchConsumer build(BlockingQueue queue) { + return new PurgeBatchConsumer(queue, purgeBatchExecutor, auditService, + auditOperation, batchSize, workerContext); + } + } + + public static class PurgeBatchManager extends WorkItemManager { + public PurgeBatchManager(PurgeBatchConsumerBuilder builder, int batchSize, int numWorkers) { + super(builder, PURGE_WORKERS_NAME, batchSize, numWorkers, true); + } + + @Override + public void shutdown() throws InterruptedException { + LOG.debug("Shutting down purge worker manager"); + drain(); + super.shutdown(); + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/services/PurgeBatchResult.java b/repository/src/main/java/org/apache/atlas/services/PurgeBatchResult.java new file mode 100644 index 00000000000..5b6141883f7 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/services/PurgeBatchResult.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.services; + +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.FailedEntity; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Represents exactly one worker batch outcome. + * Workers enqueue this result instead of mutating PurgeExecutionStats directly. + */ +class PurgeBatchResult { + private final Set batchGuids; + private final List purgedEntities; + private final List failedEntities; + private final boolean hasBatchException; + private final Throwable batchException; + + public PurgeBatchResult(Set batchGuids, List purgedEntities, + List failedEntities, boolean hasBatchException, + Throwable batchException) { + // Use defensive copies since worker collections might be cleared after execution + this.batchGuids = batchGuids != null ? new LinkedHashSet<>(batchGuids) : Collections.emptySet(); + this.purgedEntities = purgedEntities != null ? new ArrayList<>(purgedEntities) : Collections.emptyList(); + this.failedEntities = failedEntities != null ? new ArrayList<>(failedEntities) : Collections.emptyList(); + this.hasBatchException = hasBatchException; + this.batchException = batchException; + } + + public Set getBatchGuids() { + return batchGuids; + } + + public List getPurgedEntities() { + return purgedEntities; + } + + public List getFailedEntities() { + return failedEntities; + } + + public boolean hasBatchException() { + return hasBatchException; + } + + public Throwable getBatchException() { + return batchException; + } +} diff --git a/repository/src/main/java/org/apache/atlas/services/PurgeService.java b/repository/src/main/java/org/apache/atlas/services/PurgeService.java index bc89aad8bfd..e17843da9d1 100644 --- a/repository/src/main/java/org/apache/atlas/services/PurgeService.java +++ b/repository/src/main/java/org/apache/atlas/services/PurgeService.java @@ -19,20 +19,25 @@ package org.apache.atlas.services; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; -import org.apache.atlas.DeleteType; -import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.AtlasService; import org.apache.atlas.annotation.Timed; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.FailedEntity; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.pc.WorkItemBuilder; import org.apache.atlas.pc.WorkItemConsumer; import org.apache.atlas.pc.WorkItemManager; +import org.apache.atlas.repository.audit.AtlasAuditService; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.purge.PurgeExecutionStats; +import org.apache.atlas.repository.purge.PurgeUtils; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; @@ -53,15 +58,14 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; -import java.util.Queue; import java.util.Set; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.stream.Collectors; -import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PURGE; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.GUID_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.VERTEX_INDEX; @@ -72,10 +76,11 @@ public class PurgeService implements Service { private static final Logger LOG = LoggerFactory.getLogger(PurgeService.class); private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("service.Purge"); - private final AtlasGraph atlasGraph; - private static Configuration atlasProperties; - private final AtlasEntityStore entityStore; - private final AtlasTypeRegistry typeRegistry; + private final AtlasGraph atlasGraph; + private static Configuration atlasProperties; + private final AtlasEntityStore entityStore; + private final AtlasTypeRegistry typeRegistry; + private final AtlasAuditService auditService; private static final String ENABLE_PROCESS_SOFT_DELETION = "atlas.enable.process.soft.delete"; private static final boolean ENABLE_PROCESS_SOFT_DELETION_DEFAULT = false; @@ -83,21 +88,16 @@ public class PurgeService implements Service { private static final String SOFT_DELETE_ENABLED_PROCESS_TYPES = "atlas.soft.delete.enabled.process.types"; private static final String PURGE_BATCH_SIZE = "atlas.purge.batch.size"; private static final int DEFAULT_PURGE_BATCH_SIZE = 1000; // fetching limit at a time - private static final String PURGE_WORKER_BATCH_SIZE = "atlas.purge.worker.batch.size"; - private static final int DEFAULT_PURGE_WORKER_BATCH_SIZE = 100; private static final String CLEANUP_WORKER_BATCH_SIZE = "atlas.cleanup.worker.batch.size"; private static final int DEFAULT_CLEANUP_WORKER_BATCH_SIZE = 100; private static final String PURGE_RETENTION_PERIOD = "atlas.purge.deleted.entity.retention.days"; private static final int PURGE_RETENTION_PERIOD_DEFAULT = 30; // days - private static final String PURGE_WORKERS_COUNT = "atlas.purge.workers.count"; - private static final int DEFAULT_PURGE_WORKERS_COUNT = 2; private static final String CLEANUP_WORKERS_COUNT = "atlas.cleanup.workers.count"; private static final int DEFAULT_CLEANUP_WORKERS_COUNT = 2; private static final String PROCESS_ENTITY_CLEANER_THREAD_NAME = "Process-Entity-Cleaner"; private final String indexSearchPrefix = AtlasGraphUtilsV2.getIndexSearchPrefix(); private static final int DEFAULT_CLEANUP_BATCH_SIZE = 1000; private static final String CLEANUP_WORKERS_NAME = "Process-Cleanup-Worker"; - private static final String PURGE_WORKERS_NAME = "Entity-Purge-Worker"; private static final String DELETED = "DELETED"; private static final String ACTIVE = "ACTIVE"; private static final String AND_STR = " AND "; @@ -106,15 +106,17 @@ public class PurgeService implements Service { try { atlasProperties = ApplicationProperties.get(); } catch (Exception e) { - LOG.info("Failed to load application properties", e); + LOG.warn("Failed to load application properties", e); } } @Inject - public PurgeService(AtlasGraph atlasgraph, AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) { - this.atlasGraph = atlasgraph; - this.entityStore = entityStore; - this.typeRegistry = typeRegistry; + public PurgeService(AtlasGraph atlasgraph, AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry, + AtlasAuditService auditService) { + this.atlasGraph = atlasgraph; + this.entityStore = entityStore; + this.typeRegistry = typeRegistry; + this.auditService = auditService; } @Override @@ -157,13 +159,9 @@ public void launchCleanUp() { @SuppressWarnings("unchecked") @Timed public EntityMutationResponse purgeEntities() { - LOG.info("==> PurgeService.purgeEntities()"); - // index query of specific batch size + LOG.info("purgeEntities: starting"); AtlasPerfTracer perf = null; EntityMutationResponse entityMutationResponse = new EntityMutationResponse(); - RequestContext requestContext = RequestContext.get(); - requestContext.setDeleteType(DeleteType.HARD); // hard delete - requestContext.setPurgeRequested(true); try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { @@ -171,22 +169,20 @@ public EntityMutationResponse purgeEntities() { } Set allEligibleTypes = getEntityTypes(); + Set originallyRequestedGuids = new LinkedHashSet<>(); try { - //bring n number of entities like 1000 at point of type Processes - WorkItemsQualifier wiq = createQualifier(typeRegistry, entityStore, atlasGraph, getPurgeWorkerBatchSize(), getPurgeWorkersCount(), true); - String indexQuery = getBulkQueryString(allEligibleTypes, getPurgeRetentionPeriod()); Iterator itr = atlasGraph.indexQuery(VERTEX_INDEX, indexQuery).vertices(0, getPurgeBatchSize()); - LOG.info("==> fetched Deleted entities"); if (!itr.hasNext()) { - LOG.info("==> no Purge Entities found"); + LOG.info("purgeEntities: no eligible entities found"); + PurgeExecutionStats stats = new PurgeExecutionStats(originallyRequestedGuids, + originallyRequestedGuids.size()); + PurgeUtils.attachPurgeSummary(entityMutationResponse, stats, null); return entityMutationResponse; } - Set producedDeletionCandidates = new HashSet<>(); // look up - while (itr.hasNext()) { AtlasVertex vertex = itr.next().getVertex(); @@ -194,52 +190,149 @@ public EntityMutationResponse purgeEntities() { continue; } - String guid = vertex.getProperty(GUID_PROPERTY_KEY, String.class); - - if (!producedDeletionCandidates.contains(guid)) { - Set instanceVertex = new HashSet<>(); - instanceVertex.add(guid); - - Set deletionCandidates = entityStore.accumulateDeletionCandidates(instanceVertex); - - for (AtlasVertex deletionCandidate : deletionCandidates) { - String deletionCandidateGuid = deletionCandidate.getProperty(GUID_PROPERTY_KEY, String.class); - if (!producedDeletionCandidates.contains(deletionCandidateGuid)) { - producedDeletionCandidates.add(deletionCandidateGuid); - wiq.checkProduce(deletionCandidate); - } - } + String guid = AtlasGraphUtilsV2.getIdFromVertex(vertex); + if (guid != null) { + originallyRequestedGuids.add(guid); } } - wiq.shutdown(); + // Release the index-scan transaction before worker batches commit; an open read txn + // on this thread can block concurrent purge workers on graphindex write locks. + PurgeBatchOrchestrator.clearPurgeCandidateExpansionState(); + try { + atlasGraph.rollback(); + } catch (Exception rollbackEx) { + LOG.debug("purgeEntities: rollback after index scan ignored", rollbackEx); + } - // collecting all the results - Queue results = wiq.getResults(); + List failedEntities = new ArrayList<>(); + entityMutationResponse = executePurgeWithWorkers(originallyRequestedGuids, + originallyRequestedGuids, failedEntities, AuditOperation.AUTO_PURGE); - LOG.info("==> Purged {} !", results.size()); + int resultCount = (entityMutationResponse.getPurgedEntities() != null ? entityMutationResponse.getPurgedEntities().size() : 0) + + (entityMutationResponse.getFailedEntities() != null ? entityMutationResponse.getFailedEntities().size() : 0); - while (!results.isEmpty()) { - AtlasEntityHeader entityHeader = (AtlasEntityHeader) results.poll(); - if (entityHeader == null) { - continue; - } - entityMutationResponse.addEntity(PURGE, entityHeader); - } + LOG.info("purgeEntities: completed resultCount={}, summary={}", resultCount, + entityMutationResponse.getSummary()); } catch (Exception ex) { - LOG.error("purge: failed!", ex); - } finally { - LOG.info("purge: Done!"); + LOG.error("purgeEntities: failed", ex); + PurgeBatchOrchestrator.resetPurgeContext(); + PurgeExecutionStats stats = new PurgeExecutionStats(originallyRequestedGuids, + originallyRequestedGuids.size()); + stats.markExecutionFailed(); + handleCronPurgeFailure(entityMutationResponse, stats, originallyRequestedGuids); } } finally { AtlasPerfTracer.log(perf); } - LOG.info("<== PurgeService.purgeEntities()"); + LOG.info("purgeEntities: finished summary={}", entityMutationResponse.getSummary()); return entityMutationResponse; } + public EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseException { + if (CollectionUtils.isEmpty(guids)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Guid(s) not specified"); + } + + LOG.info("purgeByIds: requested {} guid(s)", guids.size()); + + Set validGuids = new LinkedHashSet<>(); + List failedEntities = new ArrayList<>(); + PurgeUtils.preScanGuids(guids, validGuids, failedEntities, atlasGraph, typeRegistry); + + LOG.info("purgeByIds: preScan valid={}, failed={}", validGuids.size(), failedEntities.size()); + if (LOG.isDebugEnabled()) { + LOG.debug("purgeByIds: preScan validGuids={}, failedEntities={}", validGuids, failedEntities); + } + + if (validGuids.isEmpty()) { + return buildPreValidationOnlyResponse(guids, failedEntities, AuditOperation.PURGE); + } + + return executePurgeWithWorkers(guids, validGuids, failedEntities, AuditOperation.PURGE); + } + + public EntityMutationResponse executePurgeWithWorkers(Set originallyRequestedGuids, + Set validGuids, + List preFailures, + AuditOperation auditOperation) throws AtlasBaseException { + PurgeBatchOrchestrator.initializePurgeContext(); + String runId = newPurgeRunId(); + LOG.info("executePurgeWithWorkers: runId={} operation={} validGuids={}", runId, auditOperation, validGuids.size()); + + PurgeExecutionStats stats = new PurgeExecutionStats(originallyRequestedGuids, validGuids.size()); + EntityMutationResponse response = new EntityMutationResponse(); + try { + PurgeBatchExecutor executor = new PurgeBatchExecutor(entityStore); + PurgeBatchOrchestrator orchestrator = new PurgeBatchOrchestrator(executor, auditService, auditOperation); + response = orchestrator.executePurge(validGuids, preFailures, stats, runId); + } catch (Exception ex) { + LOG.error("executePurgeWithWorkers: runId={} failed", runId, ex); + stats.markExecutionFailed(); + } finally { + finalizePurgeRun(response, stats, runId, originallyRequestedGuids, auditOperation, null); + PurgeBatchOrchestrator.resetPurgeContext(); + } + + LOG.info("executePurgeWithWorkers: runId={} completed purged={}, failed={}, summary={}", + runId, + response.getPurgedEntities() != null ? response.getPurgedEntities().size() : 0, + response.getFailedEntities() != null ? response.getFailedEntities().size() : 0, + response.getSummary()); + + return response; + } + + private EntityMutationResponse buildPreValidationOnlyResponse(Set requestedGuids, + List failedEntities, + AuditOperation auditOperation) { + EntityMutationResponse response = new EntityMutationResponse(); + response.setFailedEntities(failedEntities); + PurgeExecutionStats stats = new PurgeExecutionStats(requestedGuids, 0); + stats.recordFailures(failedEntities); + String runId = newPurgeRunId(); + + finalizePurgeRun(response, stats, runId, requestedGuids, auditOperation, failedEntities); + + LOG.info("purgeByIds: preValidation runId={} requested={} failed={}", + runId, requestedGuids.size(), failedEntities.size()); + + return response; + } + + private static String newPurgeRunId() { + return UUID.randomUUID().toString(); + } + + private void handleCronPurgeFailure(EntityMutationResponse response, + PurgeExecutionStats stats, + Set originallyRequestedGuids) { + if (CollectionUtils.isEmpty(originallyRequestedGuids)) { + PurgeUtils.attachPurgeSummary(response, stats, null); + LOG.info("purgeEntities: cron failure before any eligible GUIDs were collected; skipping summary audit"); + return; + } + + String runId = newPurgeRunId(); + finalizePurgeRun(response, stats, runId, originallyRequestedGuids, AuditOperation.AUTO_PURGE, null); + LOG.info("purgeEntities: cron failure runId={} requestedGuids={}", runId, originallyRequestedGuids.size()); + } + + private void finalizePurgeRun(EntityMutationResponse response, + PurgeExecutionStats stats, + String runId, + Set originallyRequestedGuids, + AuditOperation auditOperation, + List failuresToLog) { + PurgeUtils.attachPurgeSummary(response, stats, runId); + if (failuresToLog != null) { + PurgeAuditWriter.logFailures(runId, auditOperation, failuresToLog); + } + PurgeAuditWriter.finishRun(auditService, auditOperation, runId, originallyRequestedGuids, stats); + } + @SuppressWarnings("unchecked") @Timed public void softDeleteProcessEntities() { @@ -318,7 +411,7 @@ public EntityQualifier(BlockingQueue queue, AtlasTypeRegistry typeR @Override protected void processItem(AtlasVertex vertex) { - String guid = vertex.getProperty(GUID_PROPERTY_KEY, String.class); + String guid = AtlasGraphUtilsV2.getIdFromVertex(vertex); LOG.info("==> processing the entity {}", guid); try { @@ -353,15 +446,10 @@ protected void attemptCommit() { List results = Collections.emptyList(); try { - if (isPurgeEnabled) { - // purging not by directly - res = entityStore.purgeEntitiesInBatch(batch); - } else { - List batchList = new ArrayList<>(batch); - res = entityStore.deleteByIds(batchList); - } + List batchList = new ArrayList<>(batch); + res = entityStore.deleteByIds(batchList); - results = isPurgeEnabled ? res.getPurgedEntities() : res.getDeletedEntities(); + results = res.getDeletedEntities(); if (CollectionUtils.isEmpty(results)) { return; @@ -403,7 +491,7 @@ public EntityQualifier build(BlockingQueue queue) { static class WorkItemsQualifier extends WorkItemManager { public WorkItemsQualifier(WorkItemBuilder builder, int batchSize, int numWorkers, boolean isPurgeEnabled) { - super(builder, isPurgeEnabled ? PURGE_WORKERS_NAME : CLEANUP_WORKERS_NAME, batchSize, numWorkers, true); + super(builder, isPurgeEnabled ? PurgeBatchOrchestrator.PURGE_WORKERS_NAME : CLEANUP_WORKERS_NAME, batchSize, numWorkers, true); } @Override @@ -507,13 +595,6 @@ private int getPurgeBatchSize() { return DEFAULT_PURGE_BATCH_SIZE; } - private int getPurgeWorkersCount() { - if (atlasProperties != null) { - return atlasProperties.getInt(PURGE_WORKERS_COUNT, DEFAULT_PURGE_WORKERS_COUNT); - } - return DEFAULT_PURGE_WORKERS_COUNT; - } - private int getCleanUpWorkersCount() { if (atlasProperties != null) { return atlasProperties.getInt(CLEANUP_WORKERS_COUNT, DEFAULT_CLEANUP_WORKERS_COUNT); @@ -521,13 +602,6 @@ private int getCleanUpWorkersCount() { return DEFAULT_CLEANUP_WORKERS_COUNT; } - private int getPurgeWorkerBatchSize() { - if (atlasProperties != null) { - return atlasProperties.getInt(PURGE_WORKER_BATCH_SIZE, DEFAULT_PURGE_WORKER_BATCH_SIZE); - } - return DEFAULT_PURGE_WORKER_BATCH_SIZE; - } - private int getCleanupWorkerBatchSize() { if (atlasProperties != null) { return atlasProperties.getInt(CLEANUP_WORKER_BATCH_SIZE, DEFAULT_CLEANUP_WORKER_BATCH_SIZE); diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java b/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java deleted file mode 100644 index 29247132ed0..00000000000 --- a/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java +++ /dev/null @@ -1,195 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.repository.audit; - -import org.apache.atlas.RequestContext; -import org.apache.atlas.TestModules; -import org.apache.atlas.TestUtilsV2; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.audit.AtlasAuditEntry; -import org.apache.atlas.model.audit.AuditSearchParameters; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.EntityMutationResponse; -import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.repository.AtlasTestBase; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; -import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; -import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; -import org.apache.atlas.store.AtlasTypeDefStore; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.utils.TestResourceFileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.SkipException; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import javax.inject.Inject; - -import java.io.IOException; -import java.util.Comparator; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.fail; - -@Guice(modules = TestModules.TestOnlyModule.class) -public class AdminPurgeTest extends AtlasTestBase { - private static final Logger LOG = LoggerFactory.getLogger(AdminPurgeTest.class); - - private static final String CLIENT_HOST = "127.0.0.0"; - private static final String DEFAULT_USER = "Admin"; - private static final String AUDIT_PARAMETER_RESOURCE_DIR = "auditSearchParameters"; - - @Inject - AtlasTypeRegistry typeRegistry; - - @Inject - private AtlasTypeDefStore typeDefStore; - - @Inject - private AtlasAuditService auditService; - - @Inject - private AtlasEntityStoreV2 entityStore; - - @BeforeClass - public void initialize() throws Exception { - super.initialize(); - } - - @BeforeTest - public void setupTest() throws IOException, AtlasBaseException { - RequestContext.clear(); - RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); - - basicSetup(typeDefStore, typeRegistry); - } - - @AfterClass - public void clear() throws Exception { - Thread.sleep(1000); - - AtlasGraphProvider.cleanup(); - - super.cleanup(); - } - - @Test - public void testDeleteEntitiesDoesNotLookupDeletedEntity() throws Exception { - AtlasTypesDef sampleTypes = TestUtilsV2.defineDeptEmployeeTypes(); - AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(sampleTypes, typeRegistry); - - if (!typesToCreate.isEmpty()) { - typeDefStore.createTypesDef(typesToCreate); - } - - AtlasEntity.AtlasEntitiesWithExtInfo deptEg2 = TestUtilsV2.createDeptEg2(); - AtlasEntityStream entityStream = new AtlasEntityStream(deptEg2); - EntityMutationResponse emr = entityStore.createOrUpdate(entityStream, false); - - pauseForIndexCreation(); - - assertNotNull(emr); - assertNotNull(emr.getCreatedEntities()); - assertFalse(emr.getCreatedEntities().isEmpty()); - - List guids = emr.getCreatedEntities().stream().map(AtlasEntityHeader::getGuid).collect(Collectors.toList()); - - EntityMutationResponse response = entityStore.deleteByIds(guids); - - pauseForIndexCreation(); - - List responseDeletedEntities = response.getDeletedEntities(); - - assertNotNull(responseDeletedEntities); - - responseDeletedEntities.sort(Comparator.comparing(AtlasEntityHeader::getGuid)); - - List toBeDeletedEntities = emr.getCreatedEntities(); - - toBeDeletedEntities.sort(Comparator.comparing(AtlasEntityHeader::getGuid)); - - assertEquals(responseDeletedEntities.size(), emr.getCreatedEntities().size()); - - for (int index = 0; index < responseDeletedEntities.size(); index++) { - assertEquals(responseDeletedEntities.get(index).getGuid(), emr.getCreatedEntities().get(index).getGuid()); - } - - Date startTimestamp = new Date(); - - response = entityStore.purgeByIds(new HashSet<>(guids)); - - pauseForIndexCreation(); - - List responsePurgedEntities = response.getPurgedEntities(); - - responsePurgedEntities.sort(Comparator.comparing(AtlasEntityHeader::getGuid)); - - assertEquals(responsePurgedEntities.size(), responseDeletedEntities.size()); - - for (int index = 0; index < responsePurgedEntities.size(); index++) { - assertEquals(responsePurgedEntities.get(index).getGuid(), responseDeletedEntities.get(index).getGuid()); - } - - auditService.add(DEFAULT_USER, AtlasAuditEntry.AuditOperation.PURGE, CLIENT_HOST, startTimestamp, new Date(), guids.toString(), response.getPurgedEntitiesIds(), response.getPurgedEntities().size()); - - AuditSearchParameters auditParameterNull = createAuditParameter("audit-search-parameter-without-filter"); - - assertAuditEntry(auditService, auditParameterNull); - - AuditSearchParameters auditSearchParameters = createAuditParameter("audit-search-parameter-purge"); - - assertAuditEntry(auditService, auditSearchParameters); - } - - private AuditSearchParameters createAuditParameter(String fileName) { - try { - return TestResourceFileUtils.readObjectFromJson(AUDIT_PARAMETER_RESOURCE_DIR, fileName, AuditSearchParameters.class); - } catch (IOException e) { - fail(e.getMessage()); - } - - return null; - } - - private void assertAuditEntry(AtlasAuditService auditService, AuditSearchParameters auditSearchParameters) { - pauseForIndexCreation(); - - List result; - - try { - result = auditService.get(auditSearchParameters); - } catch (Exception e) { - throw new SkipException("audit entries not retrieved."); - } - - assertNotNull(result); - assertFalse(result.isEmpty()); - } -} diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java index de8b5ab8b97..9aee74dab16 100644 --- a/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java @@ -18,30 +18,61 @@ package org.apache.atlas.repository.audit; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.TestModules; +import org.apache.atlas.discovery.AtlasDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AtlasAuditEntry; import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditRowKind; import org.apache.atlas.model.audit.AuditSearchParameters; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.EntityMutations.EntityOperation; +import org.apache.atlas.model.instance.FailedEntity; +import org.apache.atlas.model.instance.PurgeSummary; +import org.apache.atlas.repository.ogm.AtlasAuditEntryDTO; +import org.apache.atlas.repository.ogm.DataAccess; +import org.apache.atlas.repository.purge.PurgeExecutionStats; +import org.apache.atlas.repository.purge.PurgeUtils; +import org.apache.atlas.services.PurgeAuditWriter; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasJson; import org.apache.atlas.utils.TestResourceFileUtils; +import org.mockito.ArgumentCaptor; import org.testng.SkipException; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @Guice(modules = TestModules.TestOnlyModule.class) @@ -49,6 +80,7 @@ public class AtlasAuditServiceTest { private static final int WAIT_TIME_FOR_INDEX_CREATION_IN_MILLI = 5000; private static final String AUDIT_PARAMETER_RESOURCE_DIR = "auditSearchParameters"; private static final String DEFAULT_USER = "admin"; + private static final String TEST_RUN_ID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; @Inject AtlasTypeRegistry typeRegistry; @@ -59,11 +91,20 @@ public class AtlasAuditServiceTest { @Inject private AtlasTypeDefStore typeDefStore; + private AtlasDiscoveryService mockDiscoveryService; + private AtlasAuditService purgeRunLookupAuditService; + @BeforeClass public void setup() throws IOException, AtlasBaseException { loadBaseModel(typeDefStore, typeRegistry); } + @BeforeMethod + public void setupPurgeRunLookupMocks() { + mockDiscoveryService = mock(AtlasDiscoveryService.class); + purgeRunLookupAuditService = new AtlasAuditService(mock(DataAccess.class), mockDiscoveryService); + } + @Test public void checkTypeRegistered() throws AtlasBaseException { AtlasType auditEntryType = typeRegistry.getType("__" + AtlasAuditEntry.class.getSimpleName()); @@ -128,6 +169,148 @@ public void checkStoringMultipleAuditEntries() throws AtlasBaseException { assertEquals(results.size(), (maxEntries - limitParam)); } + @Test + public void purgeAuditWriter_writeBatchAndFinishRun_writesBatchAndSummaryAudits() throws Exception { + AtlasAuditService mockAuditService = mock(AtlasAuditService.class); + + Set batchGuids = new LinkedHashSet<>(Arrays.asList("guid2", "guid1")); + EntityMutationResponse response = new EntityMutationResponse(); + AtlasEntityHeader header1 = new AtlasEntityHeader(); + header1.setGuid("guid1"); + AtlasEntityHeader header2 = new AtlasEntityHeader(); + header2.setGuid("guid2"); + response.addEntity(EntityOperation.PURGE, header1); + response.addEntity(EntityOperation.PURGE, header2); + + PurgeAuditWriter.writeBatch(mockAuditService, AuditOperation.PURGE, TEST_RUN_ID, batchGuids, response); + + ArgumentCaptor batchParamsCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor batchResultCaptor = ArgumentCaptor.forClass(String.class); + verify(mockAuditService).add(eq(AuditOperation.PURGE), batchParamsCaptor.capture(), batchResultCaptor.capture(), + eq(2L), eq(TEST_RUN_ID), eq(AuditRowKind.BATCH)); + + assertEquals(batchParamsCaptor.getValue(), "guid1,guid2"); + assertEquals(batchResultCaptor.getValue(), "guid1,guid2"); + + AtlasAuditEntry batchEntry = new AtlasAuditEntry(); + batchEntry.setAuditRowKind(AuditRowKind.BATCH); + batchEntry.setResult(batchResultCaptor.getValue()); + assertFalse(PurgeUtils.isPurgeSummaryAudit(batchEntry)); + + Set originallyRequestedGuids = new LinkedHashSet<>(Arrays.asList( + "11111111-1111-1111-1111-111111111111", + "22222222-2222-2222-2222-222222222222")); + PurgeExecutionStats stats = new PurgeExecutionStats(originallyRequestedGuids, originallyRequestedGuids.size()); + + AtlasEntityHeader purged = new AtlasEntityHeader(); + purged.setGuid("11111111-1111-1111-1111-111111111111"); + stats.recordBatchOutcome(Arrays.asList(purged), null, originallyRequestedGuids); + + PurgeAuditWriter.finishRun(mockAuditService, AuditOperation.PURGE, TEST_RUN_ID, originallyRequestedGuids, stats); + + ArgumentCaptor summaryParamsCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor summaryResultCaptor = ArgumentCaptor.forClass(String.class); + verify(mockAuditService).add(eq(AuditOperation.PURGE), summaryParamsCaptor.capture(), summaryResultCaptor.capture(), + eq(1L), eq(TEST_RUN_ID), eq(AuditRowKind.SUMMARY)); + + assertEquals(summaryParamsCaptor.getValue(), "11111111-1111-1111-1111-111111111111,22222222-2222-2222-2222-222222222222"); + + PurgeSummary summary = AtlasJson.fromJson(summaryResultCaptor.getValue(), PurgeSummary.class); + assertEquals(summary.getRunId(), TEST_RUN_ID); + assertEquals(summary.getRequestedCount(), 2); + assertEquals(summary.getPurgedCount(), 1); + + AtlasAuditEntry summaryEntry = new AtlasAuditEntry(); + summaryEntry.setAuditRowKind(AuditRowKind.SUMMARY); + summaryEntry.setResult(summaryResultCaptor.getValue()); + assertTrue(PurgeUtils.isPurgeSummaryAudit(summaryEntry)); + + Set emptyBatchGuids = new LinkedHashSet<>(Arrays.asList("guid-b", "guid-a")); + EntityMutationResponse emptyBatchResponse = new EntityMutationResponse(); + emptyBatchResponse.addFailedEntity(new FailedEntity("guid-a", "ATLAS-500-00-001", "batch failed")); + emptyBatchResponse.addFailedEntity(new FailedEntity("guid-b", "ATLAS-500-00-001", "batch failed")); + + PurgeAuditWriter.writeBatch(mockAuditService, AuditOperation.PURGE, TEST_RUN_ID, emptyBatchGuids, emptyBatchResponse); + + ArgumentCaptor emptyBatchParamsCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor emptyBatchResultCaptor = ArgumentCaptor.forClass(String.class); + verify(mockAuditService).add(eq(AuditOperation.PURGE), emptyBatchParamsCaptor.capture(), + emptyBatchResultCaptor.capture(), eq(0L), eq(TEST_RUN_ID), eq(AuditRowKind.BATCH)); + + assertEquals(emptyBatchParamsCaptor.getValue(), "guid-a,guid-b"); + assertEquals(emptyBatchResultCaptor.getValue(), ""); + + AtlasAuditEntry emptyBatchEntry = new AtlasAuditEntry(); + emptyBatchEntry.setAuditRowKind(AuditRowKind.BATCH); + emptyBatchEntry.setOperation(AuditOperation.PURGE); + emptyBatchEntry.setResult(emptyBatchResultCaptor.getValue()); + assertTrue(PurgeUtils.isPurgeBatchAudit(emptyBatchEntry)); + assertFalse(PurgeUtils.isPurgeSummaryAudit(emptyBatchEntry)); + } + + @Test + public void getPurgeBatchAuditGuidsForRun_pagesUntilEmpty() throws Exception { + when(mockDiscoveryService.searchWithParameters(any(SearchParameters.class))).thenAnswer(invocation -> { + SearchParameters params = invocation.getArgument(0); + int limit = params.getLimit(); + int offset = params.getOffset(); + + AtlasSearchResult result = new AtlasSearchResult(params); + if (offset == 0) { + List fullPage = new ArrayList<>(); + for (int i = 0; i < limit; i++) { + fullPage.add(buildAuditHeader("batch-guid-" + i, "entity-" + i, AuditRowKind.BATCH)); + } + result.setEntities(fullPage); + } else if (offset == limit) { + result.setEntities(Arrays.asList(buildAuditHeader("batch-guid-last", "entity-last", AuditRowKind.BATCH))); + } else { + result.setEntities(new ArrayList<>()); + } + return result; + }); + + AtlasAuditEntry summaryEntry = new AtlasAuditEntry(); + summaryEntry.setRunId(TEST_RUN_ID); + + List batchGuids = purgeRunLookupAuditService.getPurgeBatchAuditGuidsForRun(summaryEntry); + + assertEquals(batchGuids.size(), AtlasConfiguration.SEARCH_MAX_LIMIT.getInt() + 1); + } + + @Test + public void getPurgedEntityGuidsForRun_filtersSummaryAndMergesBatchRows() throws Exception { + PurgeSummary summary = new PurgeSummary(3, 2, 1, 0, 0, 0); + summary.setRunId(TEST_RUN_ID); + + AtlasAuditEntry summaryEntry = new AtlasAuditEntry(); + summaryEntry.setGuid("summary-guid"); + summaryEntry.setRunId(TEST_RUN_ID); + summaryEntry.setAuditRowKind(AuditRowKind.SUMMARY); + summaryEntry.setResult(AtlasJson.toJson(summary)); + + Map> pagesByOffset = new HashMap<>(); + pagesByOffset.put(0, Arrays.asList( + buildAuditHeader("batch-guid-1", "entity-1,entity-2", AuditRowKind.BATCH), + buildAuditHeader("batch-guid-2", "entity-3", AuditRowKind.BATCH))); + + when(mockDiscoveryService.searchWithParameters(any(SearchParameters.class))).thenAnswer(invocation -> { + SearchParameters params = invocation.getArgument(0); + assertNotNull(params.getEntityFilters()); + assertTrue(hasBatchRowKindFilter(params.getEntityFilters()), + "Batch audit lookup should filter auditRowKind=BATCH at graph level"); + + AtlasSearchResult result = new AtlasSearchResult(params); + result.setEntities(pagesByOffset.getOrDefault(params.getOffset(), new ArrayList<>())); + return result; + }); + + assertEquals(purgeRunLookupAuditService.getPurgeBatchAuditGuidsForRun(summaryEntry), + Arrays.asList("batch-guid-1", "batch-guid-2")); + assertEquals(purgeRunLookupAuditService.getPurgedEntityGuidsForRun(summaryEntry), + Arrays.asList("entity-1", "entity-2", "entity-3")); + } + protected void waitForIndexCreation() { try { Thread.sleep(WAIT_TIME_FOR_INDEX_CREATION_IN_MILLI); @@ -166,4 +349,40 @@ private AtlasAuditEntry saveEntry(AuditOperation operation, String clientId) thr return entry; } + + private static AtlasEntityHeader buildAuditHeader(String guid, String result, AuditRowKind rowKind) { + Map attributes = new HashMap<>(); + attributes.put(AtlasAuditEntryDTO.ATTRIBUTE_OPERATION, AuditOperation.PURGE.name()); + attributes.put(AtlasAuditEntryDTO.ATTRIBUTE_RESULT, result); + attributes.put(AtlasAuditEntryDTO.ATTRIBUTE_RESULT_COUNT, 0L); + attributes.put(AtlasAuditEntryDTO.ATTRIBUTE_RUN_ID, TEST_RUN_ID); + attributes.put(AtlasAuditEntryDTO.ATTRIBUTE_AUDIT_ROW_KIND, rowKind.name()); + + AtlasEntityHeader header = new AtlasEntityHeader(); + header.setGuid(guid); + header.setAttributes(attributes); + return header; + } + + private static boolean hasBatchRowKindFilter(SearchParameters.FilterCriteria filter) { + if (filter == null) { + return false; + } + + if (AtlasAuditEntryDTO.ATTRIBUTE_AUDIT_ROW_KIND.equals(filter.getAttributeName()) + && SearchParameters.Operator.EQ.equals(filter.getOperator()) + && AuditRowKind.BATCH.name().equals(filter.getAttributeValue())) { + return true; + } + + if (filter.getCriterion() != null) { + for (SearchParameters.FilterCriteria each : filter.getCriterion()) { + if (hasBatchRowKindFilter(each)) { + return true; + } + } + } + + return false; + } } diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1Test.java index cc3d5abad81..960836ee2b1 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1Test.java @@ -17,7 +17,9 @@ */ package org.apache.atlas.repository.store.graph.v1; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.DeleteType; +import org.apache.atlas.GraphTransactionInterceptor; import org.apache.atlas.RequestContext; import org.apache.atlas.TestModules; import org.apache.atlas.TestUtilsV2; @@ -27,12 +29,15 @@ import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.AtlasTestBase; +import org.apache.atlas.repository.audit.AtlasAuditService; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.services.PurgeService; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeRegistry; import org.testng.annotations.AfterClass; @@ -48,6 +53,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.Set; import java.util.stream.Collectors; @@ -79,6 +85,12 @@ public class DeleteHandlerV1Test extends AtlasTestBase { @Inject private AtlasEntityStoreV2 entityStore; + @Inject + private AtlasGraph atlasGraph; + + @Inject + private AtlasAuditService atlasAuditService; + @Inject private DeleteHandlerDelegate deleteDelegate; @@ -132,7 +144,7 @@ public void testIsRelationshipEdgeWithPurgedEndpoint() throws Exception { assertDoesNotThrow(() -> handler.isRelationshipEdge(managerEdge)); } -// --------------------------------------------------------------- + // --------------------------------------------------------------- // deleteTraitsAndVertices / deleteAllClassifications resilience // --------------------------------------------------------------- @@ -155,7 +167,11 @@ public void testDeleteTraitsAndVerticesWithPurgedVertex() throws Exception { softDeleteGuids(Arrays.asList(mgrGuid, subGuid)); - EntityMutationResponse purgeResp = purgeGuids(new HashSet<>(Arrays.asList(mgrGuid, subGuid))); + initRequestContext(); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + + EntityMutationResponse purgeResp = entityStore.purgeEntitiesInBatch(new HashSet<>(Arrays.asList(mgrGuid, subGuid))); assertEquals(purgeResp.getPurgedEntities().size(), 2); assertNull(AtlasGraphUtilsV2.findByGuid(mgrGuid), "Entity should be removed from graph after purge"); @@ -163,6 +179,78 @@ public void testDeleteTraitsAndVerticesWithPurgedVertex() throws Exception { handler.deleteTraitsAndVertices(Collections.singleton(mgrVertex)); } + /** + * {@link DeleteHandlerV1#deleteTraitsAndVertices} must return only vertices actually deleted. + * Stale handles for already-removed entities are skipped and excluded from the return value. + */ + @Test + public void testDeleteTraitsAndVerticesReturnsOnlyDeletedVertices() throws Exception { + EntityMutationResponse createResp = createManagerWithSubordinates("delete_return", 1); + String mgrGuid = getGuidForName(createResp, "delete_return_mgr"); + String subGuid = getGuidForName(createResp, "delete_return_sub1"); + + assertNotNull(mgrGuid); + assertNotNull(subGuid); + + AtlasVertex mgrVertex = AtlasGraphUtilsV2.findByGuid(mgrGuid); + AtlasVertex subVertex = AtlasGraphUtilsV2.findByGuid(subGuid); + assertNotNull(mgrVertex); + assertNotNull(subVertex); + + softDeleteGuids(Arrays.asList(mgrGuid, subGuid)); + + initRequestContext(); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + + entityStore.purgeEntitiesInBatch(Collections.singleton(subGuid)); + + DeleteHandlerV1 handler = deleteDelegate.getHandler(); + Collection deletedVertices = handler.deleteTraitsAndVertices( + Arrays.asList(mgrVertex, subVertex)); + + assertEquals(deletedVertices.size(), 1); + assertTrue(deletedVertices.contains(mgrVertex)); + assertNull(findByGuidFresh(subGuid)); + assertNull(findByGuidFresh(mgrGuid)); + } + + /** + * When a batch contains an already-purged GUID, it must be recorded as a skippable failure + * rather than reported in {@code purgedEntities}. + */ + @Test + public void testPurgeEntitiesInBatchDoesNotReportUnconfirmedDeletes() throws Exception { + EntityMutationResponse createResp = createManagerWithSubordinates("unconfirmed", 1); + String mgrGuid = getGuidForName(createResp, "unconfirmed_mgr"); + String subGuid = getGuidForName(createResp, "unconfirmed_sub1"); + + assertNotNull(mgrGuid); + assertNotNull(subGuid); + + softDeleteGuids(Arrays.asList(mgrGuid, subGuid)); + + initRequestContext(); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + + EntityMutationResponse subPurge = entityStore.purgeEntitiesInBatch(Collections.singleton(subGuid)); + assertEntityPurged(subGuid, subPurge); + + EntityMutationResponse batchResp = entityStore.purgeEntitiesInBatch( + new LinkedHashSet<>(Arrays.asList(mgrGuid, subGuid))); + + assertNotNull(batchResp.getPurgedEntities()); + assertEquals(batchResp.getPurgedEntities().size(), 1); + assertEquals(batchResp.getPurgedEntities().get(0).getGuid(), mgrGuid); + assertNotNull(batchResp.getFailedEntities()); + assertEquals(batchResp.getFailedEntities().size(), 1); + assertEquals(batchResp.getFailedEntities().get(0).getGuid(), subGuid); + assertEquals(batchResp.getFailedEntities().get(0).getErrorCode(), + AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode()); + assertNull(findByGuidFresh(mgrGuid)); + } + // --------------------------------------------------------------- // deleteVertex resilience during purge // --------------------------------------------------------------- @@ -208,12 +296,46 @@ public void testBatchPurgeManagerAndSubordinates() throws Exception { softDeleteGuids(guidsToPurge); - EntityMutationResponse purgeResp = purgeGuids(guidsToPurge); + initRequestContext(); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + + // Single-transaction batch purge (not WIM workers) — exercises DeleteHandlerV1 edge + // iteration when manager and subordinates are removed in the same batch. + EntityMutationResponse purgeResp = entityStore.purgeEntitiesInBatch(guidsToPurge); assertEquals(purgeResp.getPurgedEntities().size(), guidsToPurge.size()); assertEntitiesPurged(guidsToPurge, purgeResp); } + /** + * Purge subordinates in a batch without the manager. The manager is soft-deleted but not + * included in the purge batch (as can happen when WIM workers split related entities). + * Inverse reference updates on the deleted manager must be skipped. + */ + @Test + public void testPurgeSubordinatesWithoutManagerInSameBatch() throws Exception { + EntityMutationResponse createResp = createManagerWithSubordinates("sub_only_purge", 2); + String mgrGuid = getGuidForName(createResp, "sub_only_purge_mgr"); + String sub1Guid = getGuidForName(createResp, "sub_only_purge_sub1"); + String sub2Guid = getGuidForName(createResp, "sub_only_purge_sub2"); + + Set allGuids = new HashSet<>(Arrays.asList(mgrGuid, sub1Guid, sub2Guid)); + softDeleteGuids(allGuids); + + initRequestContext(); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + + EntityMutationResponse purgeResp = entityStore.purgeEntitiesInBatch(new HashSet<>(Arrays.asList(sub1Guid, sub2Guid))); + + assertEquals(purgeResp.getPurgedEntities().size(), 2); + assertEntitiesPurged(new HashSet<>(Arrays.asList(sub1Guid, sub2Guid)), purgeResp); + assertNotNull(AtlasGraphUtilsV2.findByGuid(mgrGuid), "Manager should remain until explicitly purged"); + + assertEntityPurged(mgrGuid, purgeGuids(Collections.singleton(mgrGuid))); + } + // --------------------------------------------------------------- // Helpers // --------------------------------------------------------------- @@ -279,7 +401,7 @@ private EntityMutationResponse purgeGuids(Set guids) throws Exception { initRequestContext(); RequestContext.get().setDeleteType(DeleteType.HARD); RequestContext.get().setPurgeRequested(true); - return entityStore.purgeByIds(guids); + return new PurgeService(atlasGraph, entityStore, typeRegistry, atlasAuditService).purgeByIds(guids); } private void assertDoesNotThrow(Runnable runnable) { @@ -290,12 +412,18 @@ private void assertDoesNotThrow(Runnable runnable) { } } + private AtlasVertex findByGuidFresh(String guid) { + // WIM purge runs on worker threads; main-thread guidVertexCache can retain stale handles. + GraphTransactionInterceptor.clearCache(); + return AtlasGraphUtilsV2.findByGuid(guid); + } + private void assertEntityPurged(String guid, EntityMutationResponse purgeResp) { assertNotNull(purgeResp); assertNotNull(purgeResp.getPurgedEntities()); assertTrue(purgeResp.getPurgedEntities().stream().anyMatch(h -> guid.equals(h.getGuid())), "Expected guid " + guid + " in purged entities"); - assertNull(AtlasGraphUtilsV2.findByGuid(guid), "Entity should be removed from graph after purge"); + assertNull(findByGuidFresh(guid), "Entity should be removed from graph after purge"); } private void assertEntitiesPurged(Set expectedGuids, EntityMutationResponse purgeResp) { @@ -309,7 +437,7 @@ private void assertEntitiesPurged(Set expectedGuids, EntityMutationRespo assertEquals(purgedGuids, expectedGuids); for (String guid : expectedGuids) { - assertNull(AtlasGraphUtilsV2.findByGuid(guid), "Entity " + guid + " should be removed from graph after purge"); + assertNull(findByGuidFresh(guid), "Entity " + guid + " should be removed from graph after purge"); } } } diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java index 041cb8bff59..b2aa7bf24ef 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.DeleteType; import org.apache.atlas.GraphTransactionInterceptor; import org.apache.atlas.RequestContext; import org.apache.atlas.TestModules; @@ -41,12 +42,16 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.services.PurgeBatchExecutor; +import org.apache.atlas.services.PurgeBatchOrchestrator; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.util.FileUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.janusgraph.diskstorage.locking.PermanentLockingException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.BeforeClass; @@ -68,6 +73,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static org.apache.atlas.AtlasConfiguration.STORE_DIFFERENTIAL_AUDITS; import static org.apache.atlas.AtlasErrorCode.INVALID_CUSTOM_ATTRIBUTE_KEY_CHARACTERS; @@ -81,7 +90,11 @@ import static org.apache.atlas.TestUtilsV2.TABLE_TYPE; import static org.apache.atlas.TestUtilsV2.getFile; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -106,6 +119,9 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase { @Inject private EntityGraphMapper graphMapper; + @Inject + private AtlasEntityStoreV2 guiceEntityStore; + @Inject private String dbEntityGuid; @@ -141,7 +157,8 @@ public void setUp() throws Exception { @BeforeTest public void init() throws Exception { - entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper); + AtlasEntityStoreV2 entityStoreV2 = new AtlasEntityStoreV2(graph, deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper); + entityStore = entityStoreV2; RequestContext.clear(); RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); @@ -149,6 +166,13 @@ public void init() throws Exception { LOG.debug("RequestContext: activeCount={}, earliestActiveRequestTime={}", RequestContext.getActiveRequestsCount(), RequestContext.earliestActiveRequestTime()); } + private void initPurgeWorkerTest() throws Exception { + init(); + ApplicationProperties.get().setProperty("atlas.purge.workers.count", "1"); + RequestContext.clear(); + RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); + } + @Test public void testDefaultValueForPrimitiveTypes() throws Exception { init(); @@ -1964,25 +1988,6 @@ public void testDeleteByIdsWithEmptyList() throws Exception { } } - @Test - public void testPurgeByIdsWithEmptySet() throws Exception { - init(); - - try { - entityStore.purgeByIds(new HashSet<>()); - fail("Expected AtlasBaseException for empty GUID set"); - } catch (AtlasBaseException e) { - assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INVALID_PARAMETERS); - } - - try { - entityStore.purgeByIds(null); - fail("Expected AtlasBaseException for null GUID set"); - } catch (AtlasBaseException e) { - assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INVALID_PARAMETERS); - } - } - @Test public void testAddClassificationsWithInvalidParameters() throws Exception { init(); @@ -2466,15 +2471,290 @@ public void testDeleteByIdsWithNonExistentEntities() throws Exception { assertTrue(response.getDeletedEntities() == null || response.getDeletedEntities().size() == 0); } + /** + * Reproduces stale JanusGraph vertex handles on the coordinator thread when expansion runs + * without a per-root transaction boundary and workers delete vertices in parallel. + */ + @Test + public void testExpansionWithoutPerRootTxnBoundaryFailsWithStaleHandles() throws Exception { + init(); + + String dept1Guid = createSoftDeletedDepartment("stale_txn_dept1"); + + initPurgeRequestContext(); + + try { + // Direct store instance — no GraphTransaction interceptor (pre-fix coordinator path). + Set firstBatch = entityStore.accumulateDeletionCandidates( + Collections.singleton(dept1Guid)); + assertFalse(firstBatch.isEmpty(), "Expected deletion candidates from first expansion"); + + // Release Berkeley JE read locks from the coordinator txn, but keep guidVertexCache entries + // that the interceptor would have cleared after a per-root @GraphTransaction commit. + graph.rollback(); + + runPurgeBatchInWorker(firstBatch); + + try { + // Re-expand a purged root: main-thread guidVertexCache still holds stale handles. + entityStore.accumulateDeletionCandidates(Collections.singleton(dept1Guid)); + fail("Expected stale coordinator-thread graph transaction to fail second expansion"); + } catch (Exception e) { + assertTrue(hasIllegalStateInChain(e), + "Expected IllegalStateException from stale graph handles, got: " + e); + } + } finally { + graph.rollback(); + GraphTransactionInterceptor.clearCache(); + RequestContext.clear(); + } + } + + /** + * Verifies {@code @GraphTransaction} on accumulateDeletionCandidates plus + * {@link PurgeBatchOrchestrator#clearPurgeCandidateExpansionState()} prevent stale-handle failures + * when workers delete the first expansion's vertices before the next root is expanded. + */ + @Test + public void testPerRootGraphTransactionSurvivesParallelWorkerDeletes() throws Exception { + init(); + + String dept1Guid = createSoftDeletedDepartment("per_root_txn_dept1"); + String dept2Guid = createSoftDeletedDepartment("per_root_txn_dept2"); + + initPurgeRequestContext(); + + try { + Set firstBatch = guiceEntityStore.accumulateDeletionCandidates( + Collections.singleton(dept1Guid)); + assertFalse(firstBatch.isEmpty()); + + runPurgeBatchInWorker(firstBatch); + PurgeBatchOrchestrator.clearPurgeCandidateExpansionState(); + + Set secondBatch = guiceEntityStore.accumulateDeletionCandidates( + Collections.singleton(dept2Guid)); + assertFalse(secondBatch.isEmpty(), "Second expansion should succeed with per-root txn boundary"); + } finally { + graph.rollback(); + GraphTransactionInterceptor.clearCache(); + RequestContext.clear(); + } + } + + /** + * Verifies {@link PurgeBatchExecutor} clears graph and RequestContext caches before retrying a + * batch after a lock conflict, so stale vertex handles from a rolled-back attempt do not break + * the next attempt. + */ + @Test + public void testPurgeBatchRetryClearsStaleCachesAfterLockConflict() throws Exception { + initPurgeWorkerTest(); + + String guid = createSoftDeletedDepartment("retry_stale_dept"); + Set batch = Collections.singleton(guid); + + initPurgeRequestContext(); + + try { + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(graph, guid); + assertNotNull(vertex); + assertNotNull(GraphTransactionInterceptor.getVertexFromCache(guid), + "Expected guidVertexCache entry before rollback"); + + graph.rollback(); + + AtlasEntityStoreV2 storeSpy = spy(guiceEntityStore); + PermanentLockingException ple = new PermanentLockingException("simulated lock conflict"); + AtlasBaseException lockConflict = new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, ple); + doThrow(lockConflict) + .doCallRealMethod() + .when(storeSpy).purgeEntitiesInBatch(batch); + + PurgeBatchExecutor executor = new PurgeBatchExecutor(storeSpy); + EntityMutationResponse response = executor.executeBatch(batch); + + assertNotNull(response); + assertNotNull(response.getPurgedEntities()); + assertEquals(response.getPurgedEntities().size(), 1); + assertEquals(response.getPurgedEntities().get(0).getGuid(), guid); + assertNull(AtlasGraphUtilsV2.findByGuid(graph, guid), + "Entity should be purged after successful retry"); + verify(storeSpy, times(2)).purgeEntitiesInBatch(batch); + } finally { + graph.rollback(); + GraphTransactionInterceptor.clearCache(); + RequestContext.clear(); + } + } + + @Test + public void testPurgeExpansionVsWorkerDeleteConcurrency() throws Exception { + init(); + + String dept1Guid = createSoftDeletedDepartment("conc_dept1"); + String dept2Guid = createSoftDeletedDepartment("conc_dept2"); + + initPurgeRequestContext(); + + try { + Set firstBatch = guiceEntityStore.accumulateDeletionCandidates( + Collections.singleton(dept1Guid)); + assertFalse(firstBatch.isEmpty()); + + CountDownLatch done = new CountDownLatch(1); + AtomicReference workerError = new AtomicReference<>(); + PurgeBatchExecutor purgeBatchExecutor = new PurgeBatchExecutor(guiceEntityStore); + + Thread worker = new Thread(() -> { + try { + RequestContext.clear(); + RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + purgeBatchExecutor.executeBatch(firstBatch); + } catch (Exception e) { + workerError.set(e); + } finally { + done.countDown(); + RequestContext.clear(); + } + }, "purge-race-worker"); + + worker.start(); + + Thread.sleep(100); + + Set secondBatch = guiceEntityStore.accumulateDeletionCandidates( + Collections.singleton(dept2Guid)); + + assertTrue(done.await(60, TimeUnit.SECONDS), "Worker purge timed out"); + + if (workerError.get() != null) { + throw workerError.get(); + } + + assertFalse(secondBatch.isEmpty(), "Second expansion should succeed concurrently"); + + guiceEntityStore.purgeEntitiesInBatch(secondBatch); + } finally { + RequestContext.clear(); + PurgeBatchOrchestrator.clearPurgeCandidateExpansionState(); + } + } + @Test - public void testPurgeByIdsWithNonExistentEntities() throws Exception { + public void testPurgeEntitiesInBatchRejectsNonDeletedState() throws Exception { init(); - // Test purging non-existent entities (should not throw exception) - Set guids = new HashSet<>(Arrays.asList("non-existent-guid-1", "non-existent-guid-2")); - EntityMutationResponse response = entityStore.purgeByIds(guids); + AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); + EntityMutationResponse createResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); + String guid = createResponse.getCreatedEntities().get(0).getGuid(); + + RequestContext.clear(); + RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + + EntityMutationResponse response = entityStore.purgeEntitiesInBatch(Collections.singleton(guid)); assertNotNull(response); - assertTrue(response.getPurgedEntities() == null || response.getPurgedEntities().size() == 0); + assertNotNull(response.getFailedEntities()); + assertEquals(response.getFailedEntities().size(), 1); + assertEquals(response.getFailedEntities().get(0).getErrorCode(), AtlasErrorCode.NOT_IN_DELETED_STATE.getErrorCode()); + assertTrue(response.getPurgedEntities() == null || response.getPurgedEntities().isEmpty()); + } + + @Test + public void testPurgeEntitiesInBatchSkipsAlreadyRemovedGuid() throws Exception { + init(); + + AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); + EntityMutationResponse createResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); + String guid = createResponse.getCreatedEntities().get(0).getGuid(); + + entityStore.deleteById(guid); + + RequestContext.clear(); + RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + + EntityMutationResponse firstPurge = entityStore.purgeEntitiesInBatch(Collections.singleton(guid)); + assertNotNull(firstPurge); + assertNotNull(firstPurge.getPurgedEntities()); + assertEquals(firstPurge.getPurgedEntities().size(), 1); + assertEquals(firstPurge.getPurgedEntities().get(0).getGuid(), guid); + assertNotNull(firstPurge.getPurgedEntities().get(0).getTypeName()); + + EntityMutationResponse secondPurge = entityStore.purgeEntitiesInBatch(Collections.singleton(guid)); + assertNotNull(secondPurge); + assertNotNull(secondPurge.getFailedEntities()); + assertEquals(secondPurge.getFailedEntities().size(), 1); + assertEquals(secondPurge.getFailedEntities().get(0).getGuid(), guid); + assertEquals(secondPurge.getFailedEntities().get(0).getErrorCode(), + AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode()); + assertTrue(secondPurge.getPurgedEntities() == null || secondPurge.getPurgedEntities().isEmpty()); + } + + /** + * Two threads purging the same soft-deleted GUID must not both report it as purged; the loser + * should record a skippable {@code INSTANCE_GUID_NOT_FOUND} failure. + */ + @Test + public void testConcurrentPurgeSameGuidNotDoubleReported() throws Exception { + init(); + + AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); + EntityMutationResponse createResponse = guiceEntityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); + String guid = createResponse.getCreatedEntities().get(0).getGuid(); + + guiceEntityStore.deleteById(guid); + graph.rollback(); + + PurgeBatchExecutor executor = new PurgeBatchExecutor(guiceEntityStore); + CountDownLatch startGate = new CountDownLatch(1); + AtomicInteger purgedCount = new AtomicInteger(0); + AtomicInteger failedCount = new AtomicInteger(0); + AtomicReference workerError = new AtomicReference<>(); + + Runnable concurrentPurge = () -> { + try { + startGate.await(); + RequestContext.clear(); + RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + + EntityMutationResponse response = executor.executeBatch(Collections.singleton(guid)); + if (response.getPurgedEntities() != null) { + purgedCount.addAndGet(response.getPurgedEntities().size()); + } + if (response.getFailedEntities() != null) { + failedCount.addAndGet(response.getFailedEntities().size()); + } + } catch (Exception e) { + workerError.set(e); + } finally { + RequestContext.clear(); + GraphTransactionInterceptor.clearCache(); + } + }; + + Thread worker1 = new Thread(concurrentPurge, "purge-race-1"); + Thread worker2 = new Thread(concurrentPurge, "purge-race-2"); + worker1.start(); + worker2.start(); + startGate.countDown(); + worker1.join(TimeUnit.SECONDS.toMillis(60)); + worker2.join(TimeUnit.SECONDS.toMillis(60)); + + if (workerError.get() != null) { + throw workerError.get(); + } + + assertEquals(purgedCount.get(), 1, "Exactly one concurrent purge should report success"); + assertEquals(failedCount.get(), 1, "The other concurrent purge should record a skippable failure"); + assertNull(AtlasGraphUtilsV2.findByGuid(guid), "Entity should be removed from graph after purge"); } @Test @@ -2791,4 +3071,71 @@ public void testDeleteClassificationWithAssociatedEntityGuid() throws Exception List classifications = entityStore.getClassifications(guid); assertFalse(classifications.stream().anyMatch(c -> "TestClassificationForDeletion".equals(c.getTypeName()))); } + + private String createSoftDeletedDepartment(String namePrefix) throws Exception { + AtlasEntity.AtlasEntitiesWithExtInfo deptPayload = TestUtilsV2.createDeptEg2(); + String uniqueDeptName = namePrefix + "_" + System.nanoTime(); + + for (AtlasEntity entity : deptPayload.getEntities()) { + if (TestUtilsV2.DEPARTMENT_TYPE.equals(entity.getTypeName())) { + entity.setAttribute("name", uniqueDeptName); + break; + } + } + + EntityMutationResponse createResponse = guiceEntityStore.createOrUpdate(new AtlasEntityStream(deptPayload), false); + AtlasEntityHeader deptHeader = createResponse.getFirstCreatedEntityByTypeName(TestUtilsV2.DEPARTMENT_TYPE); + assertNotNull(deptHeader, "Department entity should be created"); + + guiceEntityStore.deleteById(deptHeader.getGuid()); + return deptHeader.getGuid(); + } + + private void initPurgeRequestContext() { + RequestContext.clear(); + RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + } + + private void runPurgeBatchInWorker(Set guids) throws Exception { + // Ensure the coordinator thread is not holding a Berkeley JE read txn before worker writes. + graph.rollback(); + + CountDownLatch done = new CountDownLatch(1); + AtomicReference workerError = new AtomicReference<>(); + PurgeBatchExecutor executor = new PurgeBatchExecutor(guiceEntityStore); + + Thread worker = new Thread(() -> { + try { + RequestContext.clear(); + RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); + RequestContext.get().setDeleteType(DeleteType.HARD); + RequestContext.get().setPurgeRequested(true); + executor.executeBatch(guids); + } catch (Exception e) { + workerError.set(e); + } finally { + done.countDown(); + RequestContext.clear(); + } + }, "purge-stale-handle-test-worker"); + + worker.start(); + assertTrue(done.await(60, TimeUnit.SECONDS), "Worker purge timed out"); + + if (workerError.get() != null) { + throw workerError.get(); + } + } + + private static boolean hasIllegalStateInChain(Throwable t) { + for (Throwable c = t; c != null; c = c.getCause()) { + if (c instanceof IllegalStateException) { + return true; + } + } + + return false; + } } diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java index 5df47035f94..b2eac05d126 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java @@ -103,7 +103,8 @@ public void clear() throws Exception { @BeforeTest public void init() throws Exception { - entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper); + AtlasEntityStoreV2 entityStoreV2 = new AtlasEntityStoreV2(graph, deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper); + entityStore = entityStoreV2; RequestContext.clear(); RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); diff --git a/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java b/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java new file mode 100644 index 00000000000..11cf160cc3e --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java @@ -0,0 +1,208 @@ +package org.apache.atlas.services; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.GraphTransactionInterceptor; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.janusgraph.diskstorage.locking.PermanentLockingException; +import org.mockito.MockedStatic; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.Set; + +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +public class PurgeBatchExecutorTest { + private static final Set BATCH = Collections.singleton("guid1"); + + @DataProvider(name = "retryableLockConflictExceptionClassNames") + public Object[][] retryableLockConflictExceptionClassNames() { + return new Object[][] { + {"org.janusgraph.diskstorage.locking.PermanentLockingException"}, + {"com.sleepycat.je.LockTimeoutException"}, + {"com.sleepycat.je.DeadlockException"}, + {"org.janusgraph.diskstorage.PermanentBackendException"} + }; + } + + @Test + public void testExecuteBatchSuccess() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + EntityMutationResponse mockResponse = new EntityMutationResponse(); + when(mockStore.purgeEntitiesInBatch(BATCH)).thenReturn(mockResponse); + + PurgeBatchExecutor executor = new PurgeBatchExecutor(mockStore); + EntityMutationResponse response = executor.executeBatch(BATCH); + + assertEquals(response, mockResponse); + verify(mockStore, times(1)).purgeEntitiesInBatch(BATCH); + } + + @Test + public void testIsRetryableLockConflictReturnsFalseForNull() { + assertFalse(PurgeBatchExecutor.isRetryableLockConflict(null)); + } + + @Test + public void testIsRetryableLockConflictReturnsFalseForNonRetryableException() { + assertFalse(PurgeBatchExecutor.isRetryableLockConflict(new RuntimeException("unexpected"))); + } + + @Test + public void testIsRetryableLockConflictMatchesWrappedCause() { + PermanentLockingException ple = new PermanentLockingException("lock conflict"); + RuntimeException wrapped = new RuntimeException(new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, ple)); + + assertTrue(PurgeBatchExecutor.isRetryableLockConflict(wrapped)); + } + + @Test(dataProvider = "retryableLockConflictExceptionClassNames") + public void testIsRetryableLockConflictMatchesKnownTypes(String className) throws Exception { + Exception conflict = newExceptionByClassName(className, "lock conflict"); + + assertTrue(PurgeBatchExecutor.RETRYABLE_LOCK_CONFLICT_EXCEPTION_CLASS_NAMES.contains(className)); + assertTrue(PurgeBatchExecutor.isRetryableLockConflict(conflict)); + // Use message+cause form: RuntimeException(Throwable) calls cause.toString(), which NPEs on + // partially-initialized Berkeley JE DatabaseException instances created for this test. + assertTrue(PurgeBatchExecutor.isRetryableLockConflict(wrapWithCause(conflict))); + } + + @Test + public void testExecuteBatchClearsCachesBeforeRetry() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + EntityMutationResponse mockResponse = new EntityMutationResponse(); + PermanentLockingException ple = new PermanentLockingException("lock conflict"); + AtlasBaseException wrappedException = new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, ple); + + when(mockStore.purgeEntitiesInBatch(BATCH)) + .thenThrow(wrappedException) + .thenReturn(mockResponse); + + try (MockedStatic interceptor = mockStatic(GraphTransactionInterceptor.class); + MockedStatic requestContextStatic = mockStatic(RequestContext.class)) { + RequestContext mockContext = mock(RequestContext.class); + requestContextStatic.when(RequestContext::get).thenReturn(mockContext); + interceptor.when(GraphTransactionInterceptor::clearCache).thenAnswer(invocation -> null); + doNothing().when(mockContext).clearCache(); + + PurgeBatchExecutor executor = new PurgeBatchExecutor(mockStore); + EntityMutationResponse response = executor.executeBatch(BATCH); + + assertEquals(response, mockResponse); + interceptor.verify(GraphTransactionInterceptor::clearCache, times(1)); + verify(mockContext).clearCache(); + verify(mockStore, times(2)).purgeEntitiesInBatch(BATCH); + } + } + + @Test + public void testExecuteBatchRetryOnPermanentLockingException() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + EntityMutationResponse mockResponse = new EntityMutationResponse(); + + PermanentLockingException ple = new PermanentLockingException("Locking conflict"); + AtlasBaseException wrappedException = new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, ple); + + when(mockStore.purgeEntitiesInBatch(BATCH)) + .thenThrow(wrappedException) + .thenThrow(wrappedException) + .thenReturn(mockResponse); + + PurgeBatchExecutor executor = new PurgeBatchExecutor(mockStore); + + long start = System.currentTimeMillis(); + EntityMutationResponse response = executor.executeBatch(BATCH); + long duration = System.currentTimeMillis() - start; + + assertEquals(response, mockResponse); + verify(mockStore, times(3)).purgeEntitiesInBatch(BATCH); + assertTrue(duration >= 1000, "Expected backoff delays but finished in " + duration + " ms"); + } + + @Test(dataProvider = "retryableLockConflictExceptionClassNames") + public void testExecuteBatchRetriesOnKnownLockConflictTypes(String className) throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + EntityMutationResponse mockResponse = new EntityMutationResponse(); + Exception conflict = newExceptionByClassName(className, "lock conflict"); + AtlasBaseException wrappedException = new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, conflict); + + when(mockStore.purgeEntitiesInBatch(BATCH)) + .thenThrow(wrappedException) + .thenReturn(mockResponse); + + PurgeBatchExecutor executor = new PurgeBatchExecutor(mockStore); + EntityMutationResponse response = executor.executeBatch(BATCH); + + assertEquals(response, mockResponse); + verify(mockStore, times(2)).purgeEntitiesInBatch(BATCH); + } + + @Test + public void testExecuteBatchFailsAfterMaxLockingConflicts() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + PermanentLockingException ple = new PermanentLockingException("lock conflict"); + AtlasBaseException wrappedException = new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, ple); + + when(mockStore.purgeEntitiesInBatch(BATCH)).thenThrow(wrappedException); + + PurgeBatchExecutor executor = new PurgeBatchExecutor(mockStore); + + AtlasBaseException ex = expectThrows(AtlasBaseException.class, () -> executor.executeBatch(BATCH)); + + assertEquals(ex.getAtlasErrorCode(), AtlasErrorCode.INTERNAL_ERROR); + verify(mockStore, times(3)).purgeEntitiesInBatch(BATCH); + } + + @Test + public void testExecuteBatchNoRetryOnNonLockingException() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + RuntimeException nonRetryable = new RuntimeException("unexpected"); + AtlasBaseException wrappedException = new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, nonRetryable); + + when(mockStore.purgeEntitiesInBatch(BATCH)).thenThrow(wrappedException); + + PurgeBatchExecutor executor = new PurgeBatchExecutor(mockStore); + + AtlasBaseException ex = expectThrows(AtlasBaseException.class, () -> executor.executeBatch(BATCH)); + + assertEquals(ex.getAtlasErrorCode(), AtlasErrorCode.INTERNAL_ERROR); + verify(mockStore, times(1)).purgeEntitiesInBatch(BATCH); + } + + private static RuntimeException wrapWithCause(Throwable cause) { + return new RuntimeException("wrapped", cause); + } + + private static Exception newExceptionByClassName(String className, String message) throws Exception { + try { + Class clazz = Class.forName(className); + try { + return (Exception) clazz.getConstructor(String.class).newInstance(message); + } catch (NoSuchMethodException e) { + try { + return (Exception) clazz.getConstructor().newInstance(); + } catch (NoSuchMethodException e2) { + java.lang.reflect.Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); + f.setAccessible(true); + sun.misc.Unsafe unsafe = (sun.misc.Unsafe) f.get(null); + return (Exception) unsafe.allocateInstance(clazz); + } + } + } catch (ClassNotFoundException e) { + throw new org.testng.SkipException("Required exception class not on classpath: " + className); + } + } +} diff --git a/repository/src/test/java/org/apache/atlas/services/PurgeBatchOrchestratorTest.java b/repository/src/test/java/org/apache/atlas/services/PurgeBatchOrchestratorTest.java new file mode 100644 index 00000000000..65c981258a2 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/services/PurgeBatchOrchestratorTest.java @@ -0,0 +1,657 @@ +package org.apache.atlas.services; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.EntityMutations.EntityOperation; +import org.apache.atlas.model.instance.FailedEntity; +import org.apache.atlas.model.instance.PurgeSummary; +import org.apache.atlas.repository.purge.PurgeExecutionStats; +import org.apache.atlas.repository.purge.PurgeUtils; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; + +import static org.mockito.Answers.CALLS_REAL_METHODS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +public class PurgeBatchOrchestratorTest { + private static final String TEST_RUN_ID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + + private static void attachPurgeSummaryFromResponse(EntityMutationResponse response, + Set originallyRequestedGuids, + long validGuidCount) { + PurgeExecutionStats stats = PurgeExecutionStats.fromResponse(response, originallyRequestedGuids, validGuidCount); + PurgeUtils.attachPurgeSummary(response, stats, null); + } + + private PurgeBatchExecutor mockExecutor; + private PurgeBatchOrchestrator orchestrator; + private final AtlasEntityStore entityStore = mock(AtlasEntityStore.class, CALLS_REAL_METHODS); + + @BeforeMethod + public void setup() { + mockExecutor = mock(PurgeBatchExecutor.class); + orchestrator = new PurgeBatchOrchestrator(mockExecutor, null, null); + RequestContext.clear(); + RequestContext.get().setUser("testUser", null); + } + + @AfterMethod + public void teardown() { + RequestContext.clear(); + } + + @Test + public void testBatchFlushingAndResults() throws Exception { + PurgeBatchOrchestrator.PurgeBatchManager manager = orchestrator.createManager(2, 2, TEST_RUN_ID); + + // Setup mock response + EntityMutationResponse mockResponse = new EntityMutationResponse(); + AtlasEntityHeader header1 = new AtlasEntityHeader(); + header1.setGuid("guid1"); + AtlasEntityHeader header2 = new AtlasEntityHeader(); + header2.setGuid("guid2"); + Map> mutatedEntities = new HashMap<>(); + mutatedEntities.put(EntityOperation.PURGE, Arrays.asList(header1, header2)); + mockResponse.setMutatedEntities(mutatedEntities); + + when(mockExecutor.executeBatch(anySet())).thenReturn(mockResponse); + + manager.checkProduce("guid1"); + manager.checkProduce("guid2"); + manager.checkProduce("guid3"); + + manager.shutdown(); + + Queue results = manager.getResults(); + assertNotNull(results); + + // One batch of 2 was processed, another batch of 1 was processed during shutdown + verify(mockExecutor, times(2)).executeBatch(anySet()); + } + + @Test + public void testExceptionHandlingConvertsToFailedEntities() throws Exception { + PurgeBatchOrchestrator.PurgeBatchManager manager = orchestrator.createManager(1, 1, TEST_RUN_ID); + + when(mockExecutor.executeBatch(anySet())).thenThrow(new OutOfMemoryError("OOM Test")); + + manager.checkProduce("guid1"); + manager.shutdown(); + + Queue results = manager.getResults(); + assertEquals(results.size(), 1); + + Object result = results.poll(); + assertTrue(result instanceof PurgeBatchResult); + PurgeBatchResult batchResult = (PurgeBatchResult) result; + assertTrue(batchResult.hasBatchException()); + assertEquals(batchResult.getBatchException().getMessage(), "OOM Test"); + assertEquals(batchResult.getFailedEntities().size(), 1); + assertEquals(batchResult.getFailedEntities().get(0).getGuid(), "guid1"); + } + + @Test + public void testReconcileUnprocessedGuidsMarksMissingAsFailed() { + EntityMutationResponse response = new EntityMutationResponse(); + AtlasEntityHeader header1 = new AtlasEntityHeader(); + header1.setGuid("guid1"); + response.addEntity(EntityOperation.PURGE, header1); + + List pendingFailures = new ArrayList<>(); + pendingFailures.add(new FailedEntity( + "guid2", + org.apache.atlas.AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode(), + "already failed")); + + Set submittedGuids = new LinkedHashSet<>(Arrays.asList("guid1", "guid2", "guid3")); + int reconciledCount = PurgeBatchOrchestrator.reconcileUnprocessedGuids(submittedGuids, response, pendingFailures, null); + + assertEquals(reconciledCount, 1); + assertEquals(pendingFailures.size(), 2); + assertEquals(pendingFailures.get(1).getGuid(), "guid3"); + assertEquals(pendingFailures.get(1).getErrorCode(), org.apache.atlas.AtlasErrorCode.INTERNAL_ERROR.getErrorCode()); + assertEquals(pendingFailures.get(1).getErrorMessage(), PurgeBatchOrchestrator.UNPROCESSED_PURGE_GUID_MESSAGE); + } + + @Test + public void testReconcileUnprocessedGuidsNoOpWhenAllAccounted() { + EntityMutationResponse response = new EntityMutationResponse(); + AtlasEntityHeader header1 = new AtlasEntityHeader(); + header1.setGuid("guid1"); + response.addEntity(EntityOperation.PURGE, header1); + response.addFailedEntity(new FailedEntity( + "guid2", + org.apache.atlas.AtlasErrorCode.INTERNAL_ERROR.getErrorCode(), + "batch failure")); + + int reconciledCount = PurgeBatchOrchestrator.reconcileUnprocessedGuids( + Arrays.asList("guid1", "guid2"), response, null, null); + + assertEquals(reconciledCount, 0); + assertEquals(response.getFailedEntities().size(), 1); + } + + @Test + public void testBatchWorkerReportsPurgedEntitiesOnSuccess() throws Exception { + PurgeBatchOrchestrator.PurgeBatchManager manager = orchestrator.createManager(2, 1, TEST_RUN_ID); + + EntityMutationResponse mockResponse = new EntityMutationResponse(); + AtlasEntityHeader header1 = new AtlasEntityHeader(); + header1.setGuid("guid1"); + AtlasEntityHeader header2 = new AtlasEntityHeader(); + header2.setGuid("guid2"); + Map> mutatedEntities = new HashMap<>(); + mutatedEntities.put(EntityOperation.PURGE, Arrays.asList(header1, header2)); + mockResponse.setMutatedEntities(mutatedEntities); + + when(mockExecutor.executeBatch(anySet())).thenReturn(mockResponse); + + manager.checkProduce("guid1"); + manager.checkProduce("guid2"); + manager.shutdown(); + + Queue results = manager.getResults(); + assertEquals(results.size(), 1); + + Object result = results.poll(); + assertTrue(result instanceof PurgeBatchResult); + PurgeBatchResult batchResult = (PurgeBatchResult) result; + assertEquals(batchResult.getPurgedEntities().size(), 2); + assertEquals(batchResult.getPurgedEntities().get(0).getGuid(), "guid1"); + assertEquals(batchResult.getPurgedEntities().get(1).getGuid(), "guid2"); + } + + @Test + public void testBatchAlreadyRemovedEntitiesReportedAsSkipped() throws Exception { + PurgeBatchOrchestrator.PurgeBatchManager manager = orchestrator.createManager(1, 1, TEST_RUN_ID); + + EntityMutationResponse mockResponse = new EntityMutationResponse(); + mockResponse.addFailedEntity(new FailedEntity( + "guid1", + AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode(), + "already removed")); + + when(mockExecutor.executeBatch(anySet())).thenReturn(mockResponse); + + manager.checkProduce("guid1"); + manager.shutdown(); + + Queue results = manager.getResults(); + assertEquals(results.size(), 1); + + Object result = results.poll(); + assertTrue(result instanceof PurgeBatchResult); + PurgeBatchResult batchResult = (PurgeBatchResult) result; + assertEquals(batchResult.getFailedEntities().size(), 1); + FailedEntity failedEntity = batchResult.getFailedEntities().get(0); + assertEquals(failedEntity.getGuid(), "guid1"); + assertEquals(failedEntity.getErrorCode(), AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode()); + assertTrue(PurgeUtils.isSkippablePurgeFailureCode(failedEntity.getErrorCode())); + } + + @Test + public void testExecutePurgeSuccess() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + when(mockExecutor.getEntityStore()).thenReturn(mockStore); + + String rootGuid = "11111111-1111-1111-1111-111111111111"; + String dependencyGuid = "22222222-2222-2222-2222-222222222222"; + when(mockStore.accumulateDeletionCandidates(Collections.singleton(rootGuid))) + .thenReturn(new LinkedHashSet<>(Collections.singletonList(dependencyGuid))); + + EntityMutationResponse batchResponse = new EntityMutationResponse(); + AtlasEntityHeader purgedDependency = new AtlasEntityHeader(); + purgedDependency.setGuid(dependencyGuid); + AtlasEntityHeader purgedRoot = new AtlasEntityHeader(); + purgedRoot.setGuid(rootGuid); + batchResponse.addEntity(EntityOperation.PURGE, purgedDependency); + batchResponse.addEntity(EntityOperation.PURGE, purgedRoot); + when(mockExecutor.executeBatch(anySet())).thenReturn(batchResponse); + + List failedEntities = new ArrayList<>(); + failedEntities.add(new FailedEntity( + "33333333-3333-3333-3333-333333333333", + AtlasErrorCode.INVALID_GUID.getErrorCode(), + "invalid guid")); + + Set validGuids = new LinkedHashSet<>(Collections.singletonList(rootGuid)); + PurgeExecutionStats stats = new PurgeExecutionStats(validGuids, validGuids.size()); + EntityMutationResponse response = orchestrator.executePurge(validGuids, failedEntities, stats, TEST_RUN_ID); + + assertNotNull(response.getPurgedEntities()); + assertEquals(response.getPurgedEntities().size(), 2); + assertNotNull(response.getFailedEntities()); + assertEquals(response.getFailedEntities().size(), 1); + assertEquals(response.getFailedEntities().get(0).getErrorCode(), AtlasErrorCode.INVALID_GUID.getErrorCode()); + verify(mockStore).accumulateDeletionCandidates(Collections.singleton(rootGuid)); + } + + @Test + public void testExecutePurgeRecordsExpansionFailure() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + when(mockExecutor.getEntityStore()).thenReturn(mockStore); + + String rootGuid = "11111111-1111-1111-1111-111111111111"; + when(mockStore.accumulateDeletionCandidates(Collections.singleton(rootGuid))) + .thenThrow(new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, rootGuid)); + + List failedEntities = new ArrayList<>(); + Set validGuids = new LinkedHashSet<>(Collections.singletonList(rootGuid)); + PurgeExecutionStats stats = new PurgeExecutionStats(validGuids, validGuids.size()); + EntityMutationResponse response = orchestrator.executePurge(validGuids, failedEntities, stats, TEST_RUN_ID); + + assertTrue(response.getPurgedEntities() == null || response.getPurgedEntities().isEmpty()); + assertEquals(failedEntities.size(), 1); + assertEquals(failedEntities.get(0).getGuid(), rootGuid); + assertEquals(failedEntities.get(0).getErrorCode(), AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode()); + assertNotNull(response.getFailedEntities()); + assertEquals(response.getFailedEntities().size(), 1); + } + + @Test + public void testExecutePurgeMergesPartialResultsWhenShutdownInterrupted() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + when(mockExecutor.getEntityStore()).thenReturn(mockStore); + + String rootGuid = "11111111-1111-1111-1111-111111111111"; + String dependencyGuid = "22222222-2222-2222-2222-222222222222"; + when(mockStore.accumulateDeletionCandidates(Collections.singleton(rootGuid))) + .thenReturn(new LinkedHashSet<>(Collections.singletonList(dependencyGuid))); + + EntityMutationResponse batchResponse = new EntityMutationResponse(); + AtlasEntityHeader purgedHeader = new AtlasEntityHeader(); + purgedHeader.setGuid(dependencyGuid); + batchResponse.addEntity(EntityOperation.PURGE, purgedHeader); + when(mockExecutor.executeBatch(anySet())).thenReturn(batchResponse); + + PurgeBatchOrchestrator orchestratorSpy = spy(orchestrator); + doAnswer(invocation -> { + PurgeBatchOrchestrator.PurgeBatchManager manager = + (PurgeBatchOrchestrator.PurgeBatchManager) invocation.callRealMethod(); + PurgeBatchOrchestrator.PurgeBatchManager managerSpy = spy(manager); + + doAnswer(shutdownInvocation -> { + managerSpy.getResults().offer(new PurgeBatchResult( + new LinkedHashSet<>(Collections.singletonList(dependencyGuid)), + Collections.singletonList(purgedHeader), null, false, null)); + throw new InterruptedException("simulated shutdown interrupt"); + }).when(managerSpy).shutdown(); + + return managerSpy; + }).when(orchestratorSpy).createManager(anyInt(), anyInt(), anyString()); + + List failedEntities = new ArrayList<>(); + Set validGuids = new LinkedHashSet<>(Collections.singletonList(rootGuid)); + PurgeExecutionStats stats = new PurgeExecutionStats(validGuids, validGuids.size()); + EntityMutationResponse response = orchestratorSpy.executePurge(validGuids, failedEntities, stats, TEST_RUN_ID); + + assertNotNull(response.getPurgedEntities()); + assertEquals(response.getPurgedEntities().size(), 1); + assertEquals(response.getPurgedEntities().get(0).getGuid(), dependencyGuid); + assertNotNull(response.getFailedEntities()); + assertEquals(response.getFailedEntities().size(), 1); + assertEquals(response.getFailedEntities().get(0).getGuid(), rootGuid); + assertEquals(response.getFailedEntities().get(0).getErrorCode(), AtlasErrorCode.INTERNAL_ERROR.getErrorCode()); + assertEquals(response.getFailedEntities().get(0).getErrorMessage(), + PurgeBatchOrchestrator.UNPROCESSED_PURGE_GUID_MESSAGE); + assertTrue(Thread.interrupted(), "Expected interrupt flag to be set after shutdown interruption"); + } + + @Test + public void testExecutePurgeEmptyInputReturnsEmptyResponse() throws Exception { + when(mockExecutor.getEntityStore()).thenReturn(mock(AtlasEntityStore.class)); + + List failedEntities = new ArrayList<>(); + Set validGuids = new LinkedHashSet<>(); + PurgeExecutionStats stats = new PurgeExecutionStats(validGuids, 0); + EntityMutationResponse response = orchestrator.executePurge(validGuids, failedEntities, stats, TEST_RUN_ID); + + assertNull(response.getPurgedEntities()); + assertNull(response.getFailedEntities()); + verify(mockExecutor, never()).executeBatch(anySet()); + } + + @Test + public void testAttachPurgeSummarySetsExecutionFailedForNonInternalErrorBatchFailure() { + EntityMutationResponse response = new EntityMutationResponse(); + + String requestedGuid = "11111111-1111-1111-1111-111111111111"; + String failedGuid = "22222222-2222-2222-2222-222222222222"; + + AtlasEntityHeader purgedRequested = new AtlasEntityHeader(); + purgedRequested.setGuid(requestedGuid); + response.addEntity(EntityOperation.PURGE, purgedRequested); + + response.addFailedEntity(new FailedEntity( + failedGuid, + AtlasErrorCode.REFERENCED_ENTITY_NOT_FOUND.getErrorCode(), + "Referenced entity missing during batch purge")); + + Set originallyRequestedGuids = new LinkedHashSet<>(Arrays.asList(requestedGuid, failedGuid)); + attachPurgeSummaryFromResponse(response, originallyRequestedGuids, originallyRequestedGuids.size()); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertTrue(summary.getExecutionFailed()); + assertEquals(summary.getPurgedCount(), 1); + assertEquals(summary.getFailedCount(), 1); + assertEquals(summary.getSkippedCount(), 0); + } + + @Test + public void testAttachPurgeSummaryDoesNotSetExecutionFailedForPreScanFailures() { + EntityMutationResponse response = new EntityMutationResponse(); + + String purgedGuid = "11111111-1111-1111-1111-111111111111"; + String invalidGuid = "not-a-valid-uuid"; + + AtlasEntityHeader purgedRequested = new AtlasEntityHeader(); + purgedRequested.setGuid(purgedGuid); + response.addEntity(EntityOperation.PURGE, purgedRequested); + + response.addFailedEntity(new FailedEntity( + invalidGuid, + AtlasErrorCode.INVALID_GUID.getErrorCode(), + "invalid guid")); + + Set originallyRequestedGuids = new LinkedHashSet<>(Arrays.asList(purgedGuid, invalidGuid)); + attachPurgeSummaryFromResponse(response, originallyRequestedGuids, 1); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertFalse(summary.getExecutionFailed()); + assertEquals(summary.getPurgedCount(), 1); + assertEquals(summary.getFailedCount(), 1); + assertEquals(summary.getSkippedCount(), 0); + } + + @Test + public void testAttachPurgeSummaryCountsNonSkippableDependencyFailuresSeparately() { + EntityMutationResponse response = new EntityMutationResponse(); + + String requestedGuid = "11111111-1111-1111-1111-111111111111"; + String dependencyGuid = "22222222-2222-2222-2222-222222222222"; + + AtlasEntityHeader purgedRequested = new AtlasEntityHeader(); + purgedRequested.setGuid(requestedGuid); + response.addEntity(EntityOperation.PURGE, purgedRequested); + + AtlasEntityHeader purgedDependency = new AtlasEntityHeader(); + purgedDependency.setGuid(dependencyGuid); + response.addEntity(EntityOperation.PURGE, purgedDependency); + + response.addFailedEntity(new FailedEntity( + dependencyGuid, + AtlasErrorCode.INTERNAL_ERROR.getErrorCode(), + "dependency batch failure")); + + Set originallyRequestedGuids = new LinkedHashSet<>(Arrays.asList(requestedGuid)); + attachPurgeSummaryFromResponse(response, originallyRequestedGuids, 1); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertEquals(summary.getRequestedCount(), 1); + assertEquals(summary.getPurgedCount(), 1); + assertEquals(summary.getPurgedDependenciesCount(), 1); + assertEquals(summary.getFailedCount(), 0); + assertEquals(summary.getFailedDependenciesCount(), 0); + assertEquals(response.getFailedEntities().size(), 1); + } + + @Test + public void testAttachPurgeSummaryIgnoresSkippableDependencyFailures() { + EntityMutationResponse response = new EntityMutationResponse(); + + String requestedGuid = "11111111-1111-1111-1111-111111111111"; + String dependencyGuid = "22222222-2222-2222-2222-222222222222"; + + AtlasEntityHeader purgedRequested = new AtlasEntityHeader(); + purgedRequested.setGuid(requestedGuid); + response.addEntity(EntityOperation.PURGE, purgedRequested); + + response.addFailedEntity(new FailedEntity( + dependencyGuid, + AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode(), + "dependency already removed by concurrent batch")); + + Set originallyRequestedGuids = new LinkedHashSet<>(Arrays.asList(requestedGuid)); + attachPurgeSummaryFromResponse(response, originallyRequestedGuids, 1); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertEquals(summary.getRequestedCount(), 1); + assertEquals(summary.getPurgedCount(), 1); + assertEquals(summary.getPurgedDependenciesCount(), 0); + assertEquals(summary.getFailedCount(), 0); + assertEquals(summary.getFailedDependenciesCount(), 0); + assertEquals(summary.getSkippedDependenciesCount(), 1); + assertEquals(summary.getSkippedCount(), 1); + } + + @Test + public void testAttachPurgeSummaryCountsRequestedAndDependencyFailuresSeparately() { + EntityMutationResponse response = new EntityMutationResponse(); + + String requestedGuid = "11111111-1111-1111-1111-111111111111"; + String dependencyGuid = "22222222-2222-2222-2222-222222222222"; + + response.addFailedEntity(new FailedEntity( + requestedGuid, + AtlasErrorCode.INTERNAL_ERROR.getErrorCode(), + "requested root expand failure")); + response.addFailedEntity(new FailedEntity( + dependencyGuid, + AtlasErrorCode.INTERNAL_ERROR.getErrorCode(), + "dependency batch failure")); + + Set originallyRequestedGuids = new LinkedHashSet<>(Arrays.asList(requestedGuid)); + attachPurgeSummaryFromResponse(response, originallyRequestedGuids, 1); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertEquals(summary.getRequestedCount(), 1); + assertEquals(summary.getPurgedCount(), 0); + assertEquals(summary.getPurgedDependenciesCount(), 0); + assertEquals(summary.getFailedCount(), 1); + assertEquals(summary.getFailedDependenciesCount(), 1); + assertEquals(summary.getSkippedCount(), 0); + assertEquals(response.getFailedEntities().size(), 2); + } + + @Test + public void testAttachPurgeSummaryBalanceFormula() { + EntityMutationResponse response = new EntityMutationResponse(); + + String requestedGuid = "11111111-1111-1111-1111-111111111111"; + String dependencyGuid = "22222222-2222-2222-2222-222222222222"; + String failedDependencyGuid = "33333333-3333-3333-3333-333333333333"; + + response.addFailedEntity(new FailedEntity( + requestedGuid, + AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode(), + "Not found")); + response.addFailedEntity(new FailedEntity( + dependencyGuid, + AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode(), + "Dependency not found")); + response.addFailedEntity(new FailedEntity( + failedDependencyGuid, + AtlasErrorCode.INTERNAL_ERROR.getErrorCode(), + "Dependency failed")); + + Set originallyRequestedGuids = new LinkedHashSet<>(Collections.singletonList(requestedGuid)); + attachPurgeSummaryFromResponse(response, originallyRequestedGuids, 1); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertEquals(summary.getRequestedCount(), 1); + assertEquals(summary.getSkippedRequestedCount(), 1); + assertEquals(summary.getSkippedDependenciesCount(), 1); + assertEquals(summary.getSkippedCount(), 2); + assertEquals(summary.getFailedCount(), 0); + assertEquals(summary.getFailedDependenciesCount(), 1); + assertEquals(summary.getPurgedCount(), 0); + assertEquals(summary.getPurgedDependenciesCount(), 0); + + assertEquals(summary.getRequestedCount(), + summary.getPurgedCount() + summary.getFailedCount() + summary.getSkippedRequestedCount()); + } + + @Test + public void testAttachPurgeSummarySetsValidGuidCount() { + EntityMutationResponse response = new EntityMutationResponse(); + Set originallyRequestedGuids = new LinkedHashSet<>( + Collections.singletonList("11111111-1111-1111-1111-111111111111")); + + attachPurgeSummaryFromResponse(response, originallyRequestedGuids, 1); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertEquals(summary.getValidGuidCount(), 1); + } + + @Test + public void testReconcileUnprocessedGuidsUpdatesExecutionStats() { + Set originallyRequestedGuids = new LinkedHashSet<>(Arrays.asList( + "11111111-1111-1111-1111-111111111111", + "22222222-2222-2222-2222-222222222222", + "33333333-3333-3333-3333-333333333333")); + PurgeExecutionStats stats = new PurgeExecutionStats(originallyRequestedGuids, originallyRequestedGuids.size()); + stats.getProducedDeletionCandidates().addAll(originallyRequestedGuids); + + EntityMutationResponse response = new EntityMutationResponse(); + int reconciledCount = PurgeBatchOrchestrator.reconcileUnprocessedGuids( + stats.getProducedDeletionCandidates(), response, null, stats); + + assertEquals(reconciledCount, 3); + assertEquals(stats.getReconciledUnprocessedCount(), 3); + assertEquals(stats.getFailedCount(), 3); + PurgeUtils.attachPurgeSummary(response, stats, null); + assertEquals(response.getPurgeSummary().getUnprocessedCount(), 3); + } + + @Test + public void testExecutePurgeExpandedWorkloadAccounting() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + when(mockExecutor.getEntityStore()).thenReturn(mockStore); + + Set requestedGuids = new LinkedHashSet<>(); + for (int i = 0; i < 10; i++) { + requestedGuids.add(String.format("11111111-1111-1111-1111-%012d", i)); + } + + when(mockStore.accumulateDeletionCandidates(any())).thenAnswer(invocation -> { + Set root = invocation.getArgument(0); + String rootGuid = root.iterator().next(); + Set deps = new LinkedHashSet<>(); + for (int j = 0; j < 4; j++) { + deps.add(rootGuid.replaceFirst("1111", String.format("%04d", j + 2))); + } + return deps; + }); + + when(mockExecutor.executeBatch(anySet())).thenAnswer(invocation -> { + EntityMutationResponse batchResponse = new EntityMutationResponse(); + Set batchGuids = invocation.getArgument(0); + for (String guid : batchGuids) { + AtlasEntityHeader header = new AtlasEntityHeader(); + header.setGuid(guid); + batchResponse.addEntity(EntityOperation.PURGE, header); + } + return batchResponse; + }); + + PurgeExecutionStats stats = new PurgeExecutionStats(requestedGuids, requestedGuids.size()); + EntityMutationResponse response = orchestrator.executePurge(requestedGuids, new ArrayList<>(), stats, TEST_RUN_ID); + PurgeUtils.attachPurgeSummary(response, stats, null); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertEquals(summary.getRequestedCount(), 10); + assertEquals(summary.getExpandedEntityCount(), 50); + assertEquals(summary.getPurgedCount(), 10); + assertEquals(summary.getPurgedDependenciesCount(), 40); + assertEquals(summary.getUnprocessedCount(), 0); + assertTrue(summary.getBatchCount() > 0); + assertEquals(summary.getExpandedEntityCount(), + summary.getPurgedCount() + summary.getPurgedDependenciesCount()); + } + + @Test + public void testMultiWorkerPurgeStatsAccounting() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + when(mockExecutor.getEntityStore()).thenReturn(mockStore); + + Set requestedGuids = new LinkedHashSet<>(); + for (int i = 0; i < 6; i++) { + requestedGuids.add(String.format("11111111-1111-1111-1111-%012d", i)); + } + + when(mockStore.accumulateDeletionCandidates(any())).thenAnswer(invocation -> { + Set root = invocation.getArgument(0); + return new LinkedHashSet<>(root); + }); + + when(mockExecutor.executeBatch(anySet())).thenAnswer(invocation -> { + Set batchGuids = invocation.getArgument(0); + EntityMutationResponse batchResponse = new EntityMutationResponse(); + for (String guid : batchGuids) { + if ("11111111-1111-1111-1111-000000000003".equals(guid)) { + batchResponse.addFailedEntity(new FailedEntity(guid, + AtlasErrorCode.INTERNAL_ERROR.getErrorCode(), "batch failure")); + } else { + AtlasEntityHeader header = new AtlasEntityHeader(); + header.setGuid(guid); + batchResponse.addEntity(EntityOperation.PURGE, header); + } + } + return batchResponse; + }); + + PurgeBatchOrchestrator orchestratorSpy = spy(orchestrator); + doAnswer(invocation -> orchestrator.createManager(1, 2, TEST_RUN_ID)) + .when(orchestratorSpy).createManager(anyInt(), anyInt(), anyString()); + + PurgeExecutionStats stats = new PurgeExecutionStats(requestedGuids, requestedGuids.size()); + EntityMutationResponse response = orchestratorSpy.executePurge(requestedGuids, new ArrayList<>(), stats, TEST_RUN_ID); + PurgeUtils.attachPurgeSummary(response, stats, null); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertEquals(summary.getRequestedCount(), 6); + assertEquals(summary.getPurgedCount(), 5); + assertEquals(summary.getFailedCount(), 1); + assertTrue(summary.getExecutionFailed()); + assertTrue(summary.getBatchCount() >= 6); + } +} diff --git a/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java b/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java index e15012128b8..76417c74a16 100644 --- a/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java @@ -18,18 +18,38 @@ package org.apache.atlas.services; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.DeleteType; +import org.apache.atlas.GraphTransactionInterceptor; import org.apache.atlas.RequestContext; +import org.apache.atlas.TestModules; +import org.apache.atlas.TestUtilsV2; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.AtlasAuditEntry; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditRowKind; +import org.apache.atlas.model.audit.AuditSearchParameters; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.FailedEntity; +import org.apache.atlas.model.instance.PurgeSummary; +import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.AtlasTestBase; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.audit.AtlasAuditService; +import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasElement; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.purge.PurgeExecutionStats; +import org.apache.atlas.repository.purge.PurgeUtils; +import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; @@ -37,27 +57,63 @@ import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; -import org.apache.atlas.utils.TestLoadModelUtils; +import org.apache.atlas.utils.TestResourceFileUtils; import org.apache.commons.lang3.RandomStringUtils; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; +import org.testng.SkipException; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; +import java.io.IOException; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; -@Guice(modules = org.apache.atlas.TestModules.TestOnlyModule.class) +/** + * Integration tests for {@link PurgeService}: REST purgeByIds, scheduled purgeEntities, + * cron failure handling, REST/cron overlap, and delete-then-purge audit correlation. + */ +@Guice(modules = TestModules.TestOnlyModule.class) public class PurgeServiceTest extends AtlasTestBase { + private static final String CRON_ELIGIBLE_GUID = "11111111-1111-1111-1111-111111111111"; + private static final String CLIENT_HOST = "127.0.0.0"; + private static final String DEFAULT_USER = "Admin"; + private static final String AUDIT_PARAMETER_RESOURCE_DIR = "auditSearchParameters"; + @Inject private AtlasTypeDefStore typeDefStore; @@ -70,68 +126,451 @@ public class PurgeServiceTest extends AtlasTestBase { @Inject private AtlasGraph atlasGraph; + @Inject + private AtlasAuditService atlasAuditService; + @BeforeClass - public void setup() throws Exception { + public void setupClass() throws Exception { RequestContext.clear(); super.initialize(); - TestLoadModelUtils.loadBaseModel(typeDefStore, typeRegistry); - TestLoadModelUtils.loadHiveModel(typeDefStore, typeRegistry); - try { - Thread.sleep(1000); - } catch (InterruptedException ignored) { } + basicSetup(typeDefStore, typeRegistry); + Thread.sleep(1000); + } + + @BeforeMethod + public void setupMethod() { + RequestContext.clear(); + RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); + } + + @AfterClass + public void tearDownClass() throws Exception { + Thread.sleep(1000); + AtlasGraphProvider.cleanup(); + super.cleanup(); } + // ------------------------------------------------------------------------- + // Scheduled purge (PurgeService.purgeEntities) + // ------------------------------------------------------------------------- + @Test - public void testPurgeEntities() throws Exception { - // Approach (pre-commit check on async flow): - // - Spy store's IAtlasEntityChangeNotifier and capture onEntitiesMutated(response,false) to assert - // the purged GUID before commit/locking paths. - // Create DB and table + public void scheduledPurge_purgesEligibleDeletedEntity() throws Exception { AtlasEntity db = newHiveDb(null); - String dbGuid = persistAndGetGuid(db); + persistAndGetGuid(db); AtlasEntity tbl = newHiveTable(db, null); String tblGuid = persistAndGetGuid(tbl); - // Soft-delete table - EntityMutationResponse del = entityStore.deleteByIds(Collections.singletonList(tblGuid)); - assertNotNull(del); + RequestContext.clear(); + entityStore.deleteByIds(Collections.singletonList(tblGuid)); - // Backdate timestamp beyond default 30d retention and reindex to reflect in Solr backdateModificationTimestamp(tblGuid, 31); reindexVertices(tblGuid); pauseForIndexCreation(); - // Enable hive purge and single worker for determinism ApplicationProperties.get().setProperty("atlas.purge.enabled.services", "hive"); ApplicationProperties.get().setProperty("atlas.purge.workers.count", "1"); ApplicationProperties.get().setProperty("atlas.purge.worker.batch.size", "1"); - // Clear context to avoid skipping due to prior delete tracking RequestContext.clear(); - // Inject notifier spy to capture pre-commit signal (pre-commit verification point) Object originalNotifier = injectNotifierSpy(entityStore); + EntityMutationResponse purgeResponse = createPurgeService().purgeEntities(); - new PurgeService(atlasGraph, entityStore, typeRegistry).purgeEntities(); - - // Verify notifier call and capture response before transaction commit (async-safe via timeout) IAtlasEntityChangeNotifier spy = (IAtlasEntityChangeNotifier) getNotifier(entityStore); ArgumentCaptor cap = ArgumentCaptor.forClass(EntityMutationResponse.class); - Mockito.verify(spy, Mockito.timeout(5000)).onEntitiesMutated(cap.capture(), Mockito.eq(false)); - - EntityMutationResponse notified = cap.getValue(); - assertNotNull(notified); - List purged = notified.getPurgedEntities(); - assertNotNull(purged); + Mockito.verify(spy, Mockito.timeout(5000).atLeastOnce()) + .onEntitiesMutated(cap.capture(), Mockito.eq(false)); - assertTrue(purged.stream().anyMatch(h -> tblGuid.equals(h.getGuid()))); + List allPurged = new ArrayList<>(); + for (EntityMutationResponse notified : cap.getAllValues()) { + assertNotNull(notified); + List batchPurged = notified.getPurgedEntities(); + if (batchPurged != null) { + allPurged.addAll(batchPurged); + } + } - // Restore original notifier to leave the store in a clean state + assertTrue(allPurged.stream().anyMatch(h -> tblGuid.equals(h.getGuid()))); restoreNotifier(entityStore, originalNotifier); - // Flag assertions - assertTrue(RequestContext.get().isPurgeRequested()); - assertEquals(RequestContext.get().getDeleteType(), DeleteType.HARD); + assertNotNull(purgeResponse); + assertNotNull(purgeResponse.getPurgeSummary()); + assertTrue(purgeResponse.getPurgeSummary().getRequestedCount() > 0, + "Expected index scan to find at least one purge-eligible entity"); + + List responsePurged = purgeResponse.getPurgedEntities(); + if (responsePurged != null && !responsePurged.isEmpty()) { + long totalPurgedInSummary = purgeResponse.getPurgeSummary().getPurgedCount() + + purgeResponse.getPurgeSummary().getPurgedDependenciesCount(); + assertTrue(totalPurgedInSummary > 0, + "Expected purge summary to report at least one purged entity"); + assertTrue(responsePurged.stream().anyMatch(h -> tblGuid.equals(h.getGuid()))); + } + + assertFalse(RequestContext.get().isPurgeRequested(), + "Scheduled purge should reset purgeRequested on the cron thread"); + assertEquals(RequestContext.get().getDeleteType(), DeleteType.DEFAULT, + "Scheduled purge should reset delete type on the cron thread"); + } + + @Test + public void scheduledPurge_purgesMultipleIndexHits() throws Exception { + AtlasEntity db = newHiveDb(null); + persistAndGetGuid(db); + + String tbl1Guid = persistAndGetGuid(newHiveTable(db, null)); + String tbl2Guid = persistAndGetGuid(newHiveTable(db, null)); + + RequestContext.clear(); + entityStore.deleteByIds(Collections.singletonList(tbl1Guid)); + entityStore.deleteByIds(Collections.singletonList(tbl2Guid)); + + backdateModificationTimestamp(tbl1Guid, 31); + backdateModificationTimestamp(tbl2Guid, 31); + reindexVertices(tbl1Guid, tbl2Guid); + pauseForIndexCreation(); + + ApplicationProperties.get().setProperty("atlas.purge.enabled.services", "hive"); + ApplicationProperties.get().setProperty("atlas.purge.workers.count", "1"); + ApplicationProperties.get().setProperty("atlas.purge.worker.batch.size", "1"); + + RequestContext.clear(); + + EntityMutationResponse purgeResponse = createPurgeService().purgeEntities(); + + assertNotNull(purgeResponse); + assertNotNull(purgeResponse.getPurgeSummary()); + assertEquals(purgeResponse.getPurgeSummary().getRequestedCount(), 2, + "Expected two index hits for two purge-eligible tables"); + + Set purgedGuids = new HashSet<>(); + if (purgeResponse.getPurgedEntities() != null) { + for (AtlasEntityHeader header : purgeResponse.getPurgedEntities()) { + purgedGuids.add(header.getGuid()); + } + } + + assertTrue(purgedGuids.contains(tbl1Guid), "Expected first table to be purged"); + assertTrue(purgedGuids.contains(tbl2Guid), "Expected second table to be purged"); + assertNull(findByGuidFresh(tbl1Guid), "First table should be removed from graph"); + assertNull(findByGuidFresh(tbl2Guid), "Second table should be removed from graph"); + + assertTrue(RequestContext.get().getDeletedEntities().isEmpty(), + "Cron producer thread should clear expansion delete records after each index hit"); + assertFalse(RequestContext.get().isPurgeRequested()); + assertEquals(RequestContext.get().getDeleteType(), DeleteType.DEFAULT); + } + + /** + * Design v3 allows REST purge while scheduled purge is in progress. Overlapping GUIDs must be + * purged exactly once across both paths. + */ + @Test + public void concurrentScheduledAndRestPurgeOverlappingGuids() throws Exception { + AtlasEntity db = newHiveDb(null); + String dbGuid = persistAndGetGuid(db); + String tbl1Guid = persistAndGetGuid(newHiveTable(db, null)); + String tbl2Guid = persistAndGetGuid(newHiveTable(db, null)); + + RequestContext.clear(); + entityStore.deleteByIds(Arrays.asList(tbl1Guid, tbl2Guid)); + + backdateModificationTimestamp(tbl1Guid, 31); + backdateModificationTimestamp(tbl2Guid, 31); + reindexVertices(tbl1Guid, tbl2Guid); + pauseForIndexCreation(); + + ApplicationProperties.get().setProperty("atlas.purge.enabled.services", "hive"); + ApplicationProperties.get().setProperty("atlas.purge.workers.count", "2"); + ApplicationProperties.get().setProperty("atlas.purge.worker.batch.size", "1"); + + RequestContext.clear(); + GraphTransactionInterceptor.clearCache(); + try { + atlasGraph.rollback(); + } catch (Exception ignored) { } + + Set overlapGuids = new HashSet<>(Arrays.asList(tbl1Guid, tbl2Guid)); + CountDownLatch startGate = new CountDownLatch(1); + AtomicReference scheduledResponse = new AtomicReference<>(); + AtomicReference restResponse = new AtomicReference<>(); + AtomicReference scheduledError = new AtomicReference<>(); + AtomicReference restError = new AtomicReference<>(); + + PurgeService purgeService = createPurgeService(); + + Thread scheduledThread = new Thread(() -> { + try { + startGate.await(); + RequestContext.clear(); + scheduledResponse.set(purgeService.purgeEntities()); + } catch (Exception e) { + scheduledError.set(e); + } finally { + RequestContext.clear(); + GraphTransactionInterceptor.clearCache(); + } + }, "scheduled-purge-overlap"); + + Thread restThread = new Thread(() -> { + try { + startGate.await(); + RequestContext.clear(); + RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); + restResponse.set(purgeService.purgeByIds(overlapGuids)); + } catch (Exception e) { + restError.set(e); + } finally { + RequestContext.clear(); + GraphTransactionInterceptor.clearCache(); + } + }, "rest-purge-overlap"); + + scheduledThread.start(); + restThread.start(); + startGate.countDown(); + scheduledThread.join(TimeUnit.SECONDS.toMillis(120)); + restThread.join(TimeUnit.SECONDS.toMillis(120)); + + if (scheduledError.get() != null) { + throw scheduledError.get(); + } + if (restError.get() != null) { + throw restError.get(); + } + + EntityMutationResponse scheduledResult = scheduledResponse.get(); + EntityMutationResponse restResult = restResponse.get(); + assertNotNull(scheduledResult, "Scheduled purge should return a response"); + assertNotNull(restResult, "REST purge should return a response"); + + assertNoInternalErrorPurgeFailures(scheduledResult, "scheduled purge"); + assertNoInternalErrorPurgeFailures(restResult, "REST purge"); + + Map purgedCountByGuid = new HashMap<>(); + collectPurgedGuidCounts(scheduledResult, purgedCountByGuid); + collectPurgedGuidCounts(restResult, purgedCountByGuid); + + for (String guid : overlapGuids) { + assertEquals(purgedCountByGuid.getOrDefault(guid, 0).intValue(), 1, + "Each overlapping GUID should be purged exactly once across both paths"); + assertNull(findByGuidFresh(guid), "Purged table should be removed from graph: " + guid); + } + + assertConcurrentOverlapFailuresAreSkippable(scheduledResult, overlapGuids); + assertConcurrentOverlapFailuresAreSkippable(restResult, overlapGuids); + assertEquals(countFailures(scheduledResult, false) + countFailures(restResult, false), 0, + "Concurrent overlap should not produce non-skippable purge failures"); + + assertNotNull(findByGuidFresh(dbGuid), "Parent DB should remain after table purge"); + assertNoOrphanEdgesToPurgedGuids(dbGuid, overlapGuids); + } + + @Test + public void cronFailure_beforeAnyGuidsCollected_skipsSummaryAudit() throws Exception { + AtlasGraph mockGraph = mock(AtlasGraph.class); + AtlasAuditService mockAuditService = mock(AtlasAuditService.class); + PurgeService purgeService = new PurgeService(mockGraph, mock(AtlasEntityStore.class), + mock(AtlasTypeRegistry.class), mockAuditService); + + when(mockGraph.indexQuery(eq(Constants.VERTEX_INDEX), anyString())) + .thenThrow(new RuntimeException("index query failed")); + + EntityMutationResponse response = purgeService.purgeEntities(); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertTrue(summary.getExecutionFailed()); + assertNull(summary.getRunId()); + + verify(mockAuditService, never()).add(eq(AuditOperation.AUTO_PURGE), anyString(), anyString(), + anyLong(), anyString(), eq(AuditRowKind.SUMMARY)); + } + + @Test + public void cronFailure_afterGuidsCollected_writesSummaryAuditWithRunId() throws Exception { + AtlasAuditService mockAuditService = mock(AtlasAuditService.class); + PurgeService purgeService = new PurgeService(mock(AtlasGraph.class), mock(AtlasEntityStore.class), + mock(AtlasTypeRegistry.class), mockAuditService); + + EntityMutationResponse response = new EntityMutationResponse(); + Set originallyRequestedGuids = new LinkedHashSet<>(Collections.singleton(CRON_ELIGIBLE_GUID)); + PurgeExecutionStats stats = new PurgeExecutionStats(originallyRequestedGuids, originallyRequestedGuids.size()); + stats.markExecutionFailed(); + + invokeHandleCronPurgeFailure(purgeService, response, stats, originallyRequestedGuids); + + PurgeSummary summary = response.getPurgeSummary(); + assertNotNull(summary); + assertTrue(summary.getExecutionFailed()); + assertNotNull(summary.getRunId()); + assertTrue(summary.getRequestedCount() > 0); + + verify(mockAuditService).add(eq(AuditOperation.AUTO_PURGE), eq(CRON_ELIGIBLE_GUID), anyString(), + eq(0L), eq(summary.getRunId()), eq(AuditRowKind.SUMMARY)); + } + + // ------------------------------------------------------------------------- + // End-to-end delete + purge + audit search + // ------------------------------------------------------------------------- + + @Test + public void deleteThenPurge_writesAuditsSearchableByAdminFilters() throws Exception { + AtlasTypesDef sampleTypes = TestUtilsV2.defineDeptEmployeeTypes(); + AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(sampleTypes, typeRegistry); + + if (!typesToCreate.isEmpty()) { + typeDefStore.createTypesDef(typesToCreate); + } + + AtlasEntitiesWithExtInfo deptEg2 = TestUtilsV2.createDeptEg2(); + AtlasEntityStream entityStream = new AtlasEntityStream(deptEg2); + EntityMutationResponse emr = entityStore.createOrUpdate(entityStream, false); + + pauseForIndexCreation(); + + assertNotNull(emr); + assertNotNull(emr.getCreatedEntities()); + assertFalse(emr.getCreatedEntities().isEmpty()); + + List guids = emr.getCreatedEntities().stream() + .map(AtlasEntityHeader::getGuid) + .collect(Collectors.toList()); + + EntityMutationResponse deleteResponse = entityStore.deleteByIds(guids); + pauseForIndexCreation(); + + assertSortedGuidsMatch(emr.getCreatedEntities(), deleteResponse.getDeletedEntities(), "deleteByIds"); + + ApplicationProperties.get().setProperty("atlas.purge.workers.count", "1"); + + Date startTimestamp = new Date(); + EntityMutationResponse purgeResponse = createPurgeService().purgeByIds(new HashSet<>(guids)); + + pauseForIndexCreation(); + assertPurgeSucceededForRequestedGuids(guids, purgeResponse); + + atlasAuditService.add(DEFAULT_USER, AuditOperation.PURGE, CLIENT_HOST, startTimestamp, new Date(), + guids.toString(), purgeResponse.getPurgedEntitiesIds(), purgeResponse.getPurgedEntities().size()); + + assertAuditEntry(atlasAuditService, createAuditParameter("audit-search-parameter-without-filter")); + assertAuditEntry(atlasAuditService, createAuditParameter("audit-search-parameter-purge")); + assertPurgeAuditRowsWrittenByPurgeService(createAuditParameter("audit-search-parameter-purge")); + } + + // ------------------------------------------------------------------------- + // PurgeService.purgeByIds — pre-validation and orchestration + // ------------------------------------------------------------------------- + + @Test + public void purgeByIds_rejectsEmptyOrNullGuids() throws Exception { + try { + createPurgeService().purgeByIds(new HashSet<>()); + fail("Expected AtlasBaseException for empty GUID set"); + } catch (AtlasBaseException e) { + assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INVALID_PARAMETERS); + } + + try { + createPurgeService().purgeByIds(null); + fail("Expected AtlasBaseException for null GUID set"); + } catch (AtlasBaseException e) { + assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INVALID_PARAMETERS); + } + } + + @Test + public void purgeByIds_preScanNonExistentEntities() throws Exception { + Set guids = new HashSet<>(Arrays.asList( + "11111111-1111-1111-1111-111111111111", + "22222222-2222-2222-2222-222222222222")); + EntityMutationResponse response = createPurgeService().purgeByIds(guids); + + assertNotNull(response); + assertTrue(response.getPurgedEntities() == null || response.getPurgedEntities().isEmpty()); + assertNotNull(response.getFailedEntities()); + assertEquals(response.getFailedEntities().size(), 2); + assertEquals(response.getFailedEntities().get(0).getErrorCode(), + AtlasErrorCode.INSTANCE_GUID_NOT_FOUND.getErrorCode()); + assertNotNull(response.getPurgeSummary()); + assertEquals(response.getPurgeSummary().getRequestedCount(), 2); + assertEquals(response.getPurgeSummary().getSkippedCount(), 2); + } + + @Test + public void purgeByIds_invalidUuid() throws Exception { + EntityMutationResponse response = createPurgeService().purgeByIds( + new HashSet<>(Collections.singletonList("invalid-uuid-format"))); + + assertNotNull(response.getFailedEntities()); + assertEquals(response.getFailedEntities().size(), 1); + assertEquals(response.getFailedEntities().get(0).getErrorCode(), AtlasErrorCode.INVALID_GUID.getErrorCode()); + assertEquals(response.getPurgeSummary().getFailedCount(), 1); + } + + @Test + public void purgeByIds_notInDeletedState() throws Exception { + String guid = persistAndGetGuid(newHiveDb(null)); + + EntityMutationResponse response = createPurgeService().purgeByIds(Collections.singleton(guid)); + + assertNotNull(response.getFailedEntities()); + assertEquals(response.getFailedEntities().size(), 1); + assertEquals(response.getFailedEntities().get(0).getErrorCode(), + AtlasErrorCode.NOT_IN_DELETED_STATE.getErrorCode()); + assertEquals(response.getPurgeSummary().getSkippedCount(), 1); + } + + @Test + public void purgeByIds_success() throws Exception { + ApplicationProperties.get().setProperty("atlas.purge.workers.count", "1"); + String guid = persistAndGetGuid(newHiveDb(null)); + entityStore.deleteById(guid); + + EntityMutationResponse response = createPurgeService().purgeByIds(Collections.singleton(guid)); + + assertNotNull(response.getPurgedEntities()); + assertEquals(response.getPurgedEntities().size(), 1); + assertEquals(response.getPurgedEntities().get(0).getGuid(), guid); + assertEquals(response.getPurgeSummary().getPurgedCount(), 1); + } + + @Test + public void purgeByIds_partialPreScanAndPurge() throws Exception { + ApplicationProperties.get().setProperty("atlas.purge.workers.count", "1"); + String guid = persistAndGetGuid(newHiveDb(null)); + entityStore.deleteById(guid); + + Set guids = new HashSet<>(Arrays.asList(guid, "22222222-2222-2222-2222-222222222222")); + EntityMutationResponse response = createPurgeService().purgeByIds(guids); + + assertNotNull(response.getPurgedEntities()); + assertEquals(response.getPurgedEntities().size(), 1); + assertEquals(response.getFailedEntities().size(), 1); + assertEquals(response.getPurgeSummary().getPurgedCount(), 1); + assertEquals(response.getPurgeSummary().getSkippedCount(), 1); + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + private PurgeService createPurgeService() { + return new PurgeService(atlasGraph, entityStore, typeRegistry, atlasAuditService); + } + + private static void invokeHandleCronPurgeFailure(PurgeService purgeService, + EntityMutationResponse response, + PurgeExecutionStats stats, + Set originallyRequestedGuids) throws Exception { + Method handleCronPurgeFailure = PurgeService.class.getDeclaredMethod( + "handleCronPurgeFailure", + EntityMutationResponse.class, + PurgeExecutionStats.class, + Set.class); + handleCronPurgeFailure.setAccessible(true); + handleCronPurgeFailure.invoke(purgeService, response, stats, originallyRequestedGuids); } private AtlasEntity newHiveDb(String nameOpt) { @@ -158,11 +597,15 @@ private AtlasEntity newHiveTable(AtlasEntity db, String nameOpt) { } private String persistAndGetGuid(AtlasEntity entity) throws AtlasBaseException { - EntityMutationResponse resp = entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntityWithExtInfo(entity)), false); - String typeName = entity.getTypeName(); - AtlasEntityHeader hdr = resp.getFirstCreatedEntityByTypeName(typeName); - String guid = hdr != null ? hdr.getGuid() : null; - return guid; + EntityMutationResponse resp = entityStore.createOrUpdate( + new AtlasEntityStream(new AtlasEntityWithExtInfo(entity)), false); + AtlasEntityHeader hdr = resp.getFirstCreatedEntityByTypeName(entity.getTypeName()); + return hdr != null ? hdr.getGuid() : null; + } + + private AtlasVertex findByGuidFresh(String guid) { + GraphTransactionInterceptor.clearCache(); + return AtlasGraphUtilsV2.findByGuid(atlasGraph, guid); } private void backdateModificationTimestamp(String guid, int days) { @@ -171,6 +614,8 @@ private void backdateModificationTimestamp(String guid, int days) { long delta = days * 24L * 60 * 60 * 1000; long ts = System.currentTimeMillis() - delta; AtlasGraphUtilsV2.setProperty(v, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, ts); + atlasGraph.commit(); + GraphTransactionInterceptor.clearCache(); } } @@ -189,12 +634,12 @@ private void reindexVertices(String... guids) { try { atlasGraph.getManagementSystem().reindex(Constants.VERTEX_INDEX, elements); atlasGraph.getManagementSystem().reindex(Constants.FULLTEXT_INDEX, elements); + atlasGraph.commit(); + GraphTransactionInterceptor.clearCache(); } catch (Exception ignored) { } } } - // Test helper: swap store's private notifier field with a Mockito spy so we can - // capture and assert the pre-commit mutation response invoked by the store. private Object injectNotifierSpy(AtlasEntityStoreV2 storeV2) throws Exception { Field f = AtlasEntityStoreV2.class.getDeclaredField("entityChangeNotifier"); f.setAccessible(true); @@ -204,17 +649,184 @@ private Object injectNotifierSpy(AtlasEntityStoreV2 storeV2) throws Exception { return original; } - // Test helper: fetch the (spied) notifier instance currently installed on the store. private Object getNotifier(AtlasEntityStoreV2 storeV2) throws Exception { Field f = AtlasEntityStoreV2.class.getDeclaredField("entityChangeNotifier"); f.setAccessible(true); return f.get(storeV2); } - // Test helper: restore the original notifier instance after verification. private void restoreNotifier(AtlasEntityStoreV2 storeV2, Object original) throws Exception { Field f = AtlasEntityStoreV2.class.getDeclaredField("entityChangeNotifier"); f.setAccessible(true); f.set(storeV2, original); } + + private static void collectPurgedGuidCounts(EntityMutationResponse response, Map purgedCountByGuid) { + if (response == null || response.getPurgedEntities() == null) { + return; + } + + for (AtlasEntityHeader header : response.getPurgedEntities()) { + purgedCountByGuid.merge(header.getGuid(), 1, Integer::sum); + } + } + + private static void assertNoInternalErrorPurgeFailures(EntityMutationResponse response, String pathLabel) { + if (response.getFailedEntities() == null) { + return; + } + + for (FailedEntity failedEntity : response.getFailedEntities()) { + assertFalse(AtlasErrorCode.INTERNAL_ERROR.getErrorCode().equals(failedEntity.getErrorCode()), + pathLabel + " should not record INTERNAL_ERROR for overlapping GUID handling: " + + failedEntity.getGuid()); + } + } + + private static void assertConcurrentOverlapFailuresAreSkippable(EntityMutationResponse response, + Set overlapGuids) { + if (response.getFailedEntities() == null) { + return; + } + + for (FailedEntity failedEntity : response.getFailedEntities()) { + assertTrue(overlapGuids.contains(failedEntity.getGuid()), + "Concurrent overlap failure should reference an overlapping GUID: " + failedEntity.getGuid()); + assertTrue(PurgeUtils.isSkippablePurgeFailureCode(failedEntity.getErrorCode()), + "Concurrent overlap failure must be skippable, not " + failedEntity.getErrorCode()); + } + } + + private static int countFailures(EntityMutationResponse response, boolean skippable) { + if (response.getFailedEntities() == null) { + return 0; + } + + int count = 0; + for (FailedEntity failedEntity : response.getFailedEntities()) { + if (PurgeUtils.isSkippablePurgeFailureCode(failedEntity.getErrorCode()) == skippable) { + count++; + } + } + + return count; + } + + private void assertNoOrphanEdgesToPurgedGuids(String anchorGuid, Set purgedGuids) { + AtlasVertex anchorVertex = findByGuidFresh(anchorGuid); + assertNotNull(anchorVertex, "Anchor vertex should exist for orphan-edge check"); + + Iterator edges = anchorVertex.getEdges(AtlasEdgeDirection.BOTH).iterator(); + while (edges.hasNext()) { + AtlasEdge edge = edges.next(); + String outGuid = AtlasGraphUtilsV2.getIdFromVertex(edge.getOutVertex()); + String inGuid = AtlasGraphUtilsV2.getIdFromVertex(edge.getInVertex()); + String otherGuid = anchorGuid.equals(outGuid) ? inGuid : outGuid; + + assertFalse(purgedGuids.contains(otherGuid), + "Anchor vertex should not retain edges to purged GUID: " + otherGuid); + } + } + + private void assertSortedGuidsMatch(List expected, List actual, String operation) { + assertNotNull(actual, operation + " returned null entities"); + assertEquals(toSortedGuidList(actual), toSortedGuidList(expected), operation + " guid mismatch"); + } + + private void assertPurgeSucceededForRequestedGuids(List requestedGuids, EntityMutationResponse response) { + assertNotNull(response.getPurgedEntities(), "purgeByIds returned no purged entities"); + assertNotNull(response.getPurgeSummary(), "purgeByIds returned no summary"); + + Set purgedGuids = response.getPurgedEntities().stream() + .map(AtlasEntityHeader::getGuid) + .collect(Collectors.toSet()); + + Set skippedGuids = new HashSet<>(); + if (response.getFailedEntities() != null) { + for (FailedEntity failedEntity : response.getFailedEntities()) { + if (PurgeUtils.isSkippablePurgeFailureCode(failedEntity.getErrorCode())) { + skippedGuids.add(failedEntity.getGuid()); + } + } + } + + for (String guid : requestedGuids) { + assertTrue(purgedGuids.contains(guid) || skippedGuids.contains(guid), + "Expected requested guid to be purged or skipped as already removed: " + guid); + } + + assertEquals(response.getPurgeSummary().getRequestedCount(), requestedGuids.size()); + assertEquals(response.getPurgeSummary().getPurgedCount() + response.getPurgeSummary().getSkippedRequestedCount(), + requestedGuids.size()); + assertEquals(response.getPurgeSummary().getFailedCount(), 0); + } + + private static List toSortedGuidList(List headers) { + return headers.stream() + .map(AtlasEntityHeader::getGuid) + .sorted() + .collect(Collectors.toList()); + } + + private AuditSearchParameters createAuditParameter(String fileName) { + try { + return TestResourceFileUtils.readObjectFromJson(AUDIT_PARAMETER_RESOURCE_DIR, fileName, AuditSearchParameters.class); + } catch (IOException e) { + fail(e.getMessage()); + } + + return null; + } + + private void assertPurgeAuditRowsWrittenByPurgeService(AuditSearchParameters auditSearchParameters) { + pauseForIndexCreation(); + + List result; + + try { + result = atlasAuditService.get(auditSearchParameters); + } catch (Exception e) { + throw new SkipException("purge audit entries not retrieved."); + } + + assertNotNull(result); + assertFalse(result.isEmpty()); + + boolean hasSummaryRow = false; + boolean hasBatchRow = false; + for (AtlasAuditEntry entry : result) { + if (entry.getOperation() != AuditOperation.PURGE && entry.getOperation() != AuditOperation.AUTO_PURGE) { + continue; + } + + if (PurgeUtils.isPurgeSummaryAudit(entry)) { + hasSummaryRow = true; + assertNotNull(entry.getRunId(), "Purge summary audit should carry runId"); + assertNotNull(PurgeUtils.parsePurgeSummary(entry), "Summary row should contain parseable PurgeSummary JSON"); + } + + if (PurgeUtils.isPurgeBatchAudit(entry)) { + hasBatchRow = true; + assertNotNull(entry.getRunId(), "Purge batch audit should carry runId"); + } + } + + assertTrue(hasSummaryRow, "Expected at least one purge summary audit row from purgeByIds"); + assertTrue(hasBatchRow, "Expected at least one purge batch audit row from purgeByIds"); + } + + private void assertAuditEntry(AtlasAuditService auditService, AuditSearchParameters auditSearchParameters) { + pauseForIndexCreation(); + + List result; + + try { + result = auditService.get(auditSearchParameters); + } catch (Exception e) { + throw new SkipException("audit entries not retrieved."); + } + + assertNotNull(result); + assertFalse(result.isEmpty()); + } } diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index db9fc4a110f..e311119947b 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -51,6 +51,7 @@ import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.PurgeSummary; import org.apache.atlas.model.metrics.AtlasMetrics; import org.apache.atlas.model.metrics.AtlasMetricsMapToChart; import org.apache.atlas.model.metrics.AtlasMetricsStat; @@ -66,6 +67,7 @@ import org.apache.atlas.repository.impexp.MigrationProgressService; import org.apache.atlas.repository.impexp.ZipSink; import org.apache.atlas.repository.patches.AtlasPatchManager; +import org.apache.atlas.repository.purge.PurgeUtils; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.server.common.filters.AtlasCSRFPreventionFilter; import org.apache.atlas.server.common.service.ServiceState; @@ -210,11 +212,11 @@ public class AdminResource { @Inject public AdminResource(ServiceState serviceState, MetricsService metricsService, AtlasTypeRegistry typeRegistry, - ExportService exportService, ImportService importService, SearchTracker activeSearches, - MigrationProgressService migrationProgressService, AtlasServerService serverService, - ExportImportAuditService exportImportAuditService, AtlasEntityStore entityStore, - AtlasPatchManager patchManager, AtlasAuditService auditService, EntityAuditRepository auditRepository, - TaskManagement taskManagement, AtlasDebugMetricsSink debugMetricsRESTSink, AtlasAuditReductionService atlasAuditReductionService, AtlasMetricsUtil atlasMetricsUtil, + ExportService exportService, ImportService importService, SearchTracker activeSearches, + MigrationProgressService migrationProgressService, AtlasServerService serverService, + ExportImportAuditService exportImportAuditService, AtlasEntityStore entityStore, + AtlasPatchManager patchManager, AtlasAuditService auditService, EntityAuditRepository auditRepository, + TaskManagement taskManagement, AtlasDebugMetricsSink debugMetricsRESTSink, AtlasAuditReductionService atlasAuditReductionService, AtlasMetricsUtil atlasMetricsUtil, PurgeService purgeService) { this.serviceState = serviceState; this.metricsService = metricsService; @@ -780,33 +782,53 @@ public AtlasAsyncImportRequest getAsyncImportStatusById(@PathParam("importId") S } } + /** + * Hard-purge entities in DELETED state by GUID. + * The response includes {@code mutatedEntities.PURGE}, {@code failedEntities}, and {@code summary}. + * Each entry in {@code failedEntities} carries {@code guid}, {@code errorCode}, and {@code errorMessage}. + * {@code summary} contains {@code requestedCount}, {@code purgedCount} (requested GUIDs purged), + * {@code purgedDependenciesCount} (dependency-expanded entities purged beyond the request), + * {@code failedCount} (non-skippable failures among originally requested GUIDs), + * {@code failedDependenciesCount} (non-skippable failures among dependency-expanded GUIDs), + * and {@code skippedCount}. + * + * @param guids set of entity GUIDs to purge + * @return consolidated purge result with per-GUID outcomes and summary counts + * @throws AtlasBaseException if the request exceeds {@code atlas.purge.api.max.request.size} or authorization fails + * @HTTP 200 the request was processed successfully. The response body contains detailed execution + * outcomes, including purged, failed, and skipped counts. + */ @PUT @Path("/purge") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) public EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseException { - if (CollectionUtils.isNotEmpty(guids)) { - for (String guid : guids) { - Servlets.validateQueryParamLength("guid", guid); - } + if (CollectionUtils.isEmpty(guids)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Guid(s) not specified"); } - AtlasPerfTracer perf = null; + int purgeApiMaxRequestSize = AtlasConfiguration.PURGE_API_MAX_REQUEST_SIZE.getInt(); + if (guids.size() > purgeApiMaxRequestSize) { + throw new AtlasBaseException(AtlasErrorCode.PURGE_REQUEST_SIZE_EXCEEDS_LIMIT, + String.valueOf(guids.size()), String.valueOf(purgeApiMaxRequestSize)); + } - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AdminResource.purgeByIds(" + guids + ")"); - } + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_PURGE), + "purge entity: guids=", guids); - EntityMutationResponse resp = entityStore.purgeByIds(guids); + for (String guid : guids) { + Servlets.validateQueryParamLength("guid", guid); + } - final List purgedEntities = resp.getPurgedEntities(); + AtlasPerfTracer perf = null; - if (purgedEntities != null && !purgedEntities.isEmpty()) { - auditService.add(AuditOperation.PURGE, guids.toString(), resp.getPurgedEntitiesIds(), resp.getPurgedEntities().size()); + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, + "AdminResource.purgeByIds(count=" + guids.size() + ")"); } - return resp; + return purgeService.purgeByIds(guids); } finally { AtlasPerfTracer.log(perf); } @@ -814,34 +836,44 @@ public EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseExce @Scheduled(cron = "#{getPurgeCronExpression}") public void schedulePurgeEntities() throws AtlasBaseException { + boolean lockAcquired = false; + String originalThreadName = Thread.currentThread().getName(); try { Thread.currentThread().setName(PURGE_THREAD_NAME); - if (acquireCronPurgeOperationLock()) { + lockAcquired = acquireCronPurgeOperationLock(); + if (!lockAcquired) { + LOG.info("==> Scheduled purge skipped because another purge operation is already running"); + } else { String state = serviceState.getState().toString(); LOG.info("==> Status of current node is {}", state); if (state.equals(ACTIVE)) { LOG.info("==> Scheduled Purging has started"); EntityMutationResponse entityMutationResponse = purgeService.purgeEntities(); - Set guids = new HashSet<>(); final List purgedEntities = entityMutationResponse.getPurgedEntities() != null ? entityMutationResponse.getPurgedEntities() : Collections.emptyList(); - if (CollectionUtils.isEmpty(purgedEntities)) { - LOG.info("==> no entities got purged"); - return; + PurgeSummary summary = entityMutationResponse.getPurgeSummary(); + if (summary != null) { + LOG.info("==> Purge execution summary: {}", summary); } - for (AtlasEntityHeader entityHeader : entityMutationResponse.getPurgedEntities()) { - guids.add(entityHeader.getGuid()); + if (CollectionUtils.isEmpty(purgedEntities)) { + if (summary != null + && (summary.getFailedCount() > 0 + || summary.getFailedDependenciesCount() > 0)) { + LOG.info("==> no entities got purged, but encountered failures"); + } else if (summary != null && summary.getSkippedCount() > 0) { + LOG.info("==> no entities got purged, but some were skipped"); + } else { + LOG.info("==> no entities got purged"); + } + return; } LOG.info("==> Purged Entities {}", purgedEntities.size()); - auditService.add(AuditOperation.AUTO_PURGE, guids.toString(), entityMutationResponse.getPurgedEntitiesIds(), - entityMutationResponse.getPurgedEntities().size()); - LOG.info("==> Scheduled Purging has finished"); } else { LOG.info("==> Current node is not active, so skipping the scheduled purge"); @@ -853,7 +885,10 @@ public void schedulePurgeEntities() throws AtlasBaseException { } finally { RequestContext.clear(); LOG.info("==> clearing the context"); - cronPurgeOperationLock.unlock(); + if (lockAcquired) { + cronPurgeOperationLock.unlock(); + } + Thread.currentThread().setName(originalThreadName); } } @@ -1012,7 +1047,21 @@ public List getAtlasAudits(AuditSearchParameters auditSearchPar AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_AUDITS), "Admin Audits"); - return auditService.get(auditSearchParameters); + boolean summaryOnlyListing = auditSearchParameters == null + || !PurgeUtils.hasRunIdFilter(auditSearchParameters.getAuditFilters()); + if (summaryOnlyListing) { + // Graph auditRowKind != BATCH: exclude batch rows before pagination. + PurgeUtils.excludeBatchRowsFromAuditSearch(auditSearchParameters); + } + + List auditEntries = auditService.get(auditSearchParameters); + + if (summaryOnlyListing) { + // Defensive post-filter for BATCH rows still returned by the index. + return PurgeUtils.excludeBatchRowsFromResults(auditEntries); + } + + return auditEntries; } finally { AtlasPerfTracer.log(perf); } @@ -1036,6 +1085,11 @@ public List getAuditDetails(@PathParam("auditGuid") String au AtlasAuditEntry auditEntry = auditService.toAtlasAuditEntry(entityStore.getById(auditGuid, false, true)); + // Summary purge audit rows store PurgeSummary JSON in result, not entity GUIDs. + if (auditEntry != null && PurgeUtils.isPurgeSummaryAudit(auditEntry)) { + return ret; + } + if (auditEntry != null && StringUtils.isNotEmpty(auditEntry.getResult())) { String[] listOfResultGuid = auditEntry.getResult().split(","); EntityAuditActionV2 auditAction = auditEntry.getOperation().toEntityAuditActionV2(); @@ -1061,6 +1115,51 @@ public List getAuditDetails(@PathParam("auditGuid") String au } } + @GET + @Path("/audit/{auditGuid}/purgedEntities") + @Produces(Servlets.JSON_MEDIA_TYPE) + public List getPurgeAuditPurgedEntities(@PathParam("auditGuid") String auditGuid, + @QueryParam("limit") @DefaultValue("100") int limit, + @QueryParam("offset") @DefaultValue("0") int offset) throws AtlasBaseException { + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_AUDITS), "Admin Audits"); + + AtlasAuditEntry auditEntry = loadPurgeAuditEntry(auditGuid); + + List purgedGuids = PurgeUtils.isPurgeSummaryAudit(auditEntry) + ? auditService.getPurgedEntityGuidsForRun(auditEntry) + : PurgeUtils.collectPurgedGuidsFromBatchEntries(Collections.singletonList(auditEntry)); + + return PurgeUtils.paginateStringList(purgedGuids, limit, offset); + } + + @GET + @Path("/audit/{auditGuid}/batches") + @Produces(Servlets.JSON_MEDIA_TYPE) + public List getPurgeAuditBatches(@PathParam("auditGuid") String auditGuid) throws AtlasBaseException { + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_AUDITS), "Admin Audits"); + + return auditService.getPurgeBatchAuditGuidsForRun(loadSummaryPurgeAuditEntry(auditGuid)); + } + + private AtlasAuditEntry loadSummaryPurgeAuditEntry(String auditGuid) throws AtlasBaseException { + AtlasAuditEntry auditEntry = loadPurgeAuditEntry(auditGuid); + if (!PurgeUtils.isPurgeSummaryAudit(auditEntry)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Not a purge summary audit entry: " + auditGuid); + } + + return auditEntry; + } + + private AtlasAuditEntry loadPurgeAuditEntry(String auditGuid) throws AtlasBaseException { + AtlasAuditEntry auditEntry = auditService.toAtlasAuditEntry(entityStore.getById(auditGuid, false, true)); + + if (auditEntry == null || !PurgeUtils.isCorrelatedPurgeAudit(auditEntry)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Not a purge audit entry: " + auditGuid); + } + + return auditEntry; + } + @GET @Path("activeSearches") @Produces(Servlets.JSON_MEDIA_TYPE) diff --git a/webapp/src/test/java/org/apache/atlas/web/errors/AtlasBaseExceptionMapperTest.java b/webapp/src/test/java/org/apache/atlas/web/errors/AtlasBaseExceptionMapperTest.java index 8e3d09c5796..e4543854d97 100644 --- a/webapp/src/test/java/org/apache/atlas/web/errors/AtlasBaseExceptionMapperTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/errors/AtlasBaseExceptionMapperTest.java @@ -37,6 +37,25 @@ public void setUp() { atlasBaseExceptionMapper = new AtlasBaseExceptionMapper(); } + @Test + public void testPurgeRequestSizeExceedsLimitMapsToHttp400() { + AtlasBaseException exception = new AtlasBaseException( + AtlasErrorCode.PURGE_REQUEST_SIZE_EXCEEDS_LIMIT, "1001", "1000"); + + Response response = atlasBaseExceptionMapper.toResponse(exception); + + assertNotNull(response); + assertEquals(response.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); + assertNotNull(response.getEntity()); + + String entity = (String) response.getEntity(); + assertTrue(entity.contains(AtlasErrorCode.PURGE_REQUEST_SIZE_EXCEEDS_LIMIT.getErrorCode())); + assertTrue(entity.contains("errorCode")); + assertTrue(entity.contains("errorMessage")); + assertTrue(entity.contains("1001")); + assertTrue(entity.contains("1000")); + } + @Test public void testToResponse() { // Test the toResponse method execution diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java index ee69e5921bd..f8bad1b8e0f 100644 --- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java @@ -20,15 +20,19 @@ import com.fasterxml.jackson.databind.JsonNode; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.authorize.AtlasAdminAccessRequest; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.discovery.SearchContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.PList; import org.apache.atlas.model.audit.AtlasAuditEntry; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditRowKind; import org.apache.atlas.model.audit.AuditReductionCriteria; import org.apache.atlas.model.audit.AuditSearchParameters; import org.apache.atlas.model.audit.EntityAuditEventV2; +import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.impexp.AsyncImportStatus; import org.apache.atlas.model.impexp.AtlasAsyncImportRequest; import org.apache.atlas.model.impexp.AtlasExportRequest; @@ -43,6 +47,7 @@ import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.PurgeSummary; import org.apache.atlas.model.metrics.AtlasMetrics; import org.apache.atlas.model.metrics.AtlasMetricsMapToChart; import org.apache.atlas.model.metrics.AtlasMetricsStat; @@ -56,7 +61,9 @@ import org.apache.atlas.repository.impexp.ExportService; import org.apache.atlas.repository.impexp.ImportService; import org.apache.atlas.repository.impexp.MigrationProgressService; +import org.apache.atlas.repository.ogm.AtlasAuditEntryDTO; import org.apache.atlas.repository.patches.AtlasPatchManager; +import org.apache.atlas.repository.purge.PurgeUtils; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.server.common.service.ServiceState; import org.apache.atlas.server.common.util.Servlets; @@ -94,6 +101,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -104,6 +112,7 @@ import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; @@ -264,6 +273,103 @@ private void injectHttpServletResponse(AdminResource adminResource) throws Excep responseField.set(adminResource, httpServletResponse); } + private static PurgeSummary buildPurgeSummary(long requested, long purged, long purgedDeps, + long failed, long failedDeps, long skipped, + boolean executionFailed) { + return buildPurgeSummary(requested, purged, purgedDeps, failed, failedDeps, skipped, 0, executionFailed); + } + + private static AuditSearchParameters createPurgeAuditSearchParameters() { + SearchParameters.FilterCriteria root = new SearchParameters.FilterCriteria(); + root.setCondition(SearchParameters.FilterCriteria.Condition.AND); + List criterion = new ArrayList<>(); + + SearchParameters.FilterCriteria opFilter = new SearchParameters.FilterCriteria(); + opFilter.setAttributeName("operation"); + opFilter.setOperator(SearchParameters.Operator.EQ); + opFilter.setAttributeValue(AuditOperation.PURGE.name()); + criterion.add(opFilter); + + root.setCriterion(criterion); + + final SearchParameters.FilterCriteria[] filtersHolder = {root}; + AuditSearchParameters searchParameters = mock(AuditSearchParameters.class); + when(searchParameters.getAuditFilters()).thenAnswer(invocation -> filtersHolder[0]); + doAnswer(invocation -> { + filtersHolder[0] = invocation.getArgument(0); + return null; + }).when(searchParameters).setAuditFilters(any(SearchParameters.FilterCriteria.class)); + + return searchParameters; + } + + private static boolean hasExcludeBatchRowKindFilter(SearchParameters.FilterCriteria filter) { + if (filter == null) { + return false; + } + + if (AtlasAuditEntryDTO.ATTRIBUTE_AUDIT_ROW_KIND.equals(filter.getAttributeName()) + && SearchParameters.Operator.NEQ.equals(filter.getOperator()) + && AuditRowKind.BATCH.name().equals(filter.getAttributeValue())) { + return true; + } + + if (filter.getCriterion() != null) { + for (SearchParameters.FilterCriteria each : filter.getCriterion()) { + if (hasExcludeBatchRowKindFilter(each)) { + return true; + } + } + } + + return false; + } + + private static AtlasAuditEntry buildSummaryAuditEntry(String guid, String runId, long requested, long purged) { + PurgeSummary summary = buildPurgeSummary(requested, purged, 0, 0, 0, 0, false); + summary.setRunId(runId); + + AtlasAuditEntry entry = new AtlasAuditEntry(); + entry.setGuid(guid); + entry.setOperation(AuditOperation.PURGE); + entry.setRunId(runId); + entry.setAuditRowKind(AuditRowKind.SUMMARY); + entry.setResult(AtlasJson.toJson(summary)); + entry.setResultCount(purged); + return entry; + } + + private static AtlasAuditEntry buildBatchAuditEntry(String guid, String runId, String purgedGuids) { + AtlasAuditEntry entry = new AtlasAuditEntry(); + entry.setGuid(guid); + entry.setOperation(AuditOperation.PURGE); + entry.setRunId(runId); + entry.setAuditRowKind(AuditRowKind.BATCH); + entry.setParams("input-guid-1,input-guid-2"); + entry.setResult(purgedGuids); + entry.setResultCount(purgedGuids.split(",").length); + return entry; + } + + private static AtlasAuditEntry buildLegacyPurgeAuditEntry(String guid, String purgedGuids) { + AtlasAuditEntry entry = new AtlasAuditEntry(); + entry.setGuid(guid); + entry.setOperation(AuditOperation.PURGE); + entry.setResult(purgedGuids); + entry.setResultCount(purgedGuids.split(",").length); + return entry; + } + + private static PurgeSummary buildPurgeSummary(long requested, long purged, long purgedDeps, + long failed, long failedDeps, long skipped, + long validGuidCount, boolean executionFailed) { + PurgeSummary summary = + new PurgeSummary(requested, purged, purgedDeps, failed, failedDeps, skipped); + summary.setValidGuidCount(validGuidCount); + summary.setExecutionFailed(executionFailed); + return summary; + } + @Test public void testGetThreadDump() { AdminResource adminResource = createAdminResource(); @@ -367,40 +473,88 @@ public void testGetMetricsByCollectionTime() throws Exception { public void testPurgeByIds() throws Exception { Set guids = new HashSet<>(); guids.add("guid1"); - guids.add("guid2"); EntityMutationResponse mockResponse = mock(EntityMutationResponse.class); List purgedEntities = new ArrayList<>(); AtlasEntityHeader mockHeader = mock(AtlasEntityHeader.class); purgedEntities.add(mockHeader); - when(entityStore.purgeByIds(guids)).thenReturn(mockResponse); + when(purgeService.purgeByIds(guids)).thenReturn(mockResponse); when(mockResponse.getPurgedEntities()).thenReturn(purgedEntities); when(mockResponse.getPurgedEntitiesIds()).thenReturn(String.valueOf(new ArrayList<>(guids))); + when(mockResponse.getFailedEntities()).thenReturn(null); + when(mockResponse.getPurgeSummary()).thenReturn(new PurgeSummary(1, 1, 0, 0, 0)); AdminResource adminResource = createAdminResource(); + injectHttpServletResponse(adminResource); EntityMutationResponse result = adminResource.purgeByIds(guids); assertNotNull(result); - verify(entityStore).purgeByIds(guids); + verify(purgeService).purgeByIds(guids); } @Test public void testPurgeByIdsWithEmptyGuidSet() throws Exception { Set emptyGuids = new HashSet<>(); + AdminResource adminResource = createAdminResource(); + injectHttpServletResponse(adminResource); + + try { + adminResource.purgeByIds(emptyGuids); + fail("Expected AtlasBaseException for empty GUID set"); + } catch (AtlasBaseException e) { + assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INVALID_PARAMETERS); + } + + verify(purgeService, never()).purgeByIds(any()); + } + + @Test + public void testPurgeByIdsRequestSizeExceedsLimit() throws Exception { + Set guids = new LinkedHashSet<>(); + for (int i = 0; i < 1001; i++) { + guids.add(String.format("11111111-1111-1111-1111-%012d", i)); + } + + AdminResource adminResource = createAdminResource(); + injectHttpServletResponse(adminResource); + + try { + adminResource.purgeByIds(guids); + fail("Expected AtlasBaseException for request size exceeding limit"); + } catch (AtlasBaseException e) { + assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.PURGE_REQUEST_SIZE_EXCEEDS_LIMIT); + assertTrue(e.getMessage().contains("1001")); + assertTrue(e.getMessage().contains("1000")); + } + + verify(purgeService, never()).purgeByIds(any()); + verify(httpServletResponse, never()).setStatus(anyInt()); + } + + @Test + public void testPurgeByIdsDoesNotWriteAggregateAudit() throws Exception { + Set guids = new LinkedHashSet<>(); + for (int i = 0; i < 15; i++) { + guids.add(String.format("11111111-1111-1111-1111-%012d", i)); + } + EntityMutationResponse mockResponse = mock(EntityMutationResponse.class); - when(entityStore.purgeByIds(emptyGuids)).thenReturn(mockResponse); - when(mockResponse.getPurgedEntities()).thenReturn(new ArrayList<>()); + List purgedEntities = new ArrayList<>(); + purgedEntities.add(mock(AtlasEntityHeader.class)); + + when(purgeService.purgeByIds(guids)).thenReturn(mockResponse); + when(mockResponse.getPurgedEntities()).thenReturn(purgedEntities); + when(mockResponse.getFailedEntities()).thenReturn(null); + when(mockResponse.getPurgeSummary()).thenReturn(new PurgeSummary(15, 15, 0, 0, 0)); AdminResource adminResource = createAdminResource(); + injectHttpServletResponse(adminResource); - EntityMutationResponse result = adminResource.purgeByIds(emptyGuids); + adminResource.purgeByIds(guids); - assertNotNull(result); - verify(entityStore).purgeByIds(emptyGuids); - // Should not call audit service for empty results verify(auditService, never()).add(any(), anyString(), any(), anyInt()); } @@ -1078,6 +1232,7 @@ public void testGetAtlasAudits() throws Exception { mockResults.add(mockEntry); when(auditService.get(searchParameters)).thenReturn(mockResults); + when(searchParameters.getAuditFilters()).thenReturn(null); AdminResource adminResource = createAdminResource(); @@ -1094,15 +1249,59 @@ public void testGetAtlasAudits() throws Exception { }); } + @Test + public void testGetAtlasAuditsExcludesBatchRowsWithoutRunId() throws Exception { + String runId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + AuditSearchParameters searchParameters = createPurgeAuditSearchParameters(); + + AtlasAuditEntry summaryEntry = buildSummaryAuditEntry("summary-guid", runId, 2, 1); + AtlasAuditEntry batchEntry = buildBatchAuditEntry("batch-guid", runId, "entity-guid-1"); + AtlasAuditEntry legacyEntry = buildLegacyPurgeAuditEntry("legacy-guid", "entity-guid-2,entity-guid-3"); + + List mockResults = new ArrayList<>(); + mockResults.add(summaryEntry); + mockResults.add(batchEntry); + mockResults.add(legacyEntry); + + when(auditService.get(searchParameters)).thenReturn(mockResults); + + AdminResource adminResource = createAdminResource(); + + withAuthorizationBypass(() -> { + try { + List result = adminResource.getAtlasAudits(searchParameters); + + assertNotNull(result); + assertEquals(result.size(), 2); + assertTrue(hasExcludeBatchRowKindFilter(searchParameters.getAuditFilters()), + "Graph search should exclude auditRowKind=BATCH when runId filter is absent"); + + boolean hasSummary = false; + boolean hasLegacy = false; + for (AtlasAuditEntry entry : result) { + if ("summary-guid".equals(entry.getGuid())) { + hasSummary = true; + } + if ("legacy-guid".equals(entry.getGuid())) { + hasLegacy = true; + } + assertFalse(PurgeUtils.isPurgeBatchAudit(entry)); + } + assertTrue(hasSummary); + assertTrue(hasLegacy); + } catch (AtlasBaseException e) { + throw new RuntimeException(e); + } + }); + } + @Test public void testGetAuditDetails() throws Exception { String auditGuid = "audit-123"; int limit = 10; int offset = 0; - AtlasAuditEntry mockAuditEntry = mock(AtlasAuditEntry.class); - when(mockAuditEntry.getResult()).thenReturn("guid1,guid2"); - when(mockAuditEntry.getOperation()).thenReturn(AtlasAuditEntry.AuditOperation.PURGE); + AtlasAuditEntry batchEntry = buildBatchAuditEntry(auditGuid, "run-id-1", "guid1,guid2"); List mockEvents = new ArrayList<>(); EntityAuditEventV2 mockEvent = mock(EntityAuditEventV2.class); @@ -1110,9 +1309,10 @@ public void testGetAuditDetails() throws Exception { when(mockEvent.getEntityHeader()).thenReturn(mockHeader); mockEvents.add(mockEvent); - when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(mockAuditEntry); + when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(batchEntry); when(entityStore.getById(auditGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); - when(auditRepository.listEventsV2(anyString(), any(EntityAuditEventV2.EntityAuditActionV2.class), any(), any(Short.class))).thenReturn(mockEvents); + when(auditRepository.listEventsV2(anyString(), any(EntityAuditEventV2.EntityAuditActionV2.class), any(), any(Short.class))) + .thenReturn(mockEvents); AdminResource adminResource = createAdminResource(); @@ -1121,9 +1321,10 @@ public void testGetAuditDetails() throws Exception { List result = adminResource.getAuditDetails(auditGuid, limit, offset); assertNotNull(result); - assertFalse(result.isEmpty()); + assertEquals(result.size(), 2); verify(entityStore).getById(auditGuid, false, true); - verify(auditService).toAtlasAuditEntry(any()); + verify(auditRepository).listEventsV2(eq("guid1"), any(EntityAuditEventV2.EntityAuditActionV2.class), any(), any(Short.class)); + verify(auditRepository).listEventsV2(eq("guid2"), any(EntityAuditEventV2.EntityAuditActionV2.class), any(), any(Short.class)); } catch (AtlasBaseException e) { throw new RuntimeException(e); } @@ -1132,12 +1333,12 @@ public void testGetAuditDetails() throws Exception { @Test public void testGetAuditDetailsWithNullResult() throws Exception { - String auditGuid = "audit-123"; + String auditGuid = "audit-null-result"; - AtlasAuditEntry mockAuditEntry = mock(AtlasAuditEntry.class); - when(mockAuditEntry.getResult()).thenReturn(null); + AtlasAuditEntry nullResultEntry = mock(AtlasAuditEntry.class); + when(nullResultEntry.getResult()).thenReturn(null); - when(auditService.toAtlasAuditEntry(any())).thenReturn(mockAuditEntry); + when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(nullResultEntry); when(entityStore.getById(auditGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); AdminResource adminResource = createAdminResource(); @@ -1155,6 +1356,125 @@ public void testGetAuditDetailsWithNullResult() throws Exception { }); } + @Test + public void testGetAuditDetailsForSummaryPurgeAudit() throws Exception { + String summaryGuid = "summary-audit-guid"; + String runId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + AtlasAuditEntry summaryEntry = buildSummaryAuditEntry(summaryGuid, runId, 2, 1); + + when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(summaryEntry); + when(entityStore.getById(summaryGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); + + AdminResource adminResource = createAdminResource(); + + withAuthorizationBypass(() -> { + try { + List result = adminResource.getAuditDetails(summaryGuid, 10, 0); + + assertNotNull(result); + assertTrue(result.isEmpty()); + verify(entityStore).getById(summaryGuid, false, true); + } catch (AtlasBaseException e) { + throw new RuntimeException(e); + } + }); + } + + @Test + public void testPurgeAuditReadEndpoints() throws Exception { + String summaryGuid = "summary-audit-guid"; + String runId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + String entityGuid1 = "11111111-1111-1111-1111-111111111111"; + String entityGuid2 = "22222222-2222-2222-2222-222222222222"; + + AtlasAuditEntry summaryEntry = buildSummaryAuditEntry(summaryGuid, runId, 2, 2); + + when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(summaryEntry); + when(entityStore.getById(summaryGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); + when(auditService.getPurgedEntityGuidsForRun(summaryEntry)).thenReturn( + java.util.Arrays.asList(entityGuid1, entityGuid2)); + when(auditService.getPurgeBatchAuditGuidsForRun(summaryEntry)).thenReturn( + java.util.Collections.singletonList("batch-guid-1")); + + AdminResource adminResource = createAdminResource(); + + withAuthorizationBypass(() -> { + try { + List purgedEntities = adminResource.getPurgeAuditPurgedEntities(summaryGuid, 10, 0); + assertNotNull(purgedEntities); + assertEquals(purgedEntities.size(), 2); + assertEquals(purgedEntities.get(0), entityGuid1); + assertEquals(purgedEntities.get(1), entityGuid2); + + List batches = adminResource.getPurgeAuditBatches(summaryGuid); + assertNotNull(batches); + assertEquals(batches.size(), 1); + assertEquals(batches.get(0), "batch-guid-1"); + } catch (AtlasBaseException e) { + throw new RuntimeException(e); + } + }); + + verify(auditService).getPurgedEntityGuidsForRun(summaryEntry); + } + + @Test + public void testPurgeAuditPurgedEntitiesForBatchGuid() throws Exception { + String batchGuid = "batch-audit-guid"; + String runId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + String entityGuid1 = "11111111-1111-1111-1111-111111111111"; + String entityGuid2 = "22222222-2222-2222-2222-222222222222"; + + AtlasAuditEntry batchEntry = buildBatchAuditEntry(batchGuid, runId, entityGuid1 + "," + entityGuid2); + + when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(batchEntry); + when(entityStore.getById(batchGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); + + AdminResource adminResource = createAdminResource(); + + withAuthorizationBypass(() -> { + try { + List purgedEntities = adminResource.getPurgeAuditPurgedEntities(batchGuid, 10, 0); + assertNotNull(purgedEntities); + assertEquals(purgedEntities.size(), 2); + assertEquals(purgedEntities.get(0), entityGuid1); + assertEquals(purgedEntities.get(1), entityGuid2); + } catch (AtlasBaseException e) { + throw new RuntimeException(e); + } + }); + + verify(auditService, never()).getPurgedEntityGuidsForRun(any()); + } + + @Test + public void testPurgeAuditPurgedEntitiesForLegacyPurgeGuid() throws Exception { + String legacyGuid = "legacy-purge-audit-guid"; + String entityGuid1 = "11111111-1111-1111-1111-111111111111"; + String entityGuid2 = "22222222-2222-2222-2222-222222222222"; + + AtlasAuditEntry legacyEntry = buildLegacyPurgeAuditEntry(legacyGuid, entityGuid1 + "," + entityGuid2); + + when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(legacyEntry); + when(entityStore.getById(legacyGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); + + AdminResource adminResource = createAdminResource(); + + withAuthorizationBypass(() -> { + try { + List purgedEntities = adminResource.getPurgeAuditPurgedEntities(legacyGuid, 10, 0); + assertNotNull(purgedEntities); + assertEquals(purgedEntities.size(), 2); + assertEquals(purgedEntities.get(0), entityGuid1); + assertEquals(purgedEntities.get(1), entityGuid2); + } catch (AtlasBaseException e) { + throw new RuntimeException(e); + } + }); + + verify(auditService, never()).getPurgedEntityGuidsForRun(any()); + } + @Test public void testGetEditableEntityTypesWithStringValue() throws Exception { when(mockConfiguration.containsKey("atlas.ui.editable.entity.types")).thenReturn(true); From 7d553225cab03355dfe3c89e346aacbec49d713c Mon Sep 17 00:00:00 2001 From: Sheetal Shah Date: Mon, 27 Jul 2026 18:07:17 +0530 Subject: [PATCH 2/9] ATLAS-5317: Add missing Apache license headers to purge batch tests. Fix RAT check failure in atlas-repository CI. --- .../atlas/services/PurgeBatchExecutorTest.java | 17 +++++++++++++++++ .../services/PurgeBatchOrchestratorTest.java | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java b/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java index 11cf160cc3e..64303baac43 100644 --- a/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java +++ b/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.atlas.services; import org.apache.atlas.AtlasErrorCode; diff --git a/repository/src/test/java/org/apache/atlas/services/PurgeBatchOrchestratorTest.java b/repository/src/test/java/org/apache/atlas/services/PurgeBatchOrchestratorTest.java index 65c981258a2..9dabaf69206 100644 --- a/repository/src/test/java/org/apache/atlas/services/PurgeBatchOrchestratorTest.java +++ b/repository/src/test/java/org/apache/atlas/services/PurgeBatchOrchestratorTest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.atlas.services; import org.apache.atlas.AtlasErrorCode; From 6b2da6414d902e460c0d1b8b491377a9e5e82931 Mon Sep 17 00:00:00 2001 From: Sheetal Shah Date: Mon, 27 Jul 2026 21:18:08 +0530 Subject: [PATCH 3/9] ATLAS-5317: Retry purge batches only on PermanentLockingException. Simplify PurgeBatchExecutorTest without sun.misc.Unsafe. Refresh embedded Solr edismax qf fields in test-tools solrconfig.xml. --- .../atlas/services/PurgeBatchExecutor.java | 19 ++-- .../services/PurgeBatchExecutorTest.java | 93 ++++++------------- .../solr/core-template/solrconfig.xml | 2 +- 3 files changed, 38 insertions(+), 76 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/services/PurgeBatchExecutor.java b/repository/src/main/java/org/apache/atlas/services/PurgeBatchExecutor.java index 706fd413d96..364b5123990 100644 --- a/repository/src/main/java/org/apache/atlas/services/PurgeBatchExecutor.java +++ b/repository/src/main/java/org/apache/atlas/services/PurgeBatchExecutor.java @@ -26,7 +26,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -39,19 +38,15 @@ public class PurgeBatchExecutor { private static final int BASE_BACKOFF_MS = 500; /** - * Fully-qualified class names treated as retryable lock or backend conflicts during purge batch - * execution. Names are matched against the throwable cause chain to avoid a compile-time dependency - * on JanusGraph or Berkeley JE types in the service layer. + * Fully-qualified class names treated as retryable lock conflicts during purge batch execution. + * Names are matched against the throwable cause chain to avoid a compile-time dependency on + * JanusGraph types in the service layer. *

- * Design default: {@code PermanentLockingException}. Berkeley JE lock timeouts/deadlocks and - * {@code PermanentBackendException} are included for the embedded Berkeley backend. + * Design default: {@code PermanentLockingException} (see ATLAS-5317 retry strategy). */ static final Set RETRYABLE_LOCK_CONFLICT_EXCEPTION_CLASS_NAMES = Collections.unmodifiableSet( - new HashSet<>(Arrays.asList( - "org.janusgraph.diskstorage.locking.PermanentLockingException", - "com.sleepycat.je.LockTimeoutException", - "com.sleepycat.je.DeadlockException", - "org.janusgraph.diskstorage.PermanentBackendException"))); + new HashSet<>(Collections.singletonList( + "org.janusgraph.diskstorage.locking.PermanentLockingException"))); private final AtlasEntityStore entityStore; @@ -69,7 +64,7 @@ public EntityMutationResponse executeBatch(Set batch) throws AtlasBaseEx /** * Returns {@code true} when {@code throwable} or any of its causes matches a known retryable - * lock or backend conflict type. + * lock conflict type. */ static boolean isRetryableLockConflict(Throwable throwable) { if (throwable == null) { diff --git a/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java b/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java index 64303baac43..5e27c82d3ce 100644 --- a/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java +++ b/repository/src/test/java/org/apache/atlas/services/PurgeBatchExecutorTest.java @@ -23,9 +23,9 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.janusgraph.diskstorage.PermanentBackendException; import org.janusgraph.diskstorage.locking.PermanentLockingException; import org.mockito.MockedStatic; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.util.Collections; @@ -45,16 +45,6 @@ public class PurgeBatchExecutorTest { private static final Set BATCH = Collections.singleton("guid1"); - @DataProvider(name = "retryableLockConflictExceptionClassNames") - public Object[][] retryableLockConflictExceptionClassNames() { - return new Object[][] { - {"org.janusgraph.diskstorage.locking.PermanentLockingException"}, - {"com.sleepycat.je.LockTimeoutException"}, - {"com.sleepycat.je.DeadlockException"}, - {"org.janusgraph.diskstorage.PermanentBackendException"} - }; - } - @Test public void testExecuteBatchSuccess() throws Exception { AtlasEntityStore mockStore = mock(AtlasEntityStore.class); @@ -79,30 +69,33 @@ public void testIsRetryableLockConflictReturnsFalseForNonRetryableException() { } @Test - public void testIsRetryableLockConflictMatchesWrappedCause() { + public void testIsRetryableLockConflictReturnsFalseForPermanentBackendException() { + PermanentBackendException backendException = new PermanentBackendException("backend failure"); + + assertFalse(PurgeBatchExecutor.isRetryableLockConflict(backendException)); + } + + @Test + public void testIsRetryableLockConflictMatchesPermanentLockingException() { PermanentLockingException ple = new PermanentLockingException("lock conflict"); - RuntimeException wrapped = new RuntimeException(new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, ple)); - assertTrue(PurgeBatchExecutor.isRetryableLockConflict(wrapped)); + assertTrue(PurgeBatchExecutor.isRetryableLockConflict(ple)); } - @Test(dataProvider = "retryableLockConflictExceptionClassNames") - public void testIsRetryableLockConflictMatchesKnownTypes(String className) throws Exception { - Exception conflict = newExceptionByClassName(className, "lock conflict"); + @Test + public void testIsRetryableLockConflictMatchesWrappedCause() { + PermanentLockingException ple = new PermanentLockingException("lock conflict"); + RuntimeException wrapped = new RuntimeException(new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, ple)); - assertTrue(PurgeBatchExecutor.RETRYABLE_LOCK_CONFLICT_EXCEPTION_CLASS_NAMES.contains(className)); - assertTrue(PurgeBatchExecutor.isRetryableLockConflict(conflict)); - // Use message+cause form: RuntimeException(Throwable) calls cause.toString(), which NPEs on - // partially-initialized Berkeley JE DatabaseException instances created for this test. - assertTrue(PurgeBatchExecutor.isRetryableLockConflict(wrapWithCause(conflict))); + assertTrue(PurgeBatchExecutor.isRetryableLockConflict(wrapped)); } @Test public void testExecuteBatchClearsCachesBeforeRetry() throws Exception { AtlasEntityStore mockStore = mock(AtlasEntityStore.class); EntityMutationResponse mockResponse = new EntityMutationResponse(); - PermanentLockingException ple = new PermanentLockingException("lock conflict"); - AtlasBaseException wrappedException = new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, ple); + PermanentLockingException ple = new PermanentLockingException("lock conflict"); + AtlasBaseException wrappedException = new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, ple); when(mockStore.purgeEntitiesInBatch(BATCH)) .thenThrow(wrappedException) @@ -149,24 +142,6 @@ public void testExecuteBatchRetryOnPermanentLockingException() throws Exception assertTrue(duration >= 1000, "Expected backoff delays but finished in " + duration + " ms"); } - @Test(dataProvider = "retryableLockConflictExceptionClassNames") - public void testExecuteBatchRetriesOnKnownLockConflictTypes(String className) throws Exception { - AtlasEntityStore mockStore = mock(AtlasEntityStore.class); - EntityMutationResponse mockResponse = new EntityMutationResponse(); - Exception conflict = newExceptionByClassName(className, "lock conflict"); - AtlasBaseException wrappedException = new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, conflict); - - when(mockStore.purgeEntitiesInBatch(BATCH)) - .thenThrow(wrappedException) - .thenReturn(mockResponse); - - PurgeBatchExecutor executor = new PurgeBatchExecutor(mockStore); - EntityMutationResponse response = executor.executeBatch(BATCH); - - assertEquals(response, mockResponse); - verify(mockStore, times(2)).purgeEntitiesInBatch(BATCH); - } - @Test public void testExecuteBatchFailsAfterMaxLockingConflicts() throws Exception { AtlasEntityStore mockStore = mock(AtlasEntityStore.class); @@ -199,27 +174,19 @@ public void testExecuteBatchNoRetryOnNonLockingException() throws Exception { verify(mockStore, times(1)).purgeEntitiesInBatch(BATCH); } - private static RuntimeException wrapWithCause(Throwable cause) { - return new RuntimeException("wrapped", cause); - } + @Test + public void testExecuteBatchNoRetryOnPermanentBackendException() throws Exception { + AtlasEntityStore mockStore = mock(AtlasEntityStore.class); + PermanentBackendException backendException = new PermanentBackendException("backend failure"); + AtlasBaseException wrappedException = new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, backendException); - private static Exception newExceptionByClassName(String className, String message) throws Exception { - try { - Class clazz = Class.forName(className); - try { - return (Exception) clazz.getConstructor(String.class).newInstance(message); - } catch (NoSuchMethodException e) { - try { - return (Exception) clazz.getConstructor().newInstance(); - } catch (NoSuchMethodException e2) { - java.lang.reflect.Field f = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - sun.misc.Unsafe unsafe = (sun.misc.Unsafe) f.get(null); - return (Exception) unsafe.allocateInstance(clazz); - } - } - } catch (ClassNotFoundException e) { - throw new org.testng.SkipException("Required exception class not on classpath: " + className); - } + when(mockStore.purgeEntitiesInBatch(BATCH)).thenThrow(wrappedException); + + PurgeBatchExecutor executor = new PurgeBatchExecutor(mockStore); + + AtlasBaseException ex = expectThrows(AtlasBaseException.class, () -> executor.executeBatch(BATCH)); + + assertEquals(ex.getAtlasErrorCode(), AtlasErrorCode.INTERNAL_ERROR); + verify(mockStore, times(1)).purgeEntitiesInBatch(BATCH); } } diff --git a/test-tools/src/main/resources/solr/core-template/solrconfig.xml b/test-tools/src/main/resources/solr/core-template/solrconfig.xml index 52f6db5b980..3e88b285522 100644 --- a/test-tools/src/main/resources/solr/core-template/solrconfig.xml +++ b/test-tools/src/main/resources/solr/core-template/solrconfig.xml @@ -445,7 +445,7 @@ --> edismax - 35x_t 5j9_t 7wl_t a9x_t but_t dfp_l f0l_t i6d_l iyt_l jr9_t kjp_s lc5_t m4l_s mx1_t ohx_t xz9_i 1151_t 12px_t 14at_l 15vp_t 1891_t 19tx_t 1bet_t 1czp_t 1ekl_t 1gxx_t 1iit_l 1k3p_t 1lol_t 1o1x_t 1qf9_t 1ssl_t 1v5x_t 1wqt_t 1z45_t 20p1_t 2rk5_l 2t51_l 50xx_t 5dl1_s 5c05_s 59mt_s 581x_s 5b7p_t 5j45_t 5f5x_t 5gqt_l 5hj9_t 5nut_t 5m9x_t 5pfp_t 5wjp_t 622t_t 66th_t 64g5_t 658l_t 63np_t 6d51_t 6ccl_t 6k91_l 6ltx_l 6dxh_t 6jgl_t 6uit_l 6w3p_l 6sxx_t 6rd1_t 6xol_t 6net_t 6o79_t 6ps5_t 77yd_t 78qt_l 74sl_t 71mt_l 737p_l 6yh1_t 76dh_t 7i85_t 7j0l_t 7klh_t 7abp_t 7m6d_t 7hfp_t 7jt1_t 7ldx_t 7fut_t ac5h_t a491_t adqd_t a9s5_t acxx_t abd1_t an7p_t afb9_t apl1_t akud_t amf9_t aosl_t az2d_t ar5x_t awp1_t ay9x_t fcat_t f56t_t f9xh_t fbid_t f8cl_t f3lx_l f6rp_t f951_t f4ed_l fdvp_l fimd_t fjet_t fd39_l feo5_t ffgl_i fhtx_t fmkl_t fls5_t fnd1_t foxx_i fsw5_t fuh1_t fyf9_t fz7p_i fwud_t g0sl_t g005_t g5j9_t g7wl_t g35x_t g6bp_t g4qt_t g1l1_l g745_t g2dh_l hwqt_l htl1_t hv5x_l i0p1_l i1hh_t i4n9_i i70l_t i32d_t j9qd_t jll1_l jn5x_i jg1x_t jnyd_i joqt_i jpj9_f jvut_d jv2d_l kumd_t l05h_l l3b9_t l6h1_l l81x_t l4w5_t l9mt_l lb7p_t + 35x_t 5j9_t 7wl_t a9x_t but_t dfp_l f0l_t i6d_l iyt_l jr9_t kjp_s lc5_t m4l_s mx1_t ohx_t xz9_i 1151_t 12px_t 14at_l 15vp_t 1891_t 19tx_t 1bet_t 1czp_t 1ekl_t 1gxx_t 1iit_l 1k3p_t 1lol_t 1o1x_t 1qf9_t 1ssl_t 1v5x_t 1wqt_t 1z45_t 20p1_t 2rk5_l 2t51_l 581x_t 5kp1_s 5j45_s 5gqt_s 5f5x_s 5ibp_t 5q85_t 5m9x_t 5nut_l 5on9_t 5uyt_t 5tdx_t 5wjp_t 63np_t 696t_t 6dxh_t 6bk5_t 6ccl_t 6arp_t 6k91_t 6jgl_t 6rd1_l 6sxx_l 6l1h_t 6qkl_t 71mt_l 737p_l 701x_t 6yh1_t 74sl_t 6uit_t 6vb9_t 6ww5_t 7f2d_t 7fut_l 7bwl_t 78qt_l 7abp_l 7hfp_t 75l1_t 7dhh_t 7shx_t 7tad_t 7uv9_t 7klh_t 7wg5_t 7rph_t 7u2t_t 7vnp_t 7q4l_t amf9_t aeit_t ao05_t ak1x_t an7p_t almt_t axhh_t apl1_t azut_t av45_t awp1_t az2d_t b9c5_t b1fp_t b6yt_t b8jp_t fmkl_t ffgl_t fk79_t fls5_t fimd_t fdvp_l fh1h_t fjet_t feo5_l fo5h_l fsw5_t ftol_t fnd1_l foxx_t fpqd_i fs3p_t fwud_t fw1x_t fxmt_t fz7p_i g35x_t g4qt_t g8p1_t g9hh_i g745_t gb2d_t ga9x_t gft1_t gi6d_t gdfp_t gglh_t gf0l_t gbut_l ghdx_t gcn9_l i70l_l i3ut_t i5fp_l iayt_l ibr9_t iex1_i ihad_t idc5_t jk05_t jvut_l jxfp_i jqbp_t jy85_i jz0l_i jzt1_f k64l_d k5c5_l * true true From 20a239226522ba50b64ce4c1061364957a622aad Mon Sep 17 00:00:00 2001 From: Sheetal Shah Date: Thu, 30 Jul 2026 16:17:48 +0530 Subject: [PATCH 4/9] ATLAS-5317: Restore pre-5317 purge test layout in AdminPurgeTest, PurgeServiceTest, and AdminResourceTest. --- .../repository/audit/AdminPurgeTest.java | 240 ++++++++++++++++++ .../atlas/services/PurgeServiceTest.java | 175 +------------ 2 files changed, 246 insertions(+), 169 deletions(-) create mode 100644 repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java b/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java new file mode 100644 index 00000000000..87e6d4d6edf --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.audit; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.RequestContext; +import org.apache.atlas.TestModules; +import org.apache.atlas.TestUtilsV2; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.AtlasAuditEntry; +import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; +import org.apache.atlas.model.audit.AuditSearchParameters; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.repository.AtlasTestBase; +import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.purge.PurgeUtils; +import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; +import org.apache.atlas.services.PurgeService; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.TestResourceFileUtils; +import org.testng.SkipException; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import javax.inject.Inject; + +import java.io.IOException; +import java.util.Comparator; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +/** + * End-to-end delete, purge, and audit-search integration test for admin purge flows. + */ +@Guice(modules = TestModules.TestOnlyModule.class) +public class AdminPurgeTest extends AtlasTestBase { + private static final String CLIENT_HOST = "127.0.0.0"; + private static final String DEFAULT_USER = "Admin"; + private static final String AUDIT_PARAMETER_RESOURCE_DIR = "auditSearchParameters"; + + @Inject + private AtlasTypeDefStore typeDefStore; + + @Inject + private AtlasTypeRegistry typeRegistry; + + @Inject + private AtlasEntityStoreV2 entityStore; + + @Inject + private AtlasGraph atlasGraph; + + @Inject + private AtlasAuditService auditService; + + @BeforeClass + public void setupClass() throws Exception { + RequestContext.clear(); + super.initialize(); + basicSetup(typeDefStore, typeRegistry); + Thread.sleep(1000); + } + + @BeforeMethod + public void setupMethod() { + RequestContext.clear(); + RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); + } + + @AfterClass + public void tearDownClass() throws Exception { + Thread.sleep(1000); + AtlasGraphProvider.cleanup(); + super.cleanup(); + } + + @Test + public void testDeleteEntitiesDoesNotLookupDeletedEntity() throws Exception { + AtlasTypesDef sampleTypes = TestUtilsV2.defineDeptEmployeeTypes(); + AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(sampleTypes, typeRegistry); + + if (!typesToCreate.isEmpty()) { + typeDefStore.createTypesDef(typesToCreate); + } + + AtlasEntitiesWithExtInfo deptEg2 = TestUtilsV2.createDeptEg2(); + AtlasEntityStream entityStream = new AtlasEntityStream(deptEg2); + EntityMutationResponse emr = entityStore.createOrUpdate(entityStream, false); + + pauseForIndexCreation(); + + assertNotNull(emr); + assertNotNull(emr.getCreatedEntities()); + assertFalse(emr.getCreatedEntities().isEmpty()); + + List guids = emr.getCreatedEntities().stream() + .map(AtlasEntityHeader::getGuid) + .collect(Collectors.toList()); + + EntityMutationResponse deleteResponse = entityStore.deleteByIds(guids); + pauseForIndexCreation(); + + List responseDeletedEntities = deleteResponse.getDeletedEntities(); + assertNotNull(responseDeletedEntities); + + responseDeletedEntities.sort(Comparator.comparing(AtlasEntityHeader::getGuid)); + + List toBeDeletedEntities = emr.getCreatedEntities(); + toBeDeletedEntities.sort(Comparator.comparing(AtlasEntityHeader::getGuid)); + + assertEquals(responseDeletedEntities.size(), emr.getCreatedEntities().size()); + + for (int index = 0; index < responseDeletedEntities.size(); index++) { + assertEquals(responseDeletedEntities.get(index).getGuid(), emr.getCreatedEntities().get(index).getGuid()); + } + + ApplicationProperties.get().setProperty("atlas.purge.workers.count", "1"); + + Date startTimestamp = new Date(); + EntityMutationResponse purgeResponse = createPurgeService().purgeByIds(new HashSet<>(guids)); + + pauseForIndexCreation(); + + List responsePurgedEntities = purgeResponse.getPurgedEntities(); + assertNotNull(responsePurgedEntities); + responsePurgedEntities.sort(Comparator.comparing(AtlasEntityHeader::getGuid)); + + assertEquals(responsePurgedEntities.size(), responseDeletedEntities.size()); + + for (int index = 0; index < responsePurgedEntities.size(); index++) { + assertEquals(responsePurgedEntities.get(index).getGuid(), responseDeletedEntities.get(index).getGuid()); + } + + auditService.add(DEFAULT_USER, AuditOperation.PURGE, CLIENT_HOST, startTimestamp, new Date(), + guids.toString(), purgeResponse.getPurgedEntitiesIds(), purgeResponse.getPurgedEntities().size()); + + assertAuditEntry(auditService, createAuditParameter("audit-search-parameter-without-filter")); + assertAuditEntry(auditService, createAuditParameter("audit-search-parameter-purge")); + assertPurgeAuditRowsWrittenByPurgeService(createAuditParameter("audit-search-parameter-purge")); + } + + private PurgeService createPurgeService() { + return new PurgeService(atlasGraph, entityStore, typeRegistry, auditService); + } + + private AuditSearchParameters createAuditParameter(String fileName) { + try { + return TestResourceFileUtils.readObjectFromJson(AUDIT_PARAMETER_RESOURCE_DIR, fileName, AuditSearchParameters.class); + } catch (IOException e) { + fail(e.getMessage()); + } + + return null; + } + + private void assertPurgeAuditRowsWrittenByPurgeService(AuditSearchParameters auditSearchParameters) { + pauseForIndexCreation(); + + List result; + + try { + result = auditService.get(auditSearchParameters); + } catch (Exception e) { + throw new SkipException("purge audit entries not retrieved."); + } + + assertNotNull(result); + assertFalse(result.isEmpty()); + + boolean hasSummaryRow = false; + boolean hasBatchRow = false; + for (AtlasAuditEntry entry : result) { + if (entry.getOperation() != AuditOperation.PURGE && entry.getOperation() != AuditOperation.AUTO_PURGE) { + continue; + } + + if (PurgeUtils.isPurgeSummaryAudit(entry)) { + hasSummaryRow = true; + assertNotNull(entry.getRunId(), "Purge summary audit should carry runId"); + assertNotNull(PurgeUtils.parsePurgeSummary(entry), "Summary row should contain parseable PurgeSummary JSON"); + } + + if (PurgeUtils.isPurgeBatchAudit(entry)) { + hasBatchRow = true; + assertNotNull(entry.getRunId(), "Purge batch audit should carry runId"); + } + } + + assertTrue(hasSummaryRow, "Expected at least one purge summary audit row from purgeByIds"); + assertTrue(hasBatchRow, "Expected at least one purge batch audit row from purgeByIds"); + } + + private void assertAuditEntry(AtlasAuditService auditService, AuditSearchParameters auditSearchParameters) { + pauseForIndexCreation(); + + List result; + + try { + result = auditService.get(auditSearchParameters); + } catch (Exception e) { + throw new SkipException("audit entries not retrieved."); + } + + assertNotNull(result); + assertFalse(result.isEmpty()); + } +} diff --git a/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java b/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java index 76417c74a16..4f14fbf8a0e 100644 --- a/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java @@ -25,18 +25,14 @@ import org.apache.atlas.TestModules; import org.apache.atlas.TestUtilsV2; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.audit.AtlasAuditEntry; import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; import org.apache.atlas.model.audit.AtlasAuditEntry.AuditRowKind; -import org.apache.atlas.model.audit.AuditSearchParameters; import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.FailedEntity; import org.apache.atlas.model.instance.PurgeSummary; -import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.AtlasTestBase; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.audit.AtlasAuditService; @@ -48,7 +44,6 @@ import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.purge.PurgeExecutionStats; import org.apache.atlas.repository.purge.PurgeUtils; -import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; @@ -57,11 +52,9 @@ import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; -import org.apache.atlas.utils.TestResourceFileUtils; import org.apache.commons.lang3.RandomStringUtils; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; -import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -70,13 +63,11 @@ import javax.inject.Inject; -import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -87,7 +78,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -104,15 +94,12 @@ import static org.testng.Assert.fail; /** - * Integration tests for {@link PurgeService}: REST purgeByIds, scheduled purgeEntities, - * cron failure handling, REST/cron overlap, and delete-then-purge audit correlation. + * Integration tests for {@link PurgeService}: scheduled purgeEntities, REST purgeByIds, + * cron failure handling, and REST/cron overlap. */ @Guice(modules = TestModules.TestOnlyModule.class) public class PurgeServiceTest extends AtlasTestBase { - private static final String CRON_ELIGIBLE_GUID = "11111111-1111-1111-1111-111111111111"; - private static final String CLIENT_HOST = "127.0.0.0"; - private static final String DEFAULT_USER = "Admin"; - private static final String AUDIT_PARAMETER_RESOURCE_DIR = "auditSearchParameters"; + private static final String CRON_ELIGIBLE_GUID = "11111111-1111-1111-1111-111111111111"; @Inject private AtlasTypeDefStore typeDefStore; @@ -155,7 +142,7 @@ public void tearDownClass() throws Exception { // ------------------------------------------------------------------------- @Test - public void scheduledPurge_purgesEligibleDeletedEntity() throws Exception { + public void testPurgeEntities() throws Exception { AtlasEntity db = newHiveDb(null); persistAndGetGuid(db); AtlasEntity tbl = newHiveTable(db, null); @@ -411,60 +398,12 @@ public void cronFailure_afterGuidsCollected_writesSummaryAuditWithRunId() throws eq(0L), eq(summary.getRunId()), eq(AuditRowKind.SUMMARY)); } - // ------------------------------------------------------------------------- - // End-to-end delete + purge + audit search - // ------------------------------------------------------------------------- - - @Test - public void deleteThenPurge_writesAuditsSearchableByAdminFilters() throws Exception { - AtlasTypesDef sampleTypes = TestUtilsV2.defineDeptEmployeeTypes(); - AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(sampleTypes, typeRegistry); - - if (!typesToCreate.isEmpty()) { - typeDefStore.createTypesDef(typesToCreate); - } - - AtlasEntitiesWithExtInfo deptEg2 = TestUtilsV2.createDeptEg2(); - AtlasEntityStream entityStream = new AtlasEntityStream(deptEg2); - EntityMutationResponse emr = entityStore.createOrUpdate(entityStream, false); - - pauseForIndexCreation(); - - assertNotNull(emr); - assertNotNull(emr.getCreatedEntities()); - assertFalse(emr.getCreatedEntities().isEmpty()); - - List guids = emr.getCreatedEntities().stream() - .map(AtlasEntityHeader::getGuid) - .collect(Collectors.toList()); - - EntityMutationResponse deleteResponse = entityStore.deleteByIds(guids); - pauseForIndexCreation(); - - assertSortedGuidsMatch(emr.getCreatedEntities(), deleteResponse.getDeletedEntities(), "deleteByIds"); - - ApplicationProperties.get().setProperty("atlas.purge.workers.count", "1"); - - Date startTimestamp = new Date(); - EntityMutationResponse purgeResponse = createPurgeService().purgeByIds(new HashSet<>(guids)); - - pauseForIndexCreation(); - assertPurgeSucceededForRequestedGuids(guids, purgeResponse); - - atlasAuditService.add(DEFAULT_USER, AuditOperation.PURGE, CLIENT_HOST, startTimestamp, new Date(), - guids.toString(), purgeResponse.getPurgedEntitiesIds(), purgeResponse.getPurgedEntities().size()); - - assertAuditEntry(atlasAuditService, createAuditParameter("audit-search-parameter-without-filter")); - assertAuditEntry(atlasAuditService, createAuditParameter("audit-search-parameter-purge")); - assertPurgeAuditRowsWrittenByPurgeService(createAuditParameter("audit-search-parameter-purge")); - } - // ------------------------------------------------------------------------- // PurgeService.purgeByIds — pre-validation and orchestration // ------------------------------------------------------------------------- @Test - public void purgeByIds_rejectsEmptyOrNullGuids() throws Exception { + public void testPurgeByIdsWithEmptySet() throws Exception { try { createPurgeService().purgeByIds(new HashSet<>()); fail("Expected AtlasBaseException for empty GUID set"); @@ -481,7 +420,7 @@ public void purgeByIds_rejectsEmptyOrNullGuids() throws Exception { } @Test - public void purgeByIds_preScanNonExistentEntities() throws Exception { + public void testPurgeByIdsWithNonExistentEntities() throws Exception { Set guids = new HashSet<>(Arrays.asList( "11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222")); @@ -727,106 +666,4 @@ private void assertNoOrphanEdgesToPurgedGuids(String anchorGuid, Set pur "Anchor vertex should not retain edges to purged GUID: " + otherGuid); } } - - private void assertSortedGuidsMatch(List expected, List actual, String operation) { - assertNotNull(actual, operation + " returned null entities"); - assertEquals(toSortedGuidList(actual), toSortedGuidList(expected), operation + " guid mismatch"); - } - - private void assertPurgeSucceededForRequestedGuids(List requestedGuids, EntityMutationResponse response) { - assertNotNull(response.getPurgedEntities(), "purgeByIds returned no purged entities"); - assertNotNull(response.getPurgeSummary(), "purgeByIds returned no summary"); - - Set purgedGuids = response.getPurgedEntities().stream() - .map(AtlasEntityHeader::getGuid) - .collect(Collectors.toSet()); - - Set skippedGuids = new HashSet<>(); - if (response.getFailedEntities() != null) { - for (FailedEntity failedEntity : response.getFailedEntities()) { - if (PurgeUtils.isSkippablePurgeFailureCode(failedEntity.getErrorCode())) { - skippedGuids.add(failedEntity.getGuid()); - } - } - } - - for (String guid : requestedGuids) { - assertTrue(purgedGuids.contains(guid) || skippedGuids.contains(guid), - "Expected requested guid to be purged or skipped as already removed: " + guid); - } - - assertEquals(response.getPurgeSummary().getRequestedCount(), requestedGuids.size()); - assertEquals(response.getPurgeSummary().getPurgedCount() + response.getPurgeSummary().getSkippedRequestedCount(), - requestedGuids.size()); - assertEquals(response.getPurgeSummary().getFailedCount(), 0); - } - - private static List toSortedGuidList(List headers) { - return headers.stream() - .map(AtlasEntityHeader::getGuid) - .sorted() - .collect(Collectors.toList()); - } - - private AuditSearchParameters createAuditParameter(String fileName) { - try { - return TestResourceFileUtils.readObjectFromJson(AUDIT_PARAMETER_RESOURCE_DIR, fileName, AuditSearchParameters.class); - } catch (IOException e) { - fail(e.getMessage()); - } - - return null; - } - - private void assertPurgeAuditRowsWrittenByPurgeService(AuditSearchParameters auditSearchParameters) { - pauseForIndexCreation(); - - List result; - - try { - result = atlasAuditService.get(auditSearchParameters); - } catch (Exception e) { - throw new SkipException("purge audit entries not retrieved."); - } - - assertNotNull(result); - assertFalse(result.isEmpty()); - - boolean hasSummaryRow = false; - boolean hasBatchRow = false; - for (AtlasAuditEntry entry : result) { - if (entry.getOperation() != AuditOperation.PURGE && entry.getOperation() != AuditOperation.AUTO_PURGE) { - continue; - } - - if (PurgeUtils.isPurgeSummaryAudit(entry)) { - hasSummaryRow = true; - assertNotNull(entry.getRunId(), "Purge summary audit should carry runId"); - assertNotNull(PurgeUtils.parsePurgeSummary(entry), "Summary row should contain parseable PurgeSummary JSON"); - } - - if (PurgeUtils.isPurgeBatchAudit(entry)) { - hasBatchRow = true; - assertNotNull(entry.getRunId(), "Purge batch audit should carry runId"); - } - } - - assertTrue(hasSummaryRow, "Expected at least one purge summary audit row from purgeByIds"); - assertTrue(hasBatchRow, "Expected at least one purge batch audit row from purgeByIds"); - } - - private void assertAuditEntry(AtlasAuditService auditService, AuditSearchParameters auditSearchParameters) { - pauseForIndexCreation(); - - List result; - - try { - result = auditService.get(auditSearchParameters); - } catch (Exception e) { - throw new SkipException("audit entries not retrieved."); - } - - assertNotNull(result); - assertFalse(result.isEmpty()); - } } From a09779bb6d87960dc32c81ca3c3c3a09569c03d1 Mon Sep 17 00:00:00 2001 From: Sheetal Shah Date: Thu, 30 Jul 2026 19:49:32 +0530 Subject: [PATCH 5/9] ATLAS-5317: Fix checkstyle in purge tests and remove unused Python 207 support. --- intg/src/main/python/apache_atlas/utils.py | 17 +++-------------- .../atlas/repository/audit/AdminPurgeTest.java | 1 - .../atlas/web/resources/AdminResourceTest.java | 1 - 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/intg/src/main/python/apache_atlas/utils.py b/intg/src/main/python/apache_atlas/utils.py index 6a3c709b5f4..b8540121220 100644 --- a/intg/src/main/python/apache_atlas/utils.py +++ b/intg/src/main/python/apache_atlas/utils.py @@ -112,20 +112,12 @@ def type_coerce_dict_list(obj, objType): class API: - def __init__(self, path, method, expected_status, consumes=APPLICATION_JSON, produces=APPLICATION_JSON, - alternate_expected_statuses=None): + def __init__(self, path, method, expected_status, consumes=APPLICATION_JSON, produces=APPLICATION_JSON): self.path = path self.method = method self.expected_status = expected_status self.consumes = consumes self.produces = produces - self.alternate_expected_statuses = alternate_expected_statuses or [] - - def matches_expected_status(self, status_code): - if status_code == self.expected_status: - return True - - return status_code in self.alternate_expected_statuses def multipart_urljoin(self, base_path, *path_elems): """Join a base path and multiple context path elements. Handle single @@ -144,13 +136,11 @@ def urljoin_pair(left, right): return reduce(urljoin_pair, path_elems, base_path) def format_path(self, params): - return API(self.path.format(**params), self.method, self.expected_status, self.consumes, self.produces, - self.alternate_expected_statuses) + return API(self.path.format(**params), self.method, self.expected_status, self.consumes, self.produces) def format_path_with_params(self, *params): request_path = self.multipart_urljoin(self.path, *params) - return API(request_path, self.method, self.expected_status, self.consumes, self.produces, - self.alternate_expected_statuses) + return API(request_path, self.method, self.expected_status, self.consumes, self.produces) class HTTPMethod(enum.Enum): @@ -162,6 +152,5 @@ class HTTPMethod(enum.Enum): class HTTPStatus: OK = 200 - MULTI_STATUS = 207 NO_CONTENT = 204 SERVICE_UNAVAILABLE = 503 diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java b/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java index 87e6d4d6edf..2142f4b5324 100644 --- a/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java @@ -21,7 +21,6 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.TestModules; import org.apache.atlas.TestUtilsV2; -import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AtlasAuditEntry; import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation; import org.apache.atlas.model.audit.AuditSearchParameters; diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java index f8bad1b8e0f..81f85b61ddf 100644 --- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.databind.JsonNode; import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.authorize.AtlasAdminAccessRequest; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.discovery.SearchContext; From fdd58ddd18b79ec4224447c6cc7e50e15af68a02 Mon Sep 17 00:00:00 2001 From: Sheetal Shah Date: Fri, 31 Jul 2026 12:09:12 +0530 Subject: [PATCH 6/9] ATLAS-5317: Fix purge batch accounting when related entities are removed in one txn. --- .../store/graph/v2/AtlasEntityStoreV2.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index e7959c3b4b9..126398e30e0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -600,9 +600,22 @@ public EntityMutationResponse purgeEntitiesInBatch(Set purgeCandidates) response.addEntity(PURGE, entry.getValue()); notificationResponse.addEntity(PURGE, entry.getValue()); } else { - String guid = entry.getValue().getGuid(); - LOG.debug("Purge batch skipped guid={} as vertex was not deleted by this batch, assuming concurrently removed", guid); - addPurgeBatchFailure(response, guid, AtlasErrorCode.INSTANCE_GUID_NOT_FOUND); + String guid = entry.getValue().getGuid(); + AtlasVertex vertex = entry.getKey(); + try { + if (!vertex.exists()) { + // Hard-deleted while processing an earlier vertex in this batch (ATLAS-4766). + // Count as purged using the header captured at batch start; do not notify again. + LOG.debug("Purge batch counted guid={} as purged after concurrent removal within batch", guid); + response.addEntity(PURGE, entry.getValue()); + } else { + LOG.debug("Purge batch skipped guid={} as vertex was not deleted by this batch", guid); + addPurgeBatchFailure(response, guid, AtlasErrorCode.INSTANCE_GUID_NOT_FOUND); + } + } catch (IllegalStateException e) { + LOG.debug("Purge batch counted guid={} as purged; batch vertex handle no longer valid", guid, e); + response.addEntity(PURGE, entry.getValue()); + } } } From e9f3d47c768c1fed0a59984e46832c3fbec3a01b Mon Sep 17 00:00:00 2001 From: Sheetal Shah Date: Tue, 4 Aug 2026 18:38:55 +0530 Subject: [PATCH 7/9] ATLAS-5317: Keep existing purge audit consumers working with 88342 batch/summary model --- .../atlas/repository/purge/PurgeUtils.java | 40 +++--- .../atlas/services/PurgeAuditWriter.java | 2 +- .../atlas/web/resources/AdminResource.java | 62 +++++---- .../web/resources/AdminResourceTest.java | 126 +++++++++--------- 4 files changed, 127 insertions(+), 103 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/purge/PurgeUtils.java b/repository/src/main/java/org/apache/atlas/repository/purge/PurgeUtils.java index 0f27962b3c1..24b34230f83 100644 --- a/repository/src/main/java/org/apache/atlas/repository/purge/PurgeUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/purge/PurgeUtils.java @@ -42,7 +42,6 @@ import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -188,10 +187,7 @@ public static void attachPurgeSummary(EntityMutationResponse response, PurgeExec } public static String buildGuidParams(Collection guids) { - return guids.stream() - .filter(Objects::nonNull) - .sorted() - .collect(Collectors.joining(",")); + return guids == null ? "[]" : guids.toString(); } public static boolean isPurgeSummaryAudit(AtlasAuditEntry entry) { @@ -289,16 +285,6 @@ public static void excludeBatchRowsFromAuditSearch(AuditSearchParameters auditSe auditSearchParameters.setAuditFilters(newFilters); } - public static List paginateStringList(List values, int limit, int offset) { - if (values == null || values.isEmpty()) { - return new ArrayList<>(); - } - - int from = Math.min(Math.max(offset, 0), values.size()); - int to = Math.min(from + Math.max(limit, 0), values.size()); - return new ArrayList<>(values.subList(from, to)); - } - public static boolean hasRunIdFilter(SearchParameters.FilterCriteria auditFilters) { if (auditFilters == null) { return false; @@ -336,6 +322,30 @@ public static List collectPurgedGuidsFromBatchEntries(List guids) { + if (entry == null) { + return; + } + + List purgedGuids = guids != null ? guids : new ArrayList<>(); + entry.setResult(buildGuidResult(purgedGuids)); + entry.setResultCount(purgedGuids.size()); + } + + public static String buildGuidResult(Collection guids) { + if (guids == null || guids.isEmpty()) { + return ""; + } + + return guids.stream() + .filter(StringUtils::isNotBlank) + .collect(Collectors.joining(",")); + } + private static void appendPurgedGuidsFromBatchResult(String result, List orderedEntityGuids, Set seen) { if (StringUtils.isBlank(result)) { return; diff --git a/repository/src/main/java/org/apache/atlas/services/PurgeAuditWriter.java b/repository/src/main/java/org/apache/atlas/services/PurgeAuditWriter.java index 238598f2f3f..2f6e9565a98 100644 --- a/repository/src/main/java/org/apache/atlas/services/PurgeAuditWriter.java +++ b/repository/src/main/java/org/apache/atlas/services/PurgeAuditWriter.java @@ -78,7 +78,7 @@ public static void writeBatch(AtlasAuditService auditService, AuditOperation ope result = ""; resultCount = 0; } else { - result = PurgeUtils.buildGuidParams(purgedEntities.stream() + result = PurgeUtils.buildGuidResult(purgedEntities.stream() .map(AtlasEntityHeader::getGuid) .collect(Collectors.toList())); resultCount = purgedEntities.size(); diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index e311119947b..266f75c413e 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -1058,15 +1058,28 @@ public List getAtlasAudits(AuditSearchParameters auditSearchPar if (summaryOnlyListing) { // Defensive post-filter for BATCH rows still returned by the index. - return PurgeUtils.excludeBatchRowsFromResults(auditEntries); + auditEntries = PurgeUtils.excludeBatchRowsFromResults(auditEntries); } + buildPurgeSummaryResults(auditEntries); return auditEntries; } finally { AtlasPerfTracer.log(perf); } } + private void buildPurgeSummaryResults(List auditEntries) throws AtlasBaseException { + if (CollectionUtils.isEmpty(auditEntries)) { + return; + } + + for (AtlasAuditEntry entry : auditEntries) { + if (PurgeUtils.isPurgeSummaryAudit(entry)) { + PurgeUtils.buildPurgeSummaryResult(entry, auditService.getPurgedEntityGuidsForRun(entry)); + } + } + } + @GET @Path("/audit/{auditGuid}/details") @Consumes(Servlets.JSON_MEDIA_TYPE) @@ -1085,24 +1098,24 @@ public List getAuditDetails(@PathParam("auditGuid") String au AtlasAuditEntry auditEntry = auditService.toAtlasAuditEntry(entityStore.getById(auditGuid, false, true)); - // Summary purge audit rows store PurgeSummary JSON in result, not entity GUIDs. - if (auditEntry != null && PurgeUtils.isPurgeSummaryAudit(auditEntry)) { - return ret; - } - - if (auditEntry != null && StringUtils.isNotEmpty(auditEntry.getResult())) { - String[] listOfResultGuid = auditEntry.getResult().split(","); - EntityAuditActionV2 auditAction = auditEntry.getOperation().toEntityAuditActionV2(); + if (auditEntry != null) { + boolean isSummaryPurgeAudit = PurgeUtils.isPurgeSummaryAudit(auditEntry); - if (offset <= listOfResultGuid.length) { - for (int index = offset; index < listOfResultGuid.length && index < (offset + limit); index++) { - List events = auditRepository.listEventsV2(listOfResultGuid[index], auditAction, null, (short) 1); + if (isSummaryPurgeAudit || StringUtils.isNotEmpty(auditEntry.getResult())) { + String[] listOfResultGuid = isSummaryPurgeAudit + ? auditService.getPurgedEntityGuidsForRun(auditEntry).toArray(new String[0]) + : auditEntry.getResult().split(","); + EntityAuditActionV2 auditAction = auditEntry.getOperation().toEntityAuditActionV2(); - for (EntityAuditEventV2 event : events) { - AtlasEntityHeader entityHeader = event.getEntityHeader(); + if (offset <= listOfResultGuid.length) { + for (int index = offset; index < listOfResultGuid.length && index < (offset + limit); index++) { + List events = auditRepository.listEventsV2(listOfResultGuid[index], auditAction, null, (short) 1); - if (entityHeader != null) { - ret.add(entityHeader); + for (EntityAuditEventV2 event : events) { + AtlasEntityHeader entityHeader = event.getEntityHeader(); + if (entityHeader != null) { + ret.add(entityHeader); + } } } } @@ -1116,20 +1129,19 @@ public List getAuditDetails(@PathParam("auditGuid") String au } @GET - @Path("/audit/{auditGuid}/purgedEntities") + @Path("/audit/{auditGuid}/summary") @Produces(Servlets.JSON_MEDIA_TYPE) - public List getPurgeAuditPurgedEntities(@PathParam("auditGuid") String auditGuid, - @QueryParam("limit") @DefaultValue("100") int limit, - @QueryParam("offset") @DefaultValue("0") int offset) throws AtlasBaseException { + public PurgeSummary getPurgeAuditSummary(@PathParam("auditGuid") String auditGuid) throws AtlasBaseException { AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_AUDITS), "Admin Audits"); - AtlasAuditEntry auditEntry = loadPurgeAuditEntry(auditGuid); + AtlasAuditEntry auditEntry = loadSummaryPurgeAuditEntry(auditGuid); + PurgeSummary summary = PurgeUtils.parsePurgeSummary(auditEntry); - List purgedGuids = PurgeUtils.isPurgeSummaryAudit(auditEntry) - ? auditService.getPurgedEntityGuidsForRun(auditEntry) - : PurgeUtils.collectPurgedGuidsFromBatchEntries(Collections.singletonList(auditEntry)); + if (summary == null) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "No purge summary for audit: " + auditGuid); + } - return PurgeUtils.paginateStringList(purgedGuids, limit, offset); + return summary; } @GET diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java index 81f85b61ddf..06d20676ade 100644 --- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java @@ -1264,6 +1264,8 @@ public void testGetAtlasAuditsExcludesBatchRowsWithoutRunId() throws Exception { when(auditService.get(searchParameters)).thenReturn(mockResults); + when(auditService.getPurgedEntityGuidsForRun(summaryEntry)).thenReturn(java.util.Collections.emptyList()); + AdminResource adminResource = createAdminResource(); withAuthorizationBypass(() -> { @@ -1294,6 +1296,38 @@ public void testGetAtlasAuditsExcludesBatchRowsWithoutRunId() throws Exception { }); } + @Test + public void testGetAtlasAuditsBuildPurgeSummaryResults() throws Exception { + String runId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + String entityGuid1 = "11111111-1111-1111-1111-111111111111"; + String entityGuid2 = "22222222-2222-2222-2222-222222222222"; + AuditSearchParameters searchParameters = createPurgeAuditSearchParameters(); + + AtlasAuditEntry summaryEntry = buildSummaryAuditEntry("summary-guid", runId, 2, 2); + summaryEntry.setParams(PurgeUtils.buildGuidParams(java.util.Arrays.asList(entityGuid1, entityGuid2))); + + when(auditService.get(searchParameters)).thenReturn(java.util.Collections.singletonList(summaryEntry)); + when(auditService.getPurgedEntityGuidsForRun(summaryEntry)).thenReturn( + java.util.Arrays.asList(entityGuid1, entityGuid2)); + + AdminResource adminResource = createAdminResource(); + + try (MockedStatic mockedUtils = mockStatic(AtlasAuthorizationUtils.class)) { + mockedUtils.when(() -> AtlasAuthorizationUtils.verifyAccess(any(org.apache.atlas.authorize.AtlasAdminAccessRequest.class), anyString())) + .then(invocation -> null); + + List result = adminResource.getAtlasAudits(searchParameters); + + assertNotNull(result); + assertEquals(result.size(), 1); + assertEquals(result.get(0).getParams(), "[" + entityGuid1 + ", " + entityGuid2 + "]"); + assertEquals(result.get(0).getResult(), entityGuid1 + "," + entityGuid2); + assertEquals(result.get(0).getResultCount(), 2); + } + + verify(auditService).getPurgedEntityGuidsForRun(summaryEntry); + } + @Test public void testGetAuditDetails() throws Exception { String auditGuid = "audit-123"; @@ -1359,10 +1393,21 @@ public void testGetAuditDetailsWithNullResult() throws Exception { public void testGetAuditDetailsForSummaryPurgeAudit() throws Exception { String summaryGuid = "summary-audit-guid"; String runId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + String entityGuid1 = "11111111-1111-1111-1111-111111111111"; AtlasAuditEntry summaryEntry = buildSummaryAuditEntry(summaryGuid, runId, 2, 1); + List mockEvents = new ArrayList<>(); + EntityAuditEventV2 mockEvent = mock(EntityAuditEventV2.class); + AtlasEntityHeader mockHeader = mock(AtlasEntityHeader.class); + when(mockEvent.getEntityHeader()).thenReturn(mockHeader); + mockEvents.add(mockEvent); + when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(summaryEntry); when(entityStore.getById(summaryGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); + when(auditService.getPurgedEntityGuidsForRun(summaryEntry)).thenReturn( + java.util.Collections.singletonList(entityGuid1)); + when(auditRepository.listEventsV2(eq(entityGuid1), any(EntityAuditEventV2.EntityAuditActionV2.class), any(), any(Short.class))) + .thenReturn(mockEvents); AdminResource adminResource = createAdminResource(); @@ -1371,8 +1416,9 @@ public void testGetAuditDetailsForSummaryPurgeAudit() throws Exception { List result = adminResource.getAuditDetails(summaryGuid, 10, 0); assertNotNull(result); - assertTrue(result.isEmpty()); + assertEquals(result.size(), 1); verify(entityStore).getById(summaryGuid, false, true); + verify(auditService).getPurgedEntityGuidsForRun(summaryEntry); } catch (AtlasBaseException e) { throw new RuntimeException(e); } @@ -1380,98 +1426,54 @@ public void testGetAuditDetailsForSummaryPurgeAudit() throws Exception { } @Test - public void testPurgeAuditReadEndpoints() throws Exception { + public void testGetPurgeAuditSummary() throws Exception { String summaryGuid = "summary-audit-guid"; String runId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; - String entityGuid1 = "11111111-1111-1111-1111-111111111111"; - String entityGuid2 = "22222222-2222-2222-2222-222222222222"; - AtlasAuditEntry summaryEntry = buildSummaryAuditEntry(summaryGuid, runId, 2, 2); when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(summaryEntry); when(entityStore.getById(summaryGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); - when(auditService.getPurgedEntityGuidsForRun(summaryEntry)).thenReturn( - java.util.Arrays.asList(entityGuid1, entityGuid2)); - when(auditService.getPurgeBatchAuditGuidsForRun(summaryEntry)).thenReturn( - java.util.Collections.singletonList("batch-guid-1")); AdminResource adminResource = createAdminResource(); withAuthorizationBypass(() -> { try { - List purgedEntities = adminResource.getPurgeAuditPurgedEntities(summaryGuid, 10, 0); - assertNotNull(purgedEntities); - assertEquals(purgedEntities.size(), 2); - assertEquals(purgedEntities.get(0), entityGuid1); - assertEquals(purgedEntities.get(1), entityGuid2); + PurgeSummary summary = adminResource.getPurgeAuditSummary(summaryGuid); - List batches = adminResource.getPurgeAuditBatches(summaryGuid); - assertNotNull(batches); - assertEquals(batches.size(), 1); - assertEquals(batches.get(0), "batch-guid-1"); + assertNotNull(summary); + assertEquals(summary.getRunId(), runId); + assertEquals(summary.getRequestedCount(), 2); + assertEquals(summary.getPurgedCount(), 2); } catch (AtlasBaseException e) { throw new RuntimeException(e); } }); - - verify(auditService).getPurgedEntityGuidsForRun(summaryEntry); } @Test - public void testPurgeAuditPurgedEntitiesForBatchGuid() throws Exception { - String batchGuid = "batch-audit-guid"; + public void testPurgeAuditBatchesEndpoint() throws Exception { + String summaryGuid = "summary-audit-guid"; String runId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; - String entityGuid1 = "11111111-1111-1111-1111-111111111111"; - String entityGuid2 = "22222222-2222-2222-2222-222222222222"; - - AtlasAuditEntry batchEntry = buildBatchAuditEntry(batchGuid, runId, entityGuid1 + "," + entityGuid2); - - when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(batchEntry); - when(entityStore.getById(batchGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); - - AdminResource adminResource = createAdminResource(); - - withAuthorizationBypass(() -> { - try { - List purgedEntities = adminResource.getPurgeAuditPurgedEntities(batchGuid, 10, 0); - assertNotNull(purgedEntities); - assertEquals(purgedEntities.size(), 2); - assertEquals(purgedEntities.get(0), entityGuid1); - assertEquals(purgedEntities.get(1), entityGuid2); - } catch (AtlasBaseException e) { - throw new RuntimeException(e); - } - }); - - verify(auditService, never()).getPurgedEntityGuidsForRun(any()); - } - @Test - public void testPurgeAuditPurgedEntitiesForLegacyPurgeGuid() throws Exception { - String legacyGuid = "legacy-purge-audit-guid"; - String entityGuid1 = "11111111-1111-1111-1111-111111111111"; - String entityGuid2 = "22222222-2222-2222-2222-222222222222"; - - AtlasAuditEntry legacyEntry = buildLegacyPurgeAuditEntry(legacyGuid, entityGuid1 + "," + entityGuid2); + AtlasAuditEntry summaryEntry = buildSummaryAuditEntry(summaryGuid, runId, 2, 2); - when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(legacyEntry); - when(entityStore.getById(legacyGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); + when(auditService.toAtlasAuditEntry(any(AtlasEntityWithExtInfo.class))).thenReturn(summaryEntry); + when(entityStore.getById(summaryGuid, false, true)).thenReturn(mock(AtlasEntityWithExtInfo.class)); + when(auditService.getPurgeBatchAuditGuidsForRun(summaryEntry)).thenReturn( + java.util.Collections.singletonList("batch-guid-1")); AdminResource adminResource = createAdminResource(); withAuthorizationBypass(() -> { try { - List purgedEntities = adminResource.getPurgeAuditPurgedEntities(legacyGuid, 10, 0); - assertNotNull(purgedEntities); - assertEquals(purgedEntities.size(), 2); - assertEquals(purgedEntities.get(0), entityGuid1); - assertEquals(purgedEntities.get(1), entityGuid2); + List batches = adminResource.getPurgeAuditBatches(summaryGuid); + assertNotNull(batches); + assertEquals(batches.size(), 1); + assertEquals(batches.get(0), "batch-guid-1"); } catch (AtlasBaseException e) { throw new RuntimeException(e); } }); - - verify(auditService, never()).getPurgedEntityGuidsForRun(any()); } @Test From d9dada31c9cf8486314427b05138e46614390d49 Mon Sep 17 00:00:00 2001 From: Sheetal Shah Date: Wed, 5 Aug 2026 16:58:29 +0530 Subject: [PATCH 8/9] ATLAS-5317: Merge typedef patches 009 and 010 for __AtlasAuditEntry into one patch. --- ..._model_add_audit_correlation_attributes.json | 17 +++++++++++++++++ .../patches/009-base_model_add_audit_runid.json | 16 ---------------- .../010-base_model_add_audit_row_kind.json | 16 ---------------- 3 files changed, 17 insertions(+), 32 deletions(-) create mode 100644 addons/models/0000-Area0/patches/009-base_model_add_audit_correlation_attributes.json delete mode 100644 addons/models/0000-Area0/patches/009-base_model_add_audit_runid.json delete mode 100644 addons/models/0000-Area0/patches/010-base_model_add_audit_row_kind.json diff --git a/addons/models/0000-Area0/patches/009-base_model_add_audit_correlation_attributes.json b/addons/models/0000-Area0/patches/009-base_model_add_audit_correlation_attributes.json new file mode 100644 index 00000000000..8c1188beb56 --- /dev/null +++ b/addons/models/0000-Area0/patches/009-base_model_add_audit_correlation_attributes.json @@ -0,0 +1,17 @@ +{ + "patches": [ + { + "id": "TYPEDEF_PATCH_0009_001", + "description": "Add runId and auditRowKind to __AtlasAuditEntry for purge audit correlation", + "action": "ADD_ATTRIBUTE", + "typeName": "__AtlasAuditEntry", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "params": null, + "attributeDefs": [ + { "name": "runId", "typeName": "string", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false }, + { "name": "auditRowKind", "typeName": "audit_row_kind", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false } + ] + } + ] +} diff --git a/addons/models/0000-Area0/patches/009-base_model_add_audit_runid.json b/addons/models/0000-Area0/patches/009-base_model_add_audit_runid.json deleted file mode 100644 index de140e91351..00000000000 --- a/addons/models/0000-Area0/patches/009-base_model_add_audit_runid.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "patches": [ - { - "id": "TYPEDEF_PATCH_0009_001", - "description": "Add runId attribute to __AtlasAuditEntry for purge audit correlation", - "action": "ADD_ATTRIBUTE", - "typeName": "__AtlasAuditEntry", - "applyToVersion": "1.0", - "updateToVersion": "1.1", - "params": null, - "attributeDefs": [ - { "name": "runId", "typeName": "string", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false } - ] - } - ] -} diff --git a/addons/models/0000-Area0/patches/010-base_model_add_audit_row_kind.json b/addons/models/0000-Area0/patches/010-base_model_add_audit_row_kind.json deleted file mode 100644 index 881b4a313f4..00000000000 --- a/addons/models/0000-Area0/patches/010-base_model_add_audit_row_kind.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "patches": [ - { - "id": "TYPEDEF_PATCH_0010_001", - "description": "Add auditRowKind attribute to __AtlasAuditEntry for batch/summary audit correlation", - "action": "ADD_ATTRIBUTE", - "typeName": "__AtlasAuditEntry", - "applyToVersion": "1.1", - "updateToVersion": "1.2", - "params": null, - "attributeDefs": [ - { "name": "auditRowKind", "typeName": "audit_row_kind", "cardinality": "SINGLE", "isIndexable": true, "isOptional": true, "isUnique": false } - ] - } - ] -} From 4ec4dcc795c77eb8ec7827168ff05d5c5b6a0848 Mon Sep 17 00:00:00 2001 From: Sheetal Shah Date: Thu, 6 Aug 2026 12:53:30 +0530 Subject: [PATCH 9/9] ATLAS-5317: Fix purge audit tests for legacy buildGuidParams params format. --- .../atlas/repository/audit/AtlasAuditServiceTest.java | 6 +++--- .../java/org/apache/atlas/services/PurgeServiceTest.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java index 9aee74dab16..5b96f6b11df 100644 --- a/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java @@ -189,7 +189,7 @@ public void purgeAuditWriter_writeBatchAndFinishRun_writesBatchAndSummaryAudits( verify(mockAuditService).add(eq(AuditOperation.PURGE), batchParamsCaptor.capture(), batchResultCaptor.capture(), eq(2L), eq(TEST_RUN_ID), eq(AuditRowKind.BATCH)); - assertEquals(batchParamsCaptor.getValue(), "guid1,guid2"); + assertEquals(batchParamsCaptor.getValue(), PurgeUtils.buildGuidParams(batchGuids)); assertEquals(batchResultCaptor.getValue(), "guid1,guid2"); AtlasAuditEntry batchEntry = new AtlasAuditEntry(); @@ -213,7 +213,7 @@ public void purgeAuditWriter_writeBatchAndFinishRun_writesBatchAndSummaryAudits( verify(mockAuditService).add(eq(AuditOperation.PURGE), summaryParamsCaptor.capture(), summaryResultCaptor.capture(), eq(1L), eq(TEST_RUN_ID), eq(AuditRowKind.SUMMARY)); - assertEquals(summaryParamsCaptor.getValue(), "11111111-1111-1111-1111-111111111111,22222222-2222-2222-2222-222222222222"); + assertEquals(summaryParamsCaptor.getValue(), PurgeUtils.buildGuidParams(originallyRequestedGuids)); PurgeSummary summary = AtlasJson.fromJson(summaryResultCaptor.getValue(), PurgeSummary.class); assertEquals(summary.getRunId(), TEST_RUN_ID); @@ -237,7 +237,7 @@ public void purgeAuditWriter_writeBatchAndFinishRun_writesBatchAndSummaryAudits( verify(mockAuditService).add(eq(AuditOperation.PURGE), emptyBatchParamsCaptor.capture(), emptyBatchResultCaptor.capture(), eq(0L), eq(TEST_RUN_ID), eq(AuditRowKind.BATCH)); - assertEquals(emptyBatchParamsCaptor.getValue(), "guid-a,guid-b"); + assertEquals(emptyBatchParamsCaptor.getValue(), PurgeUtils.buildGuidParams(emptyBatchGuids)); assertEquals(emptyBatchResultCaptor.getValue(), ""); AtlasAuditEntry emptyBatchEntry = new AtlasAuditEntry(); diff --git a/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java b/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java index 4f14fbf8a0e..a1546409a30 100644 --- a/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/services/PurgeServiceTest.java @@ -394,8 +394,8 @@ public void cronFailure_afterGuidsCollected_writesSummaryAuditWithRunId() throws assertNotNull(summary.getRunId()); assertTrue(summary.getRequestedCount() > 0); - verify(mockAuditService).add(eq(AuditOperation.AUTO_PURGE), eq(CRON_ELIGIBLE_GUID), anyString(), - eq(0L), eq(summary.getRunId()), eq(AuditRowKind.SUMMARY)); + verify(mockAuditService).add(eq(AuditOperation.AUTO_PURGE), eq(PurgeUtils.buildGuidParams(originallyRequestedGuids)), + anyString(), eq(0L), eq(summary.getRunId()), eq(AuditRowKind.SUMMARY)); } // -------------------------------------------------------------------------