Data deprecation & metadata stale (#1399)

Co-authored-by: nikitadementev <ironic.cosmos@gmail.com>
Co-authored-by: alexeykozyurov <akozyurov@provectus.com>

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/auth/util/SecurityConstants.java

@@ -31,12 +31,12 @@
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_DELETE_TERM;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_DESCRIPTION_UPDATE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_GROUP_CREATE;
-import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_GROUP_DELETE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_GROUP_UPDATE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_INTERNAL_NAME_UPDATE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_OWNERSHIP_CREATE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_OWNERSHIP_DELETE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_OWNERSHIP_UPDATE;
+import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_STATUS_UPDATE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_TAGS_UPDATE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_SOURCE_CREATE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_SOURCE_DELETE;
@@ -243,6 +243,11 @@ DATA_ENTITY, new PathPatternParserServerWebExchangeMatcher(
             new PathPatternParserServerWebExchangeMatcher("/api/dataentities/{data_entity_id}/links/{link_id}", DELETE),
             DATA_ENTITY_ATTACHMENT_MANAGE
         ),
+        new SecurityRule(
+            DATA_ENTITY,
+            new PathPatternParserServerWebExchangeMatcher("/api/dataentities/{data_entity_id}/statuses", PUT),
+            DATA_ENTITY_STATUS_UPDATE
+        ),
         new SecurityRule(DATASET_FIELD,
             new PathPatternParserServerWebExchangeMatcher("/api/datasetfields/{dataset_field_id}/description", PUT),
             DATASET_FIELD_DESCRIPTION_UPDATE),
@@ -269,9 +274,6 @@ DATA_ENTITY, new PathPatternParserServerWebExchangeMatcher(
         new SecurityRule(
             AuthorizationManagerType.DEG,
             new PathPatternParserServerWebExchangeMatcher("/api/dataentitygroups/{data_entity_group_id}", PUT),
-            DATA_ENTITY_GROUP_UPDATE),
-        new SecurityRule(AuthorizationManagerType.DEG,
-            new PathPatternParserServerWebExchangeMatcher("/api/dataentitygroups/{data_entity_group_id}", DELETE),
-            DATA_ENTITY_GROUP_DELETE)
+            DATA_ENTITY_GROUP_UPDATE)
     );
 }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DataEntityController.java

@@ -22,6 +22,8 @@
 import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityLineage;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityList;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusFormData;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityTermFormData;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityUsageInfo;
 import org.opendatadiscovery.oddplatform.api.contract.model.InternalDescription;
@@ -82,13 +84,6 @@ public Mono<ResponseEntity<DataEntityRef>> createDataEntityGroup(final Mono<Data
             .map(ResponseEntity::ok);
     }
 
-    @Override
-    public Mono<ResponseEntity<Void>> deleteDataEntityGroup(final Long dataEntityGroupId,
-                                                            final ServerWebExchange exchange) {
-        return dataEntityGroupService.deleteDataEntityGroup(dataEntityGroupId)
-            .thenReturn(ResponseEntity.noContent().build());
-    }
-
     @Override
     public Mono<ResponseEntity<DataEntityRef>> updateDataEntityGroup(final Long dataEntityGroupId,
                                                                      final Mono<DataEntityGroupFormData> formData,
@@ -191,6 +186,15 @@ public Mono<ResponseEntity<Ownership>> updateOwnership(final Long dataEntityId,
             .map(ResponseEntity::ok);
     }
 
+    @Override
+    public Mono<ResponseEntity<DataEntityStatus>> updateStatus(final Long dataEntityId,
+                                                               final Mono<DataEntityStatusFormData> dataEntityStatus,
+                                                               final ServerWebExchange exchange) {
+        return dataEntityStatus
+            .flatMap(status -> dataEntityService.updateStatus(dataEntityId, status))
+            .map(ResponseEntity::ok);
+    }
+
     @Override
     public Mono<ResponseEntity<InternalDescription>> upsertDataEntityInternalDescription(
         final Long dataEntityId,

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityStatusDto.java

@@ -0,0 +1,36 @@
+package org.opendatadiscovery.oddplatform.dto;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import lombok.Getter;
+
+import static java.util.function.Function.identity;
+
+public enum DataEntityStatusDto {
+    UNASSIGNED(1, false),
+    DRAFT(2, true),
+    STABLE(3, false),
+    DEPRECATED(4, true),
+    DELETED(5, false);
+
+    @Getter
+    private final short id;
+    @Getter
+    private final boolean isSwitchable;
+
+    DataEntityStatusDto(final int id,
+                        final boolean isSwitchable) {
+        this.id = (short) id;
+        this.isSwitchable = isSwitchable;
+    }
+
+    private static final Map<Short, DataEntityStatusDto> MAP = Arrays
+        .stream(DataEntityStatusDto.values())
+        .collect(Collectors.toMap(DataEntityStatusDto::getId, identity()));
+
+    public static Optional<DataEntityStatusDto> findById(final short id) {
+        return Optional.ofNullable(MAP.get(id));
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java

@@ -7,7 +7,8 @@ public enum FacetType {
     DATA_SOURCES,
     OWNERS,
     TAGS,
-    GROUPS;
+    GROUPS,
+    STATUSES;
 
     public static FacetType lookup(final String facetType) {
         for (final FacetType ft : FacetType.values()) {

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/activity/ActivityEventTypeDto.java

@@ -13,6 +13,7 @@ public enum ActivityEventTypeDto {
     TERM_ASSIGNMENT_UPDATED,
     DESCRIPTION_UPDATED,
     BUSINESS_NAME_UPDATED,
+    DATA_ENTITY_STATUS_UPDATED,
     CUSTOM_METADATA_CREATED,
     CUSTOM_METADATA_UPDATED,
     CUSTOM_METADATA_DELETED,
@@ -22,7 +23,6 @@ public enum ActivityEventTypeDto {
     DATASET_FIELD_TERM_ASSIGNMENT_UPDATED,
     CUSTOM_GROUP_CREATED,
     CUSTOM_GROUP_UPDATED,
-    CUSTOM_GROUP_DELETED,
     ALERT_HALT_CONFIG_UPDATED,
     ALERT_STATUS_UPDATED,
     OPEN_ALERT_RECEIVED,

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/activity/DataEntityStatusUpdatedDto.java

@@ -0,0 +1,8 @@
+package org.opendatadiscovery.oddplatform.dto.activity;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.time.LocalDateTime;
+
+public record DataEntityStatusUpdatedDto(String status,
+                                         @JsonProperty("status_switch_time") LocalDateTime statusSwitchTime) {
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/DataEntityIngestionDto.java

@@ -22,8 +22,8 @@ public class DataEntityIngestionDto {
     protected String oddrn;
     protected long dataSourceId;
     protected String externalDescription;
-    protected OffsetDateTime createdAt;
-    protected OffsetDateTime updatedAt;
+    protected OffsetDateTime sourceCreatedAt;
+    protected OffsetDateTime sourceUpdatedAt;
     protected Set<DataEntityClassDto> entityClasses;
     protected DataEntityTypeDto type;
     protected Map<String, Object> metadata;

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/EnrichedDataEntityIngestionDto.java

@@ -2,29 +2,25 @@
 
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo;
 
 @EqualsAndHashCode(callSuper = true)
 @Data
 public class EnrichedDataEntityIngestionDto extends DataEntityIngestionDto {
     private long id;
-    private boolean updated;
-
+    private DataEntityPojo previousVersionPojo;
     private Boolean datasetSchemaChanged;
 
-    public EnrichedDataEntityIngestionDto(final long id, final DataEntityIngestionDto ingestionDto) {
-        this(id, ingestionDto, true);
-    }
-
     public EnrichedDataEntityIngestionDto(final long id,
-                                          final DataEntityIngestionDto ingestionDto,
-                                          final boolean updated) {
+                                          final DataEntityPojo previousVersionPojo,
+                                          final DataEntityIngestionDto ingestionDto) {
         super(ingestionDto.name, ingestionDto.oddrn, ingestionDto.dataSourceId, ingestionDto.externalDescription,
-            ingestionDto.createdAt, ingestionDto.updatedAt, ingestionDto.entityClasses, ingestionDto.type,
+            ingestionDto.sourceCreatedAt, ingestionDto.sourceUpdatedAt, ingestionDto.entityClasses, ingestionDto.type,
             ingestionDto.metadata, ingestionDto.tags, ingestionDto.specificAttributesJson, ingestionDto.dataSet,
             ingestionDto.dataTransformer, ingestionDto.dataConsumer, ingestionDto.dataQualityTest,
             ingestionDto.dataInput, ingestionDto.dataEntityGroup);
 
         this.id = id;
-        this.updated = updated;
+        this.previousVersionPojo = previousVersionPojo;
     }
 }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/policy/PolicyPermissionDto.java

@@ -33,8 +33,8 @@ public enum PolicyPermissionDto {
     DATASET_FIELD_DELETE_TERM(DATA_ENTITY),
     DATA_ENTITY_GROUP_CREATE(MANAGEMENT),
     DATA_ENTITY_GROUP_UPDATE(DATA_ENTITY),
-    DATA_ENTITY_GROUP_DELETE(DATA_ENTITY),
     DATA_ENTITY_ATTACHMENT_MANAGE(DATA_ENTITY),
+    DATA_ENTITY_STATUS_UPDATE(DATA_ENTITY),
     TERM_CREATE(MANAGEMENT),
     TERM_UPDATE(TERM),
     TERM_DELETE(TERM),

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/config/HousekeepingTTLProperties.java

@@ -8,4 +8,5 @@
 public class HousekeepingTTLProperties {
     private int resolvedAlertsDays;
     private int searchFacetsDays;
+    private int dataEntityDeleteDays;
 }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/job/AlertHousekeepingJob.java

@@ -8,6 +8,7 @@
 import org.jooq.impl.DSL;
 import org.opendatadiscovery.oddplatform.dto.alert.AlertStatusEnum;
 import org.opendatadiscovery.oddplatform.housekeeping.config.HousekeepingTTLProperties;
+import org.opendatadiscovery.oddplatform.service.ingestion.util.DateTimeUtil;
 import org.springframework.stereotype.Component;
 
 import static org.opendatadiscovery.oddplatform.model.Tables.ALERT_CHUNK;
@@ -29,7 +30,7 @@ public void doHousekeeping(final Connection connection) {
                 .where(ALERT.STATUS.eq(AlertStatusEnum.RESOLVED.getCode()))
                 .or(ALERT.STATUS.eq(AlertStatusEnum.RESOLVED_AUTOMATICALLY.getCode()))
                 .and(ALERT.STATUS_UPDATED_AT.lessOrEqual(
-                    DSL.currentLocalDateTime().minus(housekeepingTTLProperties.getResolvedAlertsDays())))
+                    DateTimeUtil.generateNow().minusDays(housekeepingTTLProperties.getResolvedAlertsDays())))
                 .fetch(ALERT.ID);
 
             dslContext.deleteFrom(ALERT_CHUNK)