1575 - implementation of ingest relationships and GET Relationships API

gradle/libs.versions.toml

@@ -2,7 +2,7 @@
 spring-webflux = '6.0.9'
 reactor-extra = '3.5.1'
 micrometer-registry-prometheus = '1.9.0'
-ingestion-contract-server = '0.1.32'
+ingestion-contract-server = '0.1.38'
 oddrn-generator-java = '0.1.21'
 odd-integration-manifests = '0.0.6'
 apache-collections = '4.4'

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/config/R2DBCConfiguration.java

@@ -47,8 +47,7 @@ public ConnectionPool connectionFactory(final DataSourceProperties dataSourcePro
         return new ConnectionPool(builder.build());
     }
 
-    @Bean(destroyMethod = "dispose")
-    @Qualifier("customConnectionPool")
+    @Bean(name = "customConnectionPool", destroyMethod = "dispose")
     public ConnectionPool databaseClientForCustomSchema(
         @Value("${spring.custom-datasource.url}") final String url,
         @Value("${spring.custom-datasource.username}") final String username,
@@ -83,8 +82,7 @@ public DatabaseClient databaseClient(final ConnectionFactory schema1ConnectionFa
         return DatabaseClient.create(schema1ConnectionFactory);
     }
 
-    @Bean
-    @Qualifier("customDataClient")
+    @Bean(name = "customDataClient")
     public DatabaseClient databaseClientCustomTables(
         @Qualifier("customConnectionPool") final ConnectionFactory connectionFactory) {
         return DatabaseClient.create(connectionFactory);
@@ -96,8 +94,7 @@ public ReactiveTransactionManager reactiveTransactionManager(final ConnectionFac
         return new R2dbcTransactionManager(connectionFactory);
     }
 
-    @Bean
-    @Qualifier("customTransactionManager")
+    @Bean(name = "customTransactionManager")
     public ReactiveTransactionManager reactiveCustomTransactionManager(
         @Qualifier("customConnectionPool") final ConnectionFactory connectionFactory) {
         return new R2dbcTransactionManager(connectionFactory);

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DatasetController.java

@@ -2,9 +2,12 @@
 
 import lombok.RequiredArgsConstructor;
 import org.opendatadiscovery.oddplatform.api.contract.api.DataSetApi;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRelationshipList;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataSetStructure;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataSetVersionDiffList;
+import org.opendatadiscovery.oddplatform.api.contract.model.RelationshipsType;
 import org.opendatadiscovery.oddplatform.service.DatasetVersionService;
+import org.opendatadiscovery.oddplatform.service.RelationshipsService;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.server.ServerWebExchange;
@@ -14,6 +17,7 @@
 @RequiredArgsConstructor
 public class DatasetController implements DataSetApi {
     private final DatasetVersionService datasetVersionService;
+    private final RelationshipsService relationshipsService;
 
     @Override
     public Mono<ResponseEntity<DataSetStructure>> getDataSetStructureByVersionId(
@@ -44,4 +48,12 @@ public Mono<ResponseEntity<DataSetVersionDiffList>> getDataSetStructureDiff(fina
         return datasetVersionService.getDatasetVersionDiff(dataEntityId, firstVersionId, secondVersionId)
             .map(ResponseEntity::ok);
     }
+
+    @Override
+    public Mono<ResponseEntity<DataEntityRelationshipList>> getDataSetRelationships(final Long dataEntityId,
+                                                                                 final RelationshipsType type,
+                                                                                 final ServerWebExchange exchange) {
+        return relationshipsService.getRelationsByDatasetId(dataEntityId, type)
+            .map(ResponseEntity::ok);
+    }
 }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/RelationshipController.java

@@ -0,0 +1,45 @@
+package org.opendatadiscovery.oddplatform.controller;
+
+import lombok.RequiredArgsConstructor;
+import org.opendatadiscovery.oddplatform.api.contract.api.RelationshipApi;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityERDRelationshipDetails;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityGraphRelationshipDetails;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRelationshipList;
+import org.opendatadiscovery.oddplatform.api.contract.model.RelationshipsType;
+import org.opendatadiscovery.oddplatform.service.RelationshipsService;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.server.ServerWebExchange;
+import reactor.core.publisher.Mono;
+
+@RestController
+@RequiredArgsConstructor
+public class RelationshipController implements RelationshipApi {
+    private final RelationshipsService relationshipsService;
+
+    @Override
+    public Mono<ResponseEntity<DataEntityRelationshipList>> getRelationships(final Integer page,
+                                                                             final Integer size,
+                                                                             final RelationshipsType type,
+                                                                             final String query,
+                                                                             final ServerWebExchange exchange) {
+        return relationshipsService.getRelationships(page, size, type, query)
+            .map(ResponseEntity::ok);
+    }
+
+    @Override
+    public Mono<ResponseEntity<DataEntityERDRelationshipDetails>>
+        getERDRelationshipById(final Long relationshipId,
+                               final ServerWebExchange exchange) {
+        return relationshipsService.getERDRelationshipById(relationshipId)
+            .map(ResponseEntity::ok);
+    }
+
+    @Override
+    public Mono<ResponseEntity<DataEntityGraphRelationshipDetails>>
+        getGraphRelationshipById(final Long relationshipId,
+                                 final ServerWebExchange exchange) {
+        return relationshipsService.getGraphRelationshipById(relationshipId)
+            .map(ResponseEntity::ok);
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityClassDto.java

@@ -32,6 +32,7 @@
 import static org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto.ML_MODEL_ARTIFACT;
 import static org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto.ML_MODEL_INSTANCE;
 import static org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto.ML_MODEL_TRAINING;
+import static org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto.RELATIONSHIP;
 import static org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto.TABLE;
 import static org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto.VECTOR_STORE;
 import static org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto.VIEW;
@@ -45,7 +46,8 @@ public enum DataEntityClassDto {
     DATA_QUALITY_TEST_RUN(5, JOB_RUN),
     DATA_CONSUMER(6, Set.of(ML_MODEL_ARTIFACT, DASHBOARD)),
     DATA_INPUT(7, API_CALL),
-    DATA_ENTITY_GROUP(8, Set.of(ML_EXPERIMENT, DAG, DATABASE_SERVICE, API_SERVICE, KAFKA_SERVICE, DOMAIN));
+    DATA_ENTITY_GROUP(8, Set.of(ML_EXPERIMENT, DAG, DATABASE_SERVICE, API_SERVICE, KAFKA_SERVICE, DOMAIN)),
+    DATA_RELATIONSHIP(9, RELATIONSHIP);
 
     private static final Map<Integer, DataEntityClassDto> MAP = Arrays.stream(DataEntityClassDto.values())
         .collect(Collectors.toMap(DataEntityClassDto::getId, identity()));

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityTypeDto.java

@@ -33,7 +33,8 @@ public enum DataEntityTypeDto {
     KAFKA_SERVICE(21),
     DOMAIN(22),
     VECTOR_STORE(23),
-    LOOKUP_TABLE(24);
+    LOOKUP_TABLE(24),
+    RELATIONSHIP(25);
 
     private static final Map<Integer, DataEntityTypeDto> MAP = Arrays
         .stream(DataEntityTypeDto.values())

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipDetailsDto.java

@@ -0,0 +1,16 @@
+package org.opendatadiscovery.oddplatform.dto;
+
+import lombok.Builder;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.ErdRelationshipDetailsPojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.GraphRelationshipPojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.RelationshipsPojo;
+
+@Builder
+public record RelationshipDetailsDto(DataEntityPojo dataEntityRelationship,
+                                     RelationshipsPojo relationshipPojo,
+                                     DataEntityPojo sourceDataEntity,
+                                     DataEntityPojo targetDataEntity,
+                                     ErdRelationshipDetailsPojo erdRelationshipDetailsPojo,
+                                     GraphRelationshipPojo graphRelationshipPojo) {
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipDto.java

@@ -0,0 +1,12 @@
+package org.opendatadiscovery.oddplatform.dto;
+
+import lombok.Builder;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.RelationshipsPojo;
+
+@Builder
+public record RelationshipDto(DataEntityPojo dataEntityRelationship,
+                              RelationshipsPojo relationshipPojo,
+                              DataEntityPojo sourceDataEntity,
+                              DataEntityPojo targetDataEntity) {
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipStatusDto.java

@@ -0,0 +1,29 @@
+package org.opendatadiscovery.oddplatform.dto;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import lombok.Getter;
+
+import static java.util.function.Function.identity;
+
+public enum RelationshipStatusDto {
+    ACTIVE(1),
+    DELETED(2);
+
+    @Getter
+    private final short id;
+
+    RelationshipStatusDto(final int id) {
+        this.id = (short) id;
+    }
+
+    private static final Map<Short, DataEntityStatusDto> MAP = Arrays
+        .stream(DataEntityStatusDto.values())
+        .collect(Collectors.toMap(DataEntityStatusDto::getId, identity()));
+
+    public static Optional<DataEntityStatusDto> findById(final short id) {
+        return Optional.ofNullable(MAP.get(id));
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipTypeDto.java

@@ -0,0 +1,6 @@
+package org.opendatadiscovery.oddplatform.dto;
+
+public enum RelationshipTypeDto {
+    ERD,
+    GRAPH
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/DataEntityIngestionDto.java

@@ -10,8 +10,11 @@
 import lombok.NoArgsConstructor;
 import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto;
 import org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto;
+import org.opendatadiscovery.oddplatform.dto.RelationshipTypeDto;
 import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataSetFieldEnumValue;
 import org.opendatadiscovery.oddplatform.model.tables.pojos.DatasetFieldPojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.ErdRelationshipDetailsPojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.GraphRelationshipPojo;
 
 @Data
 @Builder
@@ -36,6 +39,7 @@ public class DataEntityIngestionDto {
     protected DataQualityTestIngestionDto dataQualityTest;
     protected DataInputIngestionDto dataInput;
     protected DataEntityGroupDto dataEntityGroup;
+    protected DataRelationshipDto dataRelationshipDto;
 
     public record DataSetIngestionDto(String parentDatasetOddrn,
                                       List<DatasetFieldIngestionDto> fieldList,
@@ -63,4 +67,14 @@ public record DatasetFieldIngestionDto(DatasetFieldPojo field,
                                            List<DataSetFieldEnumValue> enumValues,
                                            Map<String, Object> metadata) {
     }
+
+    public record DataRelationshipDto(String sourceOddrn,
+                                      String targetOddrn,
+                                      RelationshipTypeDto relationshipType,
+                                      DataRelationshipDetailsDto details) {
+    }
+
+    public record DataRelationshipDetailsDto(ErdRelationshipDetailsPojo erdRelationshipDetailsPojo,
+                                             GraphRelationshipPojo graphRelationshipPojo) {
+    }
 }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/EnrichedDataEntityIngestionDto.java

@@ -18,7 +18,7 @@ public EnrichedDataEntityIngestionDto(final long id,
             ingestionDto.sourceCreatedAt, ingestionDto.sourceUpdatedAt, ingestionDto.entityClasses, ingestionDto.type,
             ingestionDto.metadata, ingestionDto.tags, ingestionDto.specificAttributesJson, ingestionDto.dataSet,
             ingestionDto.dataTransformer, ingestionDto.dataConsumer, ingestionDto.dataQualityTest,
-            ingestionDto.dataInput, ingestionDto.dataEntityGroup);
+            ingestionDto.dataInput, ingestionDto.dataEntityGroup, ingestionDto.dataRelationshipDto);
 
         this.id = id;
         this.previousVersionPojo = previousVersionPojo;

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/ErdRelationshipMapper.java

@@ -0,0 +1,31 @@
+package org.opendatadiscovery.oddplatform.mapper;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.opendatadiscovery.oddplatform.api.contract.model.ERDRelationshipDetails;
+import org.opendatadiscovery.oddplatform.api.contract.model.ERDRelationshipPairs;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.ErdRelationshipDetailsPojo;
+import org.springframework.stereotype.Component;
+import org.thymeleaf.util.ArrayUtils;
+
+@Component
+public class ErdRelationshipMapper {
+    public ERDRelationshipDetails mapPojoToDetails(final ErdRelationshipDetailsPojo erd) {
+        return new ERDRelationshipDetails()
+            .erdRelationshipId(erd.getId())
+            .fieldsPairs(mapPairs(erd.getSourceDatasetFieldOddrn(), erd.getTargetDatasetFieldOddrn()))
+            .isIdentifying(erd.getIsIdentifying())
+            .cardinality(erd.getCardinality());
+    }
+
+    private List<ERDRelationshipPairs> mapPairs(final String[] source, final String[] target) {
+        if (ArrayUtils.isEmpty(source)) {
+            return null;
+        }
+
+        return IntStream.range(0, source.length).mapToObj(i -> new ERDRelationshipPairs()
+                .sourceDatasetFieldOddrn(source[i])
+                .targetDatasetFieldOddrn(target[i])).collect(Collectors.toList());
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/GraphRelationshipMapper.java

@@ -0,0 +1,46 @@
+package org.opendatadiscovery.oddplatform.mapper;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import java.util.List;
+import java.util.Map;
+import org.jooq.JSONB;
+import org.opendatadiscovery.oddplatform.api.contract.model.GraphRelationshipAttributes;
+import org.opendatadiscovery.oddplatform.api.contract.model.GraphRelationshipDetails;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.GraphRelationshipPojo;
+import org.springframework.stereotype.Component;
+
+@Component
+public class GraphRelationshipMapper {
+    public GraphRelationshipDetails mapPojoToDetails(final GraphRelationshipPojo pojo) {
+        if (pojo == null) {
+            return null;
+        }
+
+        return new GraphRelationshipDetails()
+            .graphRelationshipId(pojo.getId())
+            .isDirected(pojo.getIsDirected())
+            .attributes(mapGraphAttributes(pojo.getSpecificAttributes()));
+    }
+
+    private List<GraphRelationshipAttributes> mapGraphAttributes(final JSONB specificAttributes) {
+        if (specificAttributes == null) {
+            return List.of();
+        }
+
+        final ObjectReader reader = new ObjectMapper().readerFor(Map.class);
+
+        try {
+            final Map<String, String> map = reader.readValue(specificAttributes.data());
+
+            return map.entrySet().stream()
+                .map(item -> new GraphRelationshipAttributes()
+                    .name(item.getKey())
+                    .value(item.getValue()))
+                .toList();
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/RelationshipDetailsMapper.java

@@ -0,0 +1,58 @@
+package org.opendatadiscovery.oddplatform.mapper;
+
+import lombok.RequiredArgsConstructor;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityERDRelationshipDetails;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityGraphRelationshipDetails;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRelationshipType;
+import org.opendatadiscovery.oddplatform.dto.RelationshipDetailsDto;
+import org.opendatadiscovery.oddplatform.dto.RelationshipTypeDto;
+import org.springframework.stereotype.Component;
+
+@RequiredArgsConstructor
+@Component
+public class RelationshipDetailsMapper {
+    private final ErdRelationshipMapper erdRelationshipMapper;
+    private final GraphRelationshipMapper graphRelationshipMapper;
+
+    public DataEntityERDRelationshipDetails mapToDataEntityERDRelationshipDetails(
+        final RelationshipDetailsDto relationshipDetailsDto) {
+        return new DataEntityERDRelationshipDetails()
+            .id(relationshipDetailsDto.dataEntityRelationship().getId())
+            .name(relationshipDetailsDto.dataEntityRelationship().getExternalName())
+            .oddrn(relationshipDetailsDto.dataEntityRelationship().getOddrn())
+            .sourceDatasetOddrn(relationshipDetailsDto.relationshipPojo().getSourceDatasetOddrn())
+            .targetDatasetOddrn(relationshipDetailsDto.relationshipPojo().getTargetDatasetOddrn())
+            .sourceDataEntityId(
+                relationshipDetailsDto.sourceDataEntity() != null ? relationshipDetailsDto.sourceDataEntity().getId() :
+                    null)
+            .targetDataEntityId(
+                relationshipDetailsDto.targetDataEntity() != null ? relationshipDetailsDto.targetDataEntity().getId() :
+                    null)
+            .type(RelationshipTypeDto.ERD.name().equals(relationshipDetailsDto.relationshipPojo().getRelationshipType())
+                ? DataEntityRelationshipType.ERD
+                : DataEntityRelationshipType.GRAPH)
+            .erdRelationship(
+                erdRelationshipMapper.mapPojoToDetails(relationshipDetailsDto.erdRelationshipDetailsPojo()));
+    }
+
+    public DataEntityGraphRelationshipDetails mapToDataEntityGraphRelationshipDetails(
+        final RelationshipDetailsDto relationshipDetailsDto) {
+        return new DataEntityGraphRelationshipDetails()
+            .id(relationshipDetailsDto.dataEntityRelationship().getId())
+            .name(relationshipDetailsDto.dataEntityRelationship().getExternalName())
+            .oddrn(relationshipDetailsDto.dataEntityRelationship().getOddrn())
+            .sourceDatasetOddrn(relationshipDetailsDto.relationshipPojo().getSourceDatasetOddrn())
+            .targetDatasetOddrn(relationshipDetailsDto.relationshipPojo().getTargetDatasetOddrn())
+            .sourceDataEntityId(
+                relationshipDetailsDto.sourceDataEntity() != null ? relationshipDetailsDto.sourceDataEntity().getId() :
+                    null)
+            .targetDataEntityId(
+                relationshipDetailsDto.targetDataEntity() != null ? relationshipDetailsDto.targetDataEntity().getId() :
+                    null)
+            .type(RelationshipTypeDto.ERD.name().equals(relationshipDetailsDto.relationshipPojo().getRelationshipType())
+                ? DataEntityRelationshipType.ERD
+                : DataEntityRelationshipType.GRAPH)
+            .graphRelationship(
+                graphRelationshipMapper.mapPojoToDetails(relationshipDetailsDto.graphRelationshipPojo()));
+    }
+}