1456 - created new entity DataSourceSafe without token which used in … (

#1466)
opendatadiscovery · Oct 13, 2023 · 3ca6572 · 3ca6572
1 parent edb15cf
commit 3ca6572
Show file tree

Hide file tree

Showing 12 changed files with 84 additions and 22 deletions.
diff --git a/...form-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapperImpl.java b/...form-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapperImpl.java
@@ -62,6 +62,7 @@ public class DataEntityMapperImpl implements DataEntityMapper {
     private static DataEntityClassAndTypeDictionary TYPE_DICTIONARY = null;
 
     private final DataSourceMapper dataSourceMapper;
+    private final DataSourceSafeMapper dataSourceSafeMapper;
     private final OwnershipMapper ownershipMapper;
     private final TagMapper tagMapper;
     private final MetadataFieldValueMapper metadataFieldValueMapper;
@@ -86,7 +87,7 @@ public DataEntity mapPojo(final DataEntityDimensionsDto dto) {
             .entityClasses(entityClasses.stream().map(this::mapEntityClass).toList())
             .type(type)
             .ownership(ownershipMapper.mapDtos(dto.getOwnership()))
-            .dataSource(dataSourceMapper.mapDto(new DataSourceDto(dto.getDataSource(), dto.getNamespace(), null)))
+            .dataSource(dataSourceSafeMapper.mapDto(new DataSourceDto(dto.getDataSource(), dto.getNamespace(), null)))
             .dataEntityGroups(groups);
 
         if (entityClasses.contains(DataEntityClassDto.DATA_SET)) {
@@ -250,7 +251,7 @@ public DataEntityDetails mapDtoDetails(final DataEntityDetailsDto dto) {
             .type(type)
             .status(dataEntityStatusMapper.mapStatus(pojo))
             .ownership(ownershipMapper.mapDtos(dto.getOwnership()))
-            .dataSource(dataSourceMapper.mapDto(new DataSourceDto(dto.getDataSource(), dto.getNamespace(), null)))
+            .dataSource(dataSourceSafeMapper.mapDto(new DataSourceDto(dto.getDataSource(), dto.getNamespace(), null)))
             .tags(tagMapper.mapToTagList(dto.getTags()))
             .metadataFieldValues(metadataFieldValueMapper.mapDtos(dto.getMetadata()))
             .terms(linkedTerms)

diff --git a/...form-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataSourceSafeMapper.java b/...form-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataSourceSafeMapper.java
@@ -0,0 +1,15 @@
+package org.opendatadiscovery.oddplatform.mapper;
+
+import org.mapstruct.Mapper;
+import org.mapstruct.Mapping;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataSourceSafe;
+import org.opendatadiscovery.oddplatform.dto.DataSourceDto;
+
+@Mapper(
+    config = MapperConfig.class,
+    uses = {NamespaceMapper.class}
+)
+public interface DataSourceSafeMapper {
+    @Mapping(source = "dataSource", target = ".")
+    DataSourceSafe mapDto(DataSourceDto dataSourceDto);
+}
diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/BaseIngestionTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/BaseIngestionTest.java
@@ -20,6 +20,7 @@
 import org.opendatadiscovery.oddplatform.api.contract.model.DataSetStructure;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataSource;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataSourceFormData;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataSourceSafe;
 import org.opendatadiscovery.oddplatform.api.contract.model.MetricSet;
 import org.opendatadiscovery.oddplatform.api.contract.model.SearchFacetsData;
 import org.opendatadiscovery.oddplatform.api.contract.model.SearchFilterState;
@@ -53,6 +54,12 @@ protected DataSource createDataSource(final DataSourceFormData dataSource) {
         return ingestDataSource(dataSource);
     }
 
+    protected DataSourceSafe createDataSourceSafeFromDataSource(final DataSource dataSource) {
+        return new DataSourceSafe(dataSource.getId(), dataSource.getOddrn(), dataSource.getName())
+            .description(dataSource.getDescription())
+            .namespace(dataSource.getNamespace());
+    }
+
     private DataSource ingestDataSource(final DataSourceFormData dataSource) {
         return webTestClient.post()
             .uri("/api/datasources")

diff --git a/...api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/AlertIngestionTest.java b/...api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/AlertIngestionTest.java
@@ -76,7 +76,8 @@ public void backwardsIncompatibleAlertIngestionTest() {
         final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource);
 
         final var expectedDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())
@@ -98,7 +99,8 @@ public void backwardsIncompatibleAlertIngestionTest() {
         ingestAndAssert(dataEntityList);
 
         final var updatedExpectedDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())
@@ -152,7 +154,8 @@ public void failedJobAlertIngestionTest() {
         final long jobId = extractIngestedEntityIdAndAssert(createdDataSource);
 
         final DataEntityDetails expectedDataEntityDetails =
-            IngestionModelMapper.buildExpectedBaseDEDetails(jobId, job, createdDataSource);
+            IngestionModelMapper.buildExpectedBaseDEDetails(jobId, job,
+                createDataSourceSafeFromDataSource(createdDataSource));
 
         assertDataEntityDetailsEqual(expectedDataEntityDetails, (expected, actual) -> {
             assertThat(actual.getSourceList()).isEmpty();
@@ -218,7 +221,8 @@ public void failedDQTestAlertIngestionTest() {
             .forEach((expectedOddrn, dataEntity) -> {
                 final long entityId = ingestionMap.get(expectedOddrn);
                 final DataEntityDetails expectedDetails =
-                    IngestionModelMapper.buildExpectedBaseDEDetails(entityId, dataEntity, createdDataSource);
+                    IngestionModelMapper.buildExpectedBaseDEDetails(entityId, dataEntity,
+                        createDataSourceSafeFromDataSource(createdDataSource));
 
                 if (dataEntity.getDataset() != null) {
                     expectedDetails.setStats(new DataSetStats()

diff --git a/.../test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetFieldIngestionTest.java b/.../test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetFieldIngestionTest.java
@@ -69,7 +69,8 @@ public void enumValuesComplexTest() {
         final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource);
 
         final var expectedDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())
@@ -155,7 +156,8 @@ public void rewritingInternalEnumStateTest() {
         final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource);
 
         final var expectedDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())
@@ -222,7 +224,8 @@ public void externalStatisticsTagStateIngestionTest() {
         final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource);
 
         final var expectedDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())

diff --git a/...i/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetIngestionTest.java b/...i/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetIngestionTest.java
@@ -55,7 +55,8 @@ public void simpleDatasetIngestionTest() {
         final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource);
 
         final var expectedDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())
@@ -107,7 +108,8 @@ public void changeDatasetStructureTest() {
         final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource);
 
         final var expectedDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())
@@ -142,7 +144,8 @@ public void changeDatasetStructureTest() {
         ingestAndAssert(dataEntityList);
 
         final var updatedExpectedDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())
@@ -213,7 +216,8 @@ public void updateHollowDatasetSchemaTest() {
             assertThat(dataEntityId).isNotNull();
 
             final DataEntityDetails expectedDetails = IngestionModelMapper
-                .buildExpectedBaseDEDetails(dataEntityId, dataEntity, createdDataSource);
+                .buildExpectedBaseDEDetails(dataEntityId, dataEntity,
+                    createDataSourceSafeFromDataSource(createdDataSource));
 
             if (dataEntity.getDataset() != null) {
                 expectedDetails.setStats(new DataSetStats()

diff --git a/...i/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/LineageIngestionTest.java b/...i/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/LineageIngestionTest.java
@@ -100,7 +100,8 @@ public void simpleLineageIngestionTest() {
             assertThat(dataEntityId).isNotNull();
 
             final DataEntityDetails expectedDetails = IngestionModelMapper
-                .buildExpectedBaseDEDetails(dataEntityId, dataEntity, createdDataSource);
+                .buildExpectedBaseDEDetails(dataEntityId, dataEntity,
+                    createDataSourceSafeFromDataSource(createdDataSource));
 
             if (dataEntity.getDataset() != null) {
                 expectedDetails.setStats(new DataSetStats()

diff --git a/.../src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/MetadataIngestionTest.java b/.../src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/MetadataIngestionTest.java
@@ -49,7 +49,8 @@ public void simpleMetadataIngestionTest() {
         final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource);
 
         final var expectedDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())
@@ -78,7 +79,8 @@ public void simpleMetadataIngestionTest() {
         ingestAndAssert(dataEntityList);
 
         final var expectedUpdatedDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())
@@ -95,7 +97,8 @@ public void simpleMetadataIngestionTest() {
         ingestAndAssert(dataEntityList);
 
         final var expectedEmptyMetadataDataEntityDetails = IngestionModelMapper
-            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource)
+            .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest,
+                createDataSourceSafeFromDataSource(createdDataSource))
             .stats(new DataSetStats()
                 .consumersCount(0L)
                 .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size())

diff --git a/...test/java/org/opendatadiscovery/oddplatform/api/ingestion/utils/IngestionModelMapper.java b/...test/java/org/opendatadiscovery/oddplatform/api/ingestion/utils/IngestionModelMapper.java
@@ -14,7 +14,7 @@
 import org.opendatadiscovery.oddplatform.api.contract.model.DataSetField;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataSetFieldStat;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataSetFieldType;
-import org.opendatadiscovery.oddplatform.api.contract.model.DataSource;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataSourceSafe;
 import org.opendatadiscovery.oddplatform.api.contract.model.DateTimeFieldStat;
 import org.opendatadiscovery.oddplatform.api.contract.model.EnumValue;
 import org.opendatadiscovery.oddplatform.api.contract.model.IntegerFieldStat;
@@ -33,7 +33,7 @@
 public class IngestionModelMapper {
     public static DataEntityDetails buildExpectedBaseDEDetails(final long entityId,
                                                                final DataEntity dataEntity,
-                                                               final DataSource dataSource) {
+                                                               final DataSourceSafe dataSource) {
         return new DataEntityDetails()
             .id(entityId)
             .sourceCreatedAt(dataEntity.getCreatedAt())

diff --git a/...latform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/LineageMapperTest.java b/...latform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/LineageMapperTest.java
@@ -56,6 +56,9 @@ void setUp() {
                         new DateTimeMapperImpl()
                     )
                 ),
+                new DataSourceSafeMapperImpl(
+                    new NamespaceMapperImpl()
+                ),
                 new OwnershipMapperImpl(
                     new OwnerMapperImpl(),
                     new TitleMapperImpl()

diff --git a/...tform-api/src/test/java/org/opendatadiscovery/oddplatform/service/LineageServiceTest.java b/...tform-api/src/test/java/org/opendatadiscovery/oddplatform/service/LineageServiceTest.java
@@ -20,6 +20,7 @@
 import org.opendatadiscovery.oddplatform.mapper.DataEntityRunMapperImpl;
 import org.opendatadiscovery.oddplatform.mapper.DataEntityStatusMapper;
 import org.opendatadiscovery.oddplatform.mapper.DataSourceMapperImpl;
+import org.opendatadiscovery.oddplatform.mapper.DataSourceSafeMapperImpl;
 import org.opendatadiscovery.oddplatform.mapper.DatasetFieldApiMapperImpl;
 import org.opendatadiscovery.oddplatform.mapper.DatasetVersionMapperImpl;
 import org.opendatadiscovery.oddplatform.mapper.DateTimeMapperImpl;
@@ -81,6 +82,9 @@ void setUp() {
                         new DateTimeMapperImpl()
                     )
                 ),
+                new DataSourceSafeMapperImpl(
+                    new NamespaceMapperImpl()
+                ),
                 new OwnershipMapperImpl(
                     new OwnerMapperImpl(),
                     new TitleMapperImpl()

diff --git a/odd-platform-specification/components.yaml b/odd-platform-specification/components.yaml
@@ -530,7 +530,7 @@ components:
         ownership:
           $ref: '#/components/schemas/OwnershipList'
         data_source:
-          $ref: '#/components/schemas/DataSource'
+          $ref: '#/components/schemas/DataSourceSafe'
         entity_classes:
           type: array
           items:
@@ -1210,6 +1210,25 @@ components:
         - oddrn
         - name
 
+    DataSourceSafe:
+      type: object
+      properties:
+        id:
+          type: integer
+          format: int64
+        namespace:
+          $ref: '#/components/schemas/Namespace'
+        oddrn:
+          type: string
+        name:
+          type: string
+        description:
+          type: string
+      required:
+        - id
+        - oddrn
+        - name
+
     DataSourceList:
       type: object
       properties:
@@ -1462,8 +1481,6 @@ components:
           $ref: '#/components/schemas/BinaryFieldStat'
         datetime_stats:
           $ref: '#/components/schemas/DateTimeFieldStat'
-      required:
-        - name
 
     BooleanFieldStat:
       type: object