Skip to content

Commit

Permalink
1623 - added additional filterState last_runs_statuses, created API f…
Browse files Browse the repository at this point in the history 
…or generating SearchFacetsData
  • Loading branch information
@Vladysl
Vladysl committed May 6, 2024
1 parent 530463e commit 0aaaa45
Show file tree
Hide file tree
Showing 14 changed files with 393 additions and 68 deletions.
13 changes: 13 additions & 0 deletions ...src/main/java/org/opendatadiscovery/oddplatform/controller/DataQualityRunsController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.opendatadiscovery.oddplatform.api.contract.api.DataQualityRunsApi;
import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRunStatus;
import org.opendatadiscovery.oddplatform.api.contract.model.DataQualityResults;
import org.opendatadiscovery.oddplatform.api.contract.model.SearchFacetsData;
import org.opendatadiscovery.oddplatform.service.DataQualityRunsService;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.RestController;
Expand Down Expand Up @@ -31,4 +33,15 @@ public Mono<ResponseEntity<DataQualityResults>> getDataQualityTestsRuns(final Li
deNamespaceIds, deDatasourceIds, deOwnerIds, deTitleIds, deTagIds)
.map(ResponseEntity::ok);
}

@Override
public Mono<ResponseEntity<SearchFacetsData>> createDataQualityLatestRunsSearch(final List<Long> namespaceIds,
final List<Long> datasourceIds,
final List<Long> ownerIds,
final List<Long> tagIds,
final DataEntityRunStatus status,
final ServerWebExchange exchange) {
return service.createDataQualityLatestRunsSearch(namespaceIds, datasourceIds, ownerIds, tagIds, status)
.map(ResponseEntity::ok);
}
}
36 changes: 36 additions & 0 deletions ...m-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityTaskRunStatusDto.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package org.opendatadiscovery.oddplatform.dto;

import java.util.Arrays;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.Getter;

import static java.util.function.Function.identity;

@Getter
public enum DataEntityTaskRunStatusDto {
SUCCESS(1, "SUCCESS"),
FAILED(2, "FAILED"),
SKIPPED(3, "SKIPPED"),
BROKEN(4, "BROKEN"),
ABORTED(5, "ABORTED"),
UNKNOWN(6, "UNKNOWN");

private final short id;
private final String status;

DataEntityTaskRunStatusDto(final int id,
final String status) {
this.id = (short) id;
this.status = status;
}

private static final Map<String, DataEntityTaskRunStatusDto> MAP = Arrays
.stream(DataEntityTaskRunStatusDto.values())
.collect(Collectors.toMap(DataEntityTaskRunStatusDto::getStatus, identity()));

public static Optional<DataEntityTaskRunStatusDto> findByStatus(final String status) {
return Optional.ofNullable(MAP.get(status));
}
}
1 change: 1 addition & 0 deletions odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ public enum FacetType {
TAGS,
GROUPS,
STATUSES,
LAST_RUN_STATUSES,
DATA_ENTITY;

public static FacetType lookup(final String facetType) {
Expand Down
6 changes: 6 additions & 0 deletions ...platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,10 @@ public interface FacetStateMapper {
TermFacetState mapDto(final FacetStateDto state);

FacetState mapDto(final List<CountableSearchFilter> entityClasses, final FacetStateDto state);

SearchFormData mapToFormData(final List<Long> namespaceIds,
final List<Long> datasourceIds,
final List<Long> ownerIds,
final List<Long> tagIds,
final List<Integer> entityClasses);
}
53 changes: 51 additions & 2 deletions ...form-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapperImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@
import com.fasterxml.jackson.core.type.TypeReference;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.jooq.JSONB;
import org.opendatadiscovery.oddplatform.api.contract.model.CountableSearchFilter;
Expand All @@ -23,6 +25,7 @@
import org.opendatadiscovery.oddplatform.api.contract.model.TermFacetState;
import org.opendatadiscovery.oddplatform.api.contract.model.TermSearchFormData;
import org.opendatadiscovery.oddplatform.api.contract.model.TermSearchFormDataFilters;
import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto;
import org.opendatadiscovery.oddplatform.dto.FacetStateDto;
import org.opendatadiscovery.oddplatform.dto.FacetType;
import org.opendatadiscovery.oddplatform.dto.SearchFilterDto;
Expand All @@ -45,7 +48,8 @@ public class FacetStateMapperImpl implements FacetStateMapper {
SearchFormDataFilters::getOwners, FacetType.OWNERS,
SearchFormDataFilters::getTags, FacetType.TAGS,
SearchFormDataFilters::getGroups, FacetType.GROUPS,
SearchFormDataFilters::getStatuses, FacetType.STATUSES
SearchFormDataFilters::getStatuses, FacetType.STATUSES,
SearchFormDataFilters::getLastRunStatuses, FacetType.LAST_RUN_STATUSES
);

private static final Map<Function<TermSearchFormDataFilters, List<SearchFilterState>>, FacetType>
Expand Down Expand Up @@ -170,7 +174,52 @@ public FacetState mapDto(final List<CountableSearchFilter> entityClasses, final
.owners(getSearchFiltersForFacetType(state, FacetType.OWNERS))
.namespaces(getSearchFiltersForFacetType(state, FacetType.NAMESPACES))
.tags(getSearchFiltersForFacetType(state, FacetType.TAGS))
.groups(getSearchFiltersForFacetType(state, FacetType.GROUPS));
.groups(getSearchFiltersForFacetType(state, FacetType.GROUPS))
.statuses(getSearchFiltersForFacetType(state, FacetType.STATUSES))
.lastRunStatuses(getSearchFiltersForFacetType(state, FacetType.LAST_RUN_STATUSES));
}

@Override
public SearchFormData mapToFormData(final List<Long> namespaceIds,
final List<Long> datasourceIds,
final List<Long> ownerIds,
final List<Long> tagIds,
final List<Integer> entityClasses) {
final SearchFormDataFilters filters = new SearchFormDataFilters()
.namespaces(getFilterStateList(namespaceIds, FacetType.NAMESPACES))
.datasources(getFilterStateList(datasourceIds, FacetType.DATA_SOURCES))
.owners(getFilterStateList(ownerIds, FacetType.OWNERS))
.tags(getFilterStateList(tagIds, FacetType.TAGS))
.entityClasses(getFilterStateListForEntityClasses(entityClasses));

return new SearchFormData().filters(filters).query("");
}

private List<SearchFilterState> getFilterStateList(final List<Long> filterIds, final FacetType filterType) {
if (CollectionUtils.isEmpty(filterIds)) {
return Collections.emptyList();
}

return filterIds.stream()
.map(id -> new SearchFilterState()
.entityId(id)
.entityName(filterType.name())
.selected(true))
.collect(Collectors.toList());
}

private List<SearchFilterState> getFilterStateListForEntityClasses(final List<Integer> filterIds) {
if (CollectionUtils.isEmpty(filterIds)) {
return Collections.emptyList();
}

return filterIds.stream()
.map(id -> new SearchFilterState()
.entityId(Long.valueOf(id))
.entityName(DataEntityClassDto.findById(id).orElseThrow(() -> new IllegalArgumentException(
"Unknown data entity type id: %d".formatted(id))).name())
.selected(true))
.collect(Collectors.toList());
}

private SearchFilterDto mapFilter(final SearchFilterState f, final FacetType type) {
Expand Down
59 changes: 36 additions & 23 deletions ...ndatadiscovery/oddplatform/repository/reactive/ReactiveDataQualityRunsRepositoryImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public class ReactiveDataQualityRunsRepositoryImpl implements ReactiveDataQualit
public static final String DATA_QUALITY_TEST_TYPE =
"specific_attributes->'DATA_QUALITY_TEST'->'expectation'->>'category'";
public static final String ID = "id";
public static final String DATA_ENTITY_ID = "data_entity_id";
public static final String ODDRN = "oddrn";
public static final String CATEGORY = "category";
public static final String CATEGORIES_CTE = "categories_cte";
Expand All @@ -63,30 +64,9 @@ public class ReactiveDataQualityRunsRepositoryImpl implements ReactiveDataQualit

@Override
public Flux<DataQualityRunsRecord> getLatestDataQualityRunsResults(final DataQualityTestFiltersDto filtersDto) {
final Table<Record2<Long, String>> testFilters = generateTestFiltersCte(filtersDto);
final Select<Record3<Long, String, String>> deCategoryCTE =
DSL.select(DATA_ENTITY.ID.as(ID), DATA_ENTITY.ODDRN.as(ODDRN),
field(DATA_QUALITY_TEST_TYPE, String.class).as(CATEGORY))
.from(DATA_ENTITY, testFilters)
.where(DATA_ENTITY.TYPE_ID.eq(DataEntityTypeDto.JOB.getId())
.and(field(DATA_QUALITY_TEST_TYPE, String.class).isNotNull()))
.and(testFilters.field(DATA_ENTITY.ID).eq(DATA_ENTITY.ID));

final Table<Record3<Long, String, String>> categoriesSubTable = deCategoryCTE.asTable(CATEGORIES_CTE);
final Table<Record3<Long, String, String>> categoriesSubTable = getCTEForLastRuns(filtersDto);
final List<TableLike<?>> fromList = new ArrayList<>(List.of(categoriesSubTable, DATA_ENTITY_TASK_LAST_RUN));
final List<Condition> conditionList
= new ArrayList<>(List.of(categoriesSubTable.field(ODDRN, String.class)
.eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN)));

if (shouldAddFiltersForDataEntity(filtersDto)) {
final Table<Record2<Long, String>> deFilters = generateDataEntityFiltersCte(filtersDto);

fromList.addAll(List.of(deFilters, DATA_QUALITY_TEST_RELATIONS));

conditionList.add(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN
.eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN));
conditionList.add(deFilters.field(DATA_ENTITY.ODDRN).eq(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN));
}
final List<Condition> conditionList = getConditionsForLastRuns(filtersDto, fromList, categoriesSubTable);

final SelectSeekStep1<Record3<String, String, Integer>, String> query
= DSL.select(
Expand Down Expand Up @@ -195,6 +175,39 @@ public Flux<MonitoredtablesRecord> getMonitoredTables(final DataQualityTestFilte
.map(item -> item.into(MonitoredtablesRecord.class));
}

private Table<Record3<Long, String, String>> getCTEForLastRuns(final DataQualityTestFiltersDto filtersDto) {
final Table<Record2<Long, String>> testFilters = generateTestFiltersCte(filtersDto);
final Select<Record3<Long, String, String>> deCategoryCTE =
DSL.select(DATA_ENTITY.ID.as(ID), DATA_ENTITY.ODDRN.as(ODDRN),
field(DATA_QUALITY_TEST_TYPE, String.class).as(CATEGORY))
.from(DATA_ENTITY, testFilters)
.where(DATA_ENTITY.TYPE_ID.eq(DataEntityTypeDto.JOB.getId())
.and(field(DATA_QUALITY_TEST_TYPE, String.class).isNotNull()))
.and(testFilters.field(DATA_ENTITY.ID).eq(DATA_ENTITY.ID));

return deCategoryCTE.asTable(CATEGORIES_CTE);
}

private List<Condition> getConditionsForLastRuns(final DataQualityTestFiltersDto filtersDto,
final List<TableLike<?>> fromList,
final Table<Record3<Long, String, String>> categoriesSubTable) {
final List<Condition> conditionList
= new ArrayList<>(List.of(categoriesSubTable.field(ODDRN, String.class)
.eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN)));

if (shouldAddFiltersForDataEntity(filtersDto)) {
final Table<Record2<Long, String>> deFilters = generateDataEntityFiltersCte(filtersDto);

fromList.addAll(List.of(deFilters, DATA_QUALITY_TEST_RELATIONS));

conditionList.add(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN
.eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN));
conditionList.add(deFilters.field(DATA_ENTITY.ODDRN).eq(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN));
}

return conditionList;
}

private SelectConditionStep<Record> getDataQualityQuery(final Table<Record2<Long, String>> dataEntityCTE,
final DataQualityTestFiltersDto filtersDto) {
final Table<Record2<Long, String>> testFilters = generateTestFiltersCte(filtersDto);
Expand Down
5 changes: 5 additions & 0 deletions .../org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,4 +51,9 @@ Mono<Map<SearchFilterId, Long>> getStatusFacetForDataEntity(final String query,
final int page,
final int size,
final FacetStateDto state);

Mono<Map<SearchFilterId, Long>> getLastRunStatusesFacetForDataEntity(final String query,
final int page,
final int size,
final FacetStateDto state);
}
Loading

0 comments on commit 0aaaa45

Please sign in to comment.