Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

UI : New Data profiler Layout. #3184

Merged
merged 9 commits into from
Mar 6, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { Table } from 'Models';
import { ColumnTestType } from '../enums/columnTest.enum';
import { CreateTableTest } from '../generated/api/tests/createTableTest';
import { TableTestType } from '../generated/tests/tableTest';
import { CreateColumnTest } from '../interface/dataQuality.interface';
import { ColumnTest } from '../interface/dataQuality.interface';
import { getURLWithQueryFields } from '../utils/APIUtils';
import APIClient from './index';

Expand Down Expand Up @@ -138,7 +138,7 @@ export const deleteTableTestCase = (
);
};

export const addColumnTestCase = (tableId: string, data: CreateColumnTest) => {
export const addColumnTestCase = (tableId: string, data: ColumnTest) => {
const configOptions = {
headers: { 'Content-type': 'application/json' },
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { CreateTableTest } from '../../generated/api/tests/createTableTest';
import { Table } from '../../generated/entity/data/table';
import { TableTest } from '../../generated/tests/tableTest';
import {
CreateColumnTest,
ColumnTest,
TableTestDataType,
} from '../../interface/dataQuality.interface';
import ColumnTestForm from './Forms/ColumnTestForm';
Expand All @@ -28,7 +28,7 @@ type Props = {
columnOptions: Table['columns'];
tableTestCase: TableTest[];
handleAddTableTestCase: (data: CreateTableTest) => void;
handleAddColumnTestCase: (data: CreateColumnTest) => void;
handleAddColumnTestCase: (data: ColumnTest) => void;
onFormCancel: () => void;
};

Expand All @@ -53,7 +53,7 @@ const AddDataQualityTest = ({
) : (
<ColumnTestForm
column={columnOptions}
data={data as CreateColumnTest}
data={data as ColumnTest}
handleAddColumnTestCase={handleAddColumnTestCase}
onFormCancel={onFormCancel}
/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { ColumnTestType } from '../../../enums/columnTest.enum';
import { TestCaseExecutionFrequency } from '../../../generated/api/tests/createTableTest';
import { Table } from '../../../generated/entity/data/table';
import {
CreateColumnTest,
ColumnTest,
ModifiedTableColumn,
} from '../../../interface/dataQuality.interface';
import {
Expand All @@ -33,9 +33,9 @@ import { Button } from '../../buttons/Button/Button';
import MarkdownWithPreview from '../../common/editor/MarkdownWithPreview';

type Props = {
data: CreateColumnTest;
data: ColumnTest;
column: ModifiedTableColumn[];
handleAddColumnTestCase: (data: CreateColumnTest) => void;
handleAddColumnTestCase: (data: ColumnTest) => void;
onFormCancel: () => void;
};

Expand Down Expand Up @@ -118,7 +118,7 @@ const ColumnTestForm = ({
const selectedColumn = column.find((d) => d.name === name);
const existingTests =
selectedColumn?.columnTests?.map(
(d: CreateColumnTest) => d.testCase.columnTestType
(d: ColumnTest) => d.testCase.columnTestType
) || [];
if (existingTests.length) {
const newTest = Object.values(ColumnTestType).filter(
Expand Down Expand Up @@ -222,7 +222,7 @@ const ColumnTestForm = ({

const handleSave = () => {
if (validateForm()) {
const columnTestObj: CreateColumnTest = {
const columnTestObj: ColumnTest = {
columnName: columnName,
description: markdownRef.current?.getEditorContent() || undefined,
executionFrequency: frequency,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { CreateTableTest } from '../../generated/api/tests/createTableTest';
import { Table } from '../../generated/entity/data/table';
import { TableTest, TableTestType } from '../../generated/tests/tableTest';
import {
CreateColumnTest,
ColumnTest,
DatasetTestModeType,
ModifiedTableColumn,
TableTestDataType,
Expand All @@ -27,7 +27,7 @@ import DataQualityTest from '../DataQualityTest/DataQualityTest';

type Props = {
handleAddTableTestCase: (data: CreateTableTest) => void;
handleAddColumnTestCase: (data: CreateColumnTest) => void;
handleAddColumnTestCase: (data: ColumnTest) => void;
columnOptions: Table['columns'];
testMode: DatasetTestModeType;
handleTestModeChange: (mode: DatasetTestModeType) => void;
Expand Down Expand Up @@ -93,7 +93,7 @@ const DataQualityTab = ({
setActiveData(undefined);
};

const onColumnTestSave = (data: CreateColumnTest) => {
const onColumnTestSave = (data: ColumnTest) => {
handleAddColumnTestCase(data);
setActiveData(undefined);
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,11 @@ import {
import { getEntityFeedLink } from '../../utils/EntityUtils';
import { getDefaultValue } from '../../utils/FeedElementUtils';
import { getEntityFieldThreadCounts } from '../../utils/FeedUtils';
import { getTagsWithoutTier, getUsagePercentile } from '../../utils/TableUtils';
import {
getTableTestsValue,
getTagsWithoutTier,
getUsagePercentile,
} from '../../utils/TableUtils';
import ActivityFeedList from '../ActivityFeed/ActivityFeedList/ActivityFeedList';
import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThreadPanel';
import Description from '../common/description/Description';
Expand Down Expand Up @@ -111,6 +115,7 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
handleShowTestForm,
handleRemoveTableTest,
handleRemoveColumnTest,
qualityTestFormHandler,
}: DatasetDetailsProps) => {
const { isAuthDisabled } = useAuth();
const [isEdit, setIsEdit] = useState(false);
Expand Down Expand Up @@ -350,6 +355,7 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
''
),
},
{ key: 'Tests', value: getTableTestsValue(tableTestCase) },
];

const onDescriptionEdit = (): void => {
Expand Down Expand Up @@ -616,7 +622,10 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
columns={columns.map((col) => ({
constraint: col.constraint as string,
colName: col.name,
colType: col.dataTypeDisplay as string,
colTests: col.columnTests,
}))}
qualityTestFormHandler={qualityTestFormHandler}
tableProfiles={tableProfile}
/>
</div>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,9 @@ import { TableTest, TableTestType } from '../../generated/tests/tableTest';
import { EntityLineage } from '../../generated/type/entityLineage';
import { TagLabel } from '../../generated/type/tagLabel';
import {
CreateColumnTest,
ColumnTest,
DatasetTestModeType,
ModifiedTableColumn,
} from '../../interface/dataQuality.interface';
import { TitleBreadcrumbProps } from '../common/title-breadcrumb/title-breadcrumb.interface';
import { Edge, EdgeData } from '../EntityLineage/EntityLineage.interface';
Expand All @@ -60,7 +61,7 @@ export interface DatasetDetailsProps {
description: string;
tableProfile: Table['tableProfile'];
tableQueries: Table['tableQueries'];
columns: Table['columns'];
columns: ModifiedTableColumn[];
tier: TagLabel;
sampleData: TableData;
entityLineage: EntityLineage;
Expand All @@ -78,6 +79,10 @@ export interface DatasetDetailsProps {
testMode: DatasetTestModeType;
tableTestCase: TableTest[];
showTestForm: boolean;
qualityTestFormHandler: (
tabValue: number,
testMode: DatasetTestModeType
) => void;
handleShowTestForm: (value: boolean) => void;
handleTestModeChange: (mode: DatasetTestModeType) => void;
createThread: (data: CreateThread) => void;
Expand All @@ -94,7 +99,7 @@ export interface DatasetDetailsProps {
entityLineageHandler: (lineage: EntityLineage) => void;
postFeedHandler: (value: string, id: string) => void;
handleAddTableTestCase: (data: CreateTableTest) => void;
handleAddColumnTestCase: (data: CreateColumnTest) => void;
handleAddColumnTestCase: (data: ColumnTest) => void;
handleRemoveTableTest: (testType: TableTestType) => void;
handleRemoveColumnTest: (
columnName: string,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ const DatasetDetailsProps = {
handleRemoveTableTest: jest.fn(),
handleRemoveColumnTest: jest.fn(),
handleTestModeChange: jest.fn(),
qualityTestFormHandler: jest.fn(),
};
jest.mock('../ManageTab/ManageTab.component', () => {
return jest.fn().mockReturnValue(<p>ManageTab</p>);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@ import {
Table,
} from '../../generated/entity/data/table';
import { Operation } from '../../generated/entity/policies/accessControl/rule';
import { TestCaseStatus } from '../../generated/tests/tableTest';
import { LabelType, State, TagLabel } from '../../generated/type/tagLabel';
import {
ColumnTest,
ModifiedTableColumn,
} from '../../interface/dataQuality.interface';
import {
getHtmlForNonAdminAction,
getPartialNameFromFQN,
Expand All @@ -50,15 +55,15 @@ import Tags from '../tags/tags';

type Props = {
owner: Table['owner'];
tableColumns: Table['columns'];
tableColumns: ModifiedTableColumn[];
joins: Array<ColumnJoins>;
searchText?: string;
columnName: string;
hasEditAccess: boolean;
isReadOnly?: boolean;
entityFqn?: string;
entityFieldThreads?: EntityFieldThreads[];
onUpdate?: (columns: Table['columns']) => void;
onUpdate?: (columns: ModifiedTableColumn[]) => void;
onThreadLinkSelect?: (value: string) => void;
onEntityFieldSelect?: (value: string) => void;
};
Expand Down Expand Up @@ -86,6 +91,10 @@ const EntityTable = ({
Header: 'Type',
accessor: 'dataTypeDisplay',
},
{
Header: 'Data Quality',
accessor: 'columnTests',
},
{
Header: 'Description',
accessor: 'description',
Expand All @@ -98,7 +107,9 @@ const EntityTable = ({
[]
);

const [searchedColumns, setSearchedColumns] = useState<Table['columns']>([]);
const [searchedColumns, setSearchedColumns] = useState<ModifiedTableColumn[]>(
[]
);

const data = React.useMemo(
() => makeData(searchedColumns),
Expand Down Expand Up @@ -158,7 +169,7 @@ const EntityTable = ({
};

const updateColumnDescription = (
tableCols: Table['columns'],
tableCols: ModifiedTableColumn[],
changedColName: string,
description: string
) => {
Expand All @@ -167,7 +178,7 @@ const EntityTable = ({
col.description = description;
} else {
updateColumnDescription(
col?.children as Table['columns'],
col?.children as ModifiedTableColumn[],
changedColName,
description
);
Expand All @@ -176,7 +187,7 @@ const EntityTable = ({
};

const updateColumnTags = (
tableCols: Table['columns'],
tableCols: ModifiedTableColumn[],
changedColName: string,
newColumnTags: Array<string>
) => {
Expand Down Expand Up @@ -204,7 +215,7 @@ const EntityTable = ({
col.tags = getUpdatedTags(col);
} else {
updateColumnTags(
col?.children as Table['columns'],
col?.children as ModifiedTableColumn[],
changedColName,
newColumnTags
);
Expand Down Expand Up @@ -309,7 +320,8 @@ const EntityTable = ({
{headerGroup.headers.map((column: any, index: number) => (
<th
className={classNames('tableHead-cell', {
'tw-w-60': column.id === 'tags',
'tw-w-60':
column.id === 'tags' || column.id === 'columnTests',
})}
key={index}
{...column.getHeaderProps()}>
Expand All @@ -332,6 +344,22 @@ const EntityTable = ({
{...row.getRowProps()}>
{/* eslint-disable-next-line */}
{row.cells.map((cell: any, index: number) => {
const columnTests =
cell.column.id === 'columnTests'
? ((cell.value ?? []) as ColumnTest[])
: ([] as ColumnTest[]);
const columnTestLength = columnTests.length;
const failingTests = columnTests.filter((test) =>
test.results?.some(
(t) => t.testCaseStatus === TestCaseStatus.Failed
)
);
const passingTests = columnTests.filter((test) =>
test.results?.some(
(t) => t.testCaseStatus === TestCaseStatus.Success
)
);

return (
<td
className={classNames(
Expand All @@ -354,6 +382,34 @@ const EntityTable = ({
</span>
) : null}

{cell.column.id === 'columnTests' && (
<Fragment>
{columnTestLength ? (
<Fragment>
{failingTests.length ? (
<div className="tw-flex">
<p className="tw-mr-2">
<i className="fas fa-times tw-text-status-failed" />
</p>
<p>
{`${failingTests.length}/${columnTestLength} tests failing`}
</p>
</div>
) : (
<div className="tw-flex">
<div className="tw-mr-2">
<i className="fas fa-check-square tw-text-status-success" />
</div>
<p>{`${passingTests.length} tests`}</p>
</div>
)}
</Fragment>
) : (
'--'
)}
</Fragment>
)}

{cell.column.id === 'dataTypeDisplay' && (
<>
{isReadOnly ? (
Expand Down