Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
UI: Selection on graph with edges. Status for jobs/datasets (#2384)
* Selection on graph with edges. Status for jobs/datasets

Signed-off-by: tito12 <vladyslav.sedenko@gmail.com>
---------

Signed-off-by: tito12 <vladyslav.sedenko@gmail.com>
Co-authored-by: Willy Lulciuc <willy@datakin.com>
  • Loading branch information
tito12 and wslulciuc committed Feb 9, 2023
1 parent 9e0c84b commit f9465bf
Show file tree
Hide file tree
Showing 20 changed files with 329 additions and 163 deletions.
18 changes: 17 additions & 1 deletion docker/metadata.json
Expand Up @@ -1358,6 +1358,22 @@
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json",
"name": "food_delivery_db",
"uri": "postgres://food_delivery:food_delivery@postgres:5432/food_delivery"
},
"dataQualityAssertions": {
"_producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityAssertionsDatasetFacet.json",
"assertions": [
{
"assertion": "not_null",
"success": false,
"column": "driver_id"
},
{
"assertion": "is_string",
"success": true,
"column": "customer_address"
}
]
}
}
}
Expand Down Expand Up @@ -1824,4 +1840,4 @@
},
"producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json"
}
]
]
41 changes: 41 additions & 0 deletions web/src/components/core/status/MqStatus.tsx
@@ -0,0 +1,41 @@
// SPDX-License-Identifier: Apache-2.0

import { theme } from '../../../helpers/theme'
import Box from '@material-ui/core/Box'
import MqText from '../text/MqText'
import React from 'react'
import createStyles from '@material-ui/core/styles/createStyles'
import withStyles, { WithStyles } from '@material-ui/core/styles/withStyles'

const styles = () =>
createStyles({
type: {
display: 'flex',
alignItems: 'center',
gap: theme.spacing(1)
},
status: {
width: theme.spacing(2),
height: theme.spacing(2),
borderRadius: '50%'
}
})

interface OwnProps {
color: string | null
label?: string
}

const MqStatus: React.FC<OwnProps & WithStyles<typeof styles>> = ({ label, color, classes }) => {
if (!color) {
return null
}
return (
<Box className={classes.type}>
<Box className={classes.status} style={{ backgroundColor: color }} />
{label && <MqText>{label}</MqText>}
</Box>
)
}

export default withStyles(styles)(MqStatus)
5 changes: 1 addition & 4 deletions web/src/components/datasets/DatasetColumnLineage.tsx
Expand Up @@ -102,7 +102,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
dispatch
)

export default connect(
mapStateToProps,
mapDispatchToProps
)(DatasetColumnLineage)
export default connect(mapStateToProps, mapDispatchToProps)(DatasetColumnLineage)
22 changes: 15 additions & 7 deletions web/src/components/datasets/DatasetDetailPage.tsx
Expand Up @@ -15,6 +15,7 @@ import { LineageDataset } from '../lineage/types'
import { alpha } from '@material-ui/core/styles'
import { bindActionCreators } from 'redux'
import { connect } from 'react-redux'
import { datasetFacetsStatus } from '../../helpers/nodes'
import {
deleteDataset,
dialogToggle,
Expand All @@ -31,7 +32,9 @@ import DatasetInfo from './DatasetInfo'
import DatasetVersions from './DatasetVersions'
import Dialog from '../Dialog'
import IconButton from '@material-ui/core/IconButton'
import MqStatus from '../core/status/MqStatus'
import MqText from '../core/text/MqText'

import React, { ChangeEvent, FunctionComponent, SetStateAction, useEffect } from 'react'

const styles = ({ spacing }: ITheme) => {
Expand Down Expand Up @@ -143,6 +146,7 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {

const firstVersion = versions[0]
const { name, tags, description } = firstVersion
const facetsStatus = datasetFacetsStatus(firstVersion.facets)

return (
<Box my={2} className={root}>
Expand Down Expand Up @@ -202,9 +206,16 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
</IconButton>
</Box>
</Box>
<MqText heading font={'mono'}>
{name}
</MqText>
<Box display={'flex'} alignItems={'center'}>
{facetsStatus && (
<Box mr={1}>
<MqStatus color={facetsStatus} />
</Box>
)}
<MqText heading font={'mono'}>
{name}
</MqText>
</Box>
<Box mb={2}>
<MqText subdued>{description}</MqText>
</Box>
Expand Down Expand Up @@ -241,7 +252,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
dispatch
)

export default connect(
mapStateToProps,
mapDispatchToProps
)(withStyles(styles)(DatasetDetailPage))
export default connect(mapStateToProps, mapDispatchToProps)(withStyles(styles)(DatasetDetailPage))
5 changes: 2 additions & 3 deletions web/src/components/datasets/DatasetVersions.tsx
Expand Up @@ -31,9 +31,8 @@ interface DatasetVersionsProps {
versions: DatasetVersion[]
}

const DatasetVersions: FunctionComponent<
DatasetVersionsProps & IWithStyles<typeof styles>
> = props => {
const DatasetVersions: FunctionComponent<DatasetVersionsProps &
IWithStyles<typeof styles>> = props => {
const { versions, classes } = props

const [infoView, setInfoView] = React.useState<DatasetVersion | null>(null)
Expand Down
12 changes: 5 additions & 7 deletions web/src/components/jobs/JobDetailPage.tsx
Expand Up @@ -25,15 +25,16 @@ import {
resetJobs,
resetRuns
} from '../../store/actionCreators'
import { jobRunsStatus } from '../../helpers/nodes'
import { theme } from '../../helpers/theme'
import { useHistory } from 'react-router-dom'
import CloseIcon from '@material-ui/icons/Close'
import Dialog from '../Dialog'
import IconButton from '@material-ui/core/IconButton'
import MqEmpty from '../core/empty/MqEmpty'
import MqStatus from '../core/status/MqStatus'
import MqText from '../core/text/MqText'
import RunInfo from './RunInfo'
import RunStatus from './RunStatus'
import Runs from './Runs'

const styles = ({ spacing }: ITheme) => {
Expand Down Expand Up @@ -166,9 +167,9 @@ const JobDetailPage: FunctionComponent<IProps> = props => {
</Box>
</Box>
<Box display={'flex'} alignItems={'center'}>
{job.latestRun && (
{runs.length && (
<Box mr={1}>
<RunStatus run={job.latestRun} />
<MqStatus color={jobRunsStatus(runs)} />
</Box>
)}
<MqText font={'mono'} heading>
Expand Down Expand Up @@ -212,7 +213,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
dispatch
)

export default connect(
mapStateToProps,
mapDispatchToProps
)(withStyles(styles)(JobDetailPage))
export default connect(mapStateToProps, mapDispatchToProps)(withStyles(styles)(JobDetailPage))
9 changes: 6 additions & 3 deletions web/src/components/jobs/RunStatus.tsx
Expand Up @@ -3,8 +3,7 @@

import { Box, Theme, Tooltip, WithStyles, createStyles, withStyles } from '@material-ui/core'
import { Run } from '../../types/api'

import { runColorMap } from '../../helpers/runs'
import { runStateColor } from '../../helpers/nodes'

import React, { FunctionComponent } from 'react'

Expand All @@ -26,7 +25,11 @@ const RunStatus: FunctionComponent<RunStatusProps & WithStyles<typeof styles>> =
const { run, classes } = props
return (
<Tooltip title={run.state}>
<Box mr={1} className={classes.status} style={{ backgroundColor: runColorMap[run.state] }} />
<Box
mr={1}
className={classes.status}
style={{ backgroundColor: runStateColor(run.state) }}
/>
</Tooltip>
)
}
Expand Down
57 changes: 47 additions & 10 deletions web/src/components/lineage/Lineage.tsx
Expand Up @@ -85,6 +85,7 @@ class Lineage extends React.Component<LineageProps, LineageState> {
const nodeName = this.props.match.params.nodeName
const namespace = this.props.match.params.namespace
const nodeType = this.props.match.params.nodeType

if (nodeName && namespace && nodeType) {
const nodeId = generateNodeId(
this.props.match.params.nodeType.toUpperCase() as JobOrDataset,
Expand Down Expand Up @@ -114,6 +115,7 @@ class Lineage extends React.Component<LineageProps, LineageState> {
this.props.match.params.namespace,
this.props.match.params.nodeName
)
this.getEdges()
}
}

Expand All @@ -129,6 +131,48 @@ class Lineage extends React.Component<LineageProps, LineageState> {
})
}

getEdges = () => {
const selectedPaths = this.getSelectedPaths()

return g?.edges().map(e => {
const isSelected = selectedPaths.some((r: any) => e.v === r[0] && e.w === r[1])
return Object.assign(g.edge(e), { isSelected: isSelected })
})
}

getSelectedPaths = () => {
const paths = [] as Array<[string, string]>

const getSuccessors = (node: string) => {
const successors = g?.successors(node)
if (successors?.length) {
for (let i = 0; i < node.length - 1; i++) {
if (successors[i]) {
paths.push([node, (successors[i] as unknown) as string])
getSuccessors((successors[i] as unknown) as string)
}
}
}
}

const getPredecessors = (node: string) => {
const predecessors = g?.predecessors(node)
if (predecessors?.length) {
for (let i = 0; i < node.length - 1; i++) {
if (predecessors[i]) {
paths.push([(predecessors[i] as unknown) as string, node])
getPredecessors((predecessors[i] as unknown) as string)
}
}
}
}

getSuccessors(this.props.selectedNode)
getPredecessors(this.props.selectedNode)

return paths
}

buildGraphAll = (graph: LineageNode[]) => {
// nodes
for (let i = 0; i < graph.length; i++) {
Expand All @@ -150,14 +194,15 @@ class Lineage extends React.Component<LineageProps, LineageState> {

this.setState({
graph: g,
edges: g.edges().map(e => g.edge(e)),
edges: this.getEdges(),
nodes: g.nodes().map(v => g.node(v))
})
}

render() {
const { classes } = this.props
const i18next = require('i18next')

return (
<Box className={classes.lineageContainer}>
{this.props.selectedNode === null && (
Expand Down Expand Up @@ -227,9 +272,6 @@ class Lineage extends React.Component<LineageProps, LineageState> {
<Node
key={node.data.name}
node={node}
edgeEnds={this.state.edges.map(
edge => edge.points[edge.points.length - 1]
)}
selectedNode={this.props.selectedNode}
/>
))}
Expand Down Expand Up @@ -262,9 +304,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
dispatch
)

export default withStyles(styles)(
connect(
mapStateToProps,
mapDispatchToProps
)(withRouter(Lineage))
)
export default withStyles(styles)(connect(mapStateToProps, mapDispatchToProps)(withRouter(Lineage)))
5 changes: 1 addition & 4 deletions web/src/components/lineage/components/drag-bar/DragBar.tsx
Expand Up @@ -107,7 +107,4 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
},
dispatch
)
export default connect(
null,
mapDispatchToProps
)(withStyles(styles)(DragBar))
export default connect(null, mapDispatchToProps)(withStyles(styles)(DragBar))
42 changes: 41 additions & 1 deletion web/src/components/lineage/components/edge/Edge.tsx
@@ -1,19 +1,48 @@
// Copyright 2018-2023 contributors to the Marquez project
// SPDX-License-Identifier: Apache-2.0

import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
import { GraphEdge } from 'dagre'
import { LinePath } from '@visx/shape'
import { curveMonotoneX } from '@visx/curve'
import { faCaretRight } from '@fortawesome/free-solid-svg-icons/faCaretRight'
import { theme } from '../../../../helpers/theme'
import React from 'react'

type EdgeProps = {
edgePoints: GraphEdge[]
}

type EdgePoint = {
isSelected: boolean
points: {
x: number
y: number
}[]
}

const RADIUS = 14
const OUTER_RADIUS = RADIUS + 8
const ICON_SIZE = 16

class Edge extends React.Component<EdgeProps> {
getPoints = (edge: EdgePoint) => edge.points[edge.points.length - 1]

render() {
const { edgePoints } = this.props
const edgeEnds = edgePoints.map(edge => {
const isSelected = edgePoints.find(
o =>
this.getPoints(o as EdgePoint).x == this.getPoints(edge as EdgePoint).x &&
this.getPoints(o as EdgePoint).y == this.getPoints(edge as EdgePoint).y &&
o.isSelected === true
)
return {
...edge.points[edge.points.length - 1],
...{ isSelected: typeof isSelected !== 'undefined' }
}
})

return (
<>
{edgePoints.map((edge, i) => (
Expand All @@ -23,12 +52,23 @@ class Edge extends React.Component<EdgeProps> {
data={edge.points}
x={(d, index) => (index === 0 ? d.x + 20 : d.x - 25)}
y={d => d.y}
stroke={theme.palette.secondary.main}
stroke={edge.isSelected ? theme.palette.common.white : theme.palette.secondary.main}
strokeWidth={1}
opacity={1}
shapeRendering='geometricPrecision'
/>
))}
{edgeEnds.map((edge, i) => (
<FontAwesomeIcon
key={i}
icon={faCaretRight}
x={edge.x - OUTER_RADIUS - ICON_SIZE / 2}
y={edge.y - ICON_SIZE / 2}
width={ICON_SIZE}
height={ICON_SIZE}
color={edge.isSelected ? theme.palette.common.white : theme.palette.secondary.main}
/>
))}
</>
)
}
Expand Down

0 comments on commit f9465bf

Please sign in to comment.