Skip to content
Permalink
Browse files

[MLMD Lineage] Initial Lineage explorer draft implemented (#159)

* Add metadata store protos

* Add generated grpc protos for ml_metadata

* Uses the version from Pipelines

* Add generated grpc protos for ml_metadata

* Bump `react-scripts` to 3.2.0 to support `EXTEND_ESLINT` environment
  variable, which is needed to `eslintIgnore` generated protos.
* Add `api:metadata` script to regenerate grpc-web protos for ml_metadata.

* Proxy requests to server to metadata-envoy-service

* Move <rootDir>/__mocks__ into src to work around react-scripts@3.2.0 bug
  * facebook/create-react-app#7539
* Add `start-proxy.sh` to port forward to metadata-ui:3000
* Regenerate ArtifactList.test.tsx.snap
  - Old snapshots genreated with node 9.9.0 fail in node 12.13 used by
    docker image
* Whitelist @jest/* packages in gen_licenses.js and point to
  third_party/jest/LICENSE

* Switch ArtifactList getArtifactTypes to ml_metadata

* Major changes 1.0
- Migrated half of lineage components
- Rewrote all components in TS, updated and optimized to work in codebase
- Css updated
- **Erroneous, will fix**
- Added gitignore
- Added linter packages
- Fixed spacing

* Imported all elements

* All of Lineage Explorer added:
- All lineage components added
- Card styles for target / execution cards
- Rewrote the EdgeCavas renderer
- Dyanmic updates and CSS updates to match live view
- State diffing and rendering updates
- Mocked data and presented on UI, via dynamic data
- Z-indexing logic added to support reverse flipped connectors
- State logic updated to match standard
- Es6 Magic and refactor
- Massive css updates (radio button revamp, et al)

* Move changes to LineageView.tsx, which can then be used once Kwasi's changes are in

* Resolved Kwasi's comments
  • Loading branch information
avdaredevil authored and k8s-ci-robot committed Nov 12, 2019
1 parent a38effc commit db8fc5c49a9bd39b87ef6c8755b788e9821c23fe
Showing with 32,361 additions and 2,193 deletions.
  1. +1 −0 .gitignore
  2. +18 −0 frontend/gen_grpc_web_protos.sh
  3. +4 −0 frontend/gen_licenses.js
  4. +3,287 −2,100 frontend/package-lock.json
  5. +19 −7 frontend/package.json
  6. +17 −1 frontend/server/server.ts
  7. +49 −2 frontend/src/Css.tsx
  8. 0 frontend/{ → src}/__mocks__/typestyle.ts
  9. +424 −0 frontend/src/apis/metadata/metadata_store.proto
  10. +739 −0 frontend/src/apis/metadata/metadata_store_service.proto
  11. +20 −0 frontend/src/components/EdgeCanvas.css
  12. +103 −0 frontend/src/components/EdgeCanvas.tsx
  13. +1 −0 frontend/src/components/LineChart.d.ts
  14. +66 −0 frontend/src/components/LineageCard.css
  15. +39 −0 frontend/src/components/LineageCard.tsx
  16. +28 −0 frontend/src/components/LineageCardColumn.css
  17. +57 −0 frontend/src/components/LineageCardColumn.tsx
  18. +112 −0 frontend/src/components/LineageCardRow.css
  19. +40 −0 frontend/src/components/LineageCardRow.tsx
  20. +12 −0 frontend/src/components/LineageTypes.ts
  21. +21 −0 frontend/src/fonts.scss
  22. +926 −0 frontend/src/generated/src/apis/metadata/metadata_store_pb.d.ts
  23. +6,979 −0 frontend/src/generated/src/apis/metadata/metadata_store_pb.js
  24. +3 −0 frontend/src/generated/src/apis/metadata/metadata_store_pb_service.d.ts
  25. +3 −0 frontend/src/generated/src/apis/metadata/metadata_store_pb_service.js
  26. +1,809 −0 frontend/src/generated/src/apis/metadata/metadata_store_service_pb.d.ts
  27. +14,489 −0 frontend/src/generated/src/apis/metadata/metadata_store_service_pb.js
  28. +747 −0 frontend/src/generated/src/apis/metadata/metadata_store_service_pb_service.d.ts
  29. +1,501 −0 frontend/src/generated/src/apis/metadata/metadata_store_service_pb_service.js
  30. +36 −0 frontend/src/lib/Api.ts
  31. +73 −38 frontend/src/pages/ArtifactList.test.tsx
  32. +25 −16 frontend/src/pages/ArtifactList.tsx
  33. +130 −0 frontend/src/pages/LineageView.tsx
  34. +11 −0 frontend/src/pages/Page.tsx
  35. +27 −27 frontend/src/pages/__snapshots__/ArtifactList.test.tsx.snap
  36. +27 −0 frontend/start-proxy.sh
  37. +3 −1 frontend/tsconfig.json
  38. +1 −1 frontend/tslint.json
  39. +514 −0 package-lock.json
@@ -78,4 +78,5 @@ dist
.tox
sdk/python/kubeflow/metadata/openapi_client/
sdk/python/build
node_modules/
.env
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
PROTOC_GEN_TS_PATH="./node_modules/.bin/protoc-gen-ts"
OUT_DIR="./src/generated"

if [[ ! -d ${OUT_DIR} ]]; then
mkdir ${OUT_DIR}
fi

# Expects protoc to be on your PATH.
#
# From npm/google-protobuf:
# The compiler is not currently available via npm, but you can download a
# pre-built binary on GitHub (look for the protoc-*.zip files under Downloads).
protoc \
--plugin="protoc-gen-ts=${PROTOC_GEN_TS_PATH}" \
--js_out="import_style=commonjs,binary:${OUT_DIR}" \
--ts_out="service=grpc-web:${OUT_DIR}" \
src/apis/**/*.proto
@@ -29,6 +29,10 @@ const path = require('path');
const start = path.resolve(process.cwd(), process.argv[2]);
let licenseMissing = 0;
const whitelist = new Map([
['@jest/environment', 'third_party/jest/LICENSE'],
['@jest/reporters', 'third_party/jest/LICENSE'],
['@jest/test-sequencer', 'third_party/jest/LICENSE'],
['@jest/transform', 'third_party/jest/LICENSE'],
['eslint-module-utils', 'third_party/eslint/LICENSE'],
['expect', 'third_party/jest/LICENSE'],
['jest-config', 'third_party/jest/LICENSE'],

Large diffs are not rendered by default.

@@ -3,14 +3,16 @@
"version": "0.1.0",
"private": true,
"dependencies": {
"@improbable-eng/grpc-web": "^0.11.0",
"@material-ui/core": "^3.7.1",
"@material-ui/icons": "^3.0.1",
"lodash": ">=4.17.11",
"lodash": "^4.17.15",
"node-sass": "^4.13.0",
"portable-fetch": "^3.0.0",
"react": "^16.8.6",
"react-dom": "^16.8.6",
"react-router-dom": "^4.3.1",
"react-scripts": "^3.0.1",
"react-scripts": "^3.2.0",
"typestyle": "^2.0.1"
},
"devDependencies": {
@@ -23,28 +25,36 @@
"@types/react": "^16.8.18",
"@types/react-dom": "^16.8.4",
"@types/react-router-dom": "^4.3.1",
"cross-env": "^6.0.3",
"enzyme": "^3.10.0",
"enzyme-adapter-react-16": "^1.14.0",
"enzyme-to-json": "^3.3.5",
"license-checker": "^25.0.1",
"react-router-test-context": "^0.1.0",
"ts-protoc-gen": "^0.12.0",
"typescript": "^3.4.5"
},
"scripts": {
"apis": "npm run apis:api && npm run apis:service",
"apis:api": "java -jar swagger-codegen-cli.jar generate -i ../api/api.swagger.json -l typescript-fetch -o ./src/apis/api -c ./swagger-config.json",
"apis:metadata": "./generate_grpc_web_protos.sh",
"apis:service": "java -jar swagger-codegen-cli.jar generate -i ../api/service.swagger.json -l typescript-fetch -o ./src/apis/service -c ./swagger-config.json",
"build": "react-scripts build",
"docker": "COMMIT_HASH=`git rev-parse HEAD`; docker build -t gcr.io/kubeflow-images-public/metadata-frontend:${COMMIT_HASH} --build-arg COMMIT_HASH=${COMMIT_HASH} --build-arg DATE=\"`date -u`\" .",
"eject": "react-scripts eject",
"build": "cross-env EXTEND_ESLINT=true react-scripts build",
"docker": "cross-env COMMIT_HASH=`git rev-parse HEAD`; docker build -t gcr.io/kubeflow-images-public/metadata-frontend:${COMMIT_HASH} --build-arg COMMIT_HASH=${COMMIT_HASH} --build-arg DATE=\"`date -u`\" .",
"eject": "cross-env EXTEND_ESLINT=true react-scripts eject",
"postinstall": "cd ./server && npm i",
"start": "react-scripts start",
"test": "TZ='America/New_York' react-scripts test --env=jsdom",
"start": "cross-env EXTEND_ESLINT=true react-scripts start",
"start:proxy": "./start-proxy.sh",
"test": "cross-env TZ='America/New_York' EXTEND_ESLINT=true react-scripts test --env=jsdom",
"test:coverage": "npm test -- --coverage --watchAll=false"
},
"eslintConfig": {
"extends": "react-app"
},
"eslintIgnore": [
"src/__mocks__",
"src/generated"
],
"browserslist": {
"production": [
">0.2%",
@@ -61,7 +71,9 @@
"jest": {
"collectCoverageFrom": [
"src/**/*.{ts,tsx}",
"!src/__mocks__",
"!src/apis/**/*.{ts,tsx}",
"!src/generated/**/*",
"!<rootDir>/node_modules/"
],
"coverageReporters": [
@@ -26,7 +26,11 @@ const {
/** API service will listen to this host */
METADATA_SERVICE_SERVICE_HOST = 'localhost',
/** API service will listen to this port */
METADATA_SERVICE_SERVICE_PORT = '8080'
METADATA_SERVICE_SERVICE_PORT = '8080',
/** Envoy service will listen to this host */
METADATA_ENVOY_SERVICE_SERVICE_HOST = 'localhost',
/** Envoy service will listen to this port */
METADATA_ENVOY_SERVICE_SERVICE_PORT = '9090',
} = process.env;

const app = express() as Application;
@@ -50,6 +54,8 @@ const staticDir = path.resolve(process.argv[2]);
const port = process.argv[3] || 3000;
const apiServerAddress = `http://${METADATA_SERVICE_SERVICE_HOST}:${METADATA_SERVICE_SERVICE_PORT}`;

const envoyServiceAddress = `http://${METADATA_ENVOY_SERVICE_SERVICE_HOST}:${METADATA_ENVOY_SERVICE_SERVICE_PORT}`

const v1alpha1Prefix = 'api/v1alpha1';

const clusterNameHandler = async (req, res) => {
@@ -74,6 +80,16 @@ app.get(BASEPATH + '/system/cluster-name', clusterNameHandler);
app.get('/system/project-id', projectIdHandler);
app.get(BASEPATH + '/system/project-id', projectIdHandler);

// Proxy metadata requests to the Envoy instance which will handle routing to the metadata gRPC server
console.log('Starting server with gRPC proxy for /ml_metadata');
app.all('/ml_metadata.*', proxy({
changeOrigin: true,
onProxyReq: proxyReq => {
console.log('Metadata proxied request: ', (proxyReq as any).path);
},
target: envoyServiceAddress,
}));

app.all('/' + v1alpha1Prefix + '/*', proxy({
changeOrigin: true,
onProxyReq: proxyReq => {
@@ -15,8 +15,46 @@
*/

import createMuiTheme from '@material-ui/core/styles/createMuiTheme';
import { NestedCSSProperties } from 'typestyle/lib/types';
import { style, stylesheet } from 'typestyle';
import {NestedCSSProperties} from 'typestyle/lib/types';
import {style, stylesheet, cssRaw} from 'typestyle';
import './fonts.scss';

cssRaw(`
.LineageExplorer {
--blue-50: #e8f0fe;
--blue-100: #d2e3fc;
--blue-200: #aecbfa;
--blue-300: #8ab4f8;
--blue-400: #669df6;
--blue-500: #1098f7;
--blue-600: #1098f7;
--blue-700: #1967d2;
--blue-800: #185abc;
--blue-900: #174ea6;
--red-50: #fce8e6;
--red-700: #c5221f;
--yellow-600: #f9ab00;
--green-500: #34a853;
--green-600: #1e8e3e;
--grey-50: #f8f9fa;
--grey-100: #f1f3f4;
--grey-200: #e8eaed;
--grey-300: #dadce0;
--grey-400: #bdc1c6;
--grey-500: #9aa0a6;
--grey-600: #80868b;
--grey-700: #5f6368;
--grey-800: #3c4043;
--grey-900: #202124;
--orange-500: #fa7b17;
--orange-700: #d56e0c;
--orange-900: #b06000;
--pink-500: #f538a0;
--pink-700: #c92786;
--purple-500: #a142f4;
--card-radius: 6px;
}
`);

export const color = {
activeBg: '#eaf1fd',
@@ -219,6 +257,15 @@ export const commonCss = stylesheet({
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
},
fit: {
bottom: 0,
height: '100%',
left: 0,
position: 'absolute',
right: 0,
top: 0,
width: '100%',
},
flex: {
alignItems: 'center !important',
display: 'flex !important',
File renamed without changes.

0 comments on commit db8fc5c

Please sign in to comment.
You can’t perform that action at this time.