Skip to content

Commit

Permalink
fix: Add stream feature view in the Web UI (#3257)
Browse files Browse the repository at this point in the history
* add stream feature view to ui

Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com>

* update source

Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com>

* update example

Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com>

* add registry

Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com>

* fix lint

Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com>

* fix bug

Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com>

* add batch source

Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com>

* fix warning

Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com>

Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com>
  • Loading branch information
hao-affirm committed Sep 30, 2022
1 parent 532d8a1 commit 1f70b3a
Show file tree
Hide file tree
Showing 16 changed files with 625 additions and 5 deletions.
72 changes: 72 additions & 0 deletions ui/public/registry.json
Expand Up @@ -29,6 +29,24 @@
"name": "zipcode",
"timestampField": "event_timestamp",
"type": "BATCH_FILE"
},
{
"batchSource": {
"fileOptions": {
"uri": "data/zipcode_table.parquet"
},
"name": "user_stats",
"timestampField": "timestamp",
"type": "BATCH_FILE"
},
"dataSourceClassType": "feast.data_source.KafkaSource",
"description": "The Kafka stream example",
"kafkaOptions": {"messageFormat": {"jsonFormat": {"schemaJson": "id string, timestamp timestamp"}},
"watermarkDelayThreshold": "300s"},
"name": "driver_stats_stream",
"owner": "test@gmail.com",
"timestampField": "timestamp",
"type": "STREAM_KAFKA"
}
],
"entities": [
Expand Down Expand Up @@ -630,5 +648,59 @@
}
}
],
"streamFeatureViews": [
{
"meta": {
"createdTimestamp": "2022-05-11T19:27:03.171556Z",
"lastUpdatedTimestamp": "2022-05-11T19:27:03.171556Z"
},
"spec": {
"batchSource": {
"createdTimestampColumn": "created_timestamp",
"dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource",
"fileOptions": {
"uri": "data/zipcode_table.parquet"
},
"name": "zipcode",
"timestampField": "event_timestamp",
"type": "BATCH_FILE"
},
"features": [
{
"name": "conv_percentage",
"valueType": "FLOAT"
},
{
"name": "acc_percentage",
"valueType": "FLOAT"
}
],
"name": "transaction_stream_example",
"streamSource": {
"batchSource": {
"fileOptions": {
"uri": "data/zipcode_table.parquet"
},
"name": "user_stats",
"timestampField": "timestamp",
"type": "BATCH_FILE"
},
"dataSourceClassType": "feast.data_source.KafkaSource",
"description": "The Kafka stream example",
"kafkaOptions": {"messageFormat": {"jsonFormat": {"schemaJson": "id string, timestamp timestamp"}},
"watermarkDelayThreshold": "300s"},
"name": "driver_stats_stream",
"owner": "test@gmail.com",
"timestampField": "timestamp",
"type": "STREAM_KAFKA"
},
"ttl": "86400s",
"userDefinedFunction": {
"body": "@stream_feature_view(\n sources=[driver_stats_stream_source],\n mode=\"spark\",\n schema=[\n Field(name=\"conv_percentage\", dtype=Float32),\n Field(name=\"acc_percentage\", dtype=Float32),\n ],\n timestamp_field=\"event_timestamp\",\n online=True,\n source=driver_stats_stream_source,\n tags={},\n)\ndef driver_hourly_stats_stream(df: DataFrame) -> DataFrame:\n from pyspark.sql.functions import col\n return (\n df.withColumn(\"conv_percentage\", col(\"conv_rate\") * 100.0)\n .withColumn(\"acc_percentage\", col(\"acc_rate\") * 100.0)\n .drop(\"conv_rate\", \"acc_rate\")\n )\n",
"name": "driver_hourly_stats_stream"
}
}
}
],
"project": "credit_scoring_aws"
}
25 changes: 25 additions & 0 deletions ui/src/custom-tabs/TabsRegistryContext.tsx
Expand Up @@ -10,6 +10,7 @@ import {

import RegularFeatureViewCustomTabLoadingWrapper from "../utils/custom-tabs/RegularFeatureViewCustomTabLoadingWrapper";
import OnDemandFeatureViewCustomTabLoadingWrapper from "../utils/custom-tabs/OnDemandFeatureViewCustomTabLoadingWrapper";
import StreamFeatureViewCustomTabLoadingWrapper from "../utils/custom-tabs/StreamFeatureViewCustomTabLoadingWrapper";
import FeatureServiceCustomTabLoadingWrapper from "../utils/custom-tabs/FeatureServiceCustomTabLoadingWrapper";
import FeatureCustomTabLoadingWrapper from "../utils/custom-tabs/FeatureCustomTabLoadingWrapper";
import DataSourceCustomTabLoadingWrapper from "../utils/custom-tabs/DataSourceCustomTabLoadingWrapper";
Expand All @@ -19,6 +20,7 @@ import DatasetCustomTabLoadingWrapper from "../utils/custom-tabs/DatasetCustomTa
import {
RegularFeatureViewCustomTabRegistrationInterface,
OnDemandFeatureViewCustomTabRegistrationInterface,
StreamFeatureViewCustomTabRegistrationInterface,
FeatureServiceCustomTabRegistrationInterface,
FeatureCustomTabRegistrationInterface,
DataSourceCustomTabRegistrationInterface,
Expand All @@ -30,6 +32,7 @@ import {
interface FeastTabsRegistryInterface {
RegularFeatureViewCustomTabs?: RegularFeatureViewCustomTabRegistrationInterface[];
OnDemandFeatureViewCustomTabs?: OnDemandFeatureViewCustomTabRegistrationInterface[];
StreamFeatureViewCustomTabs?: StreamFeatureViewCustomTabRegistrationInterface[];
FeatureServiceCustomTabs?: FeatureServiceCustomTabRegistrationInterface[];
FeatureCustomTabs?: FeatureCustomTabRegistrationInterface[];
DataSourceCustomTabs?: DataSourceCustomTabRegistrationInterface[];
Expand Down Expand Up @@ -148,6 +151,16 @@ const useOnDemandFeatureViewCustomTabs = (navigate: NavigateFunction) => {
);
};

const useStreamFeatureViewCustomTabs = (navigate: NavigateFunction) => {
const { StreamFeatureViewCustomTabs } =
React.useContext(TabsRegistryContext);

return useGenericCustomTabsNavigation<StreamFeatureViewCustomTabRegistrationInterface>(
StreamFeatureViewCustomTabs || [],
navigate
);
};

const useFeatureServiceCustomTabs = (navigate: NavigateFunction) => {
const { FeatureServiceCustomTabs } = React.useContext(TabsRegistryContext);

Expand Down Expand Up @@ -214,6 +227,16 @@ const useOnDemandFeatureViewCustomTabRoutes = () => {
);
};

const useStreamFeatureViewCustomTabRoutes = () => {
const { StreamFeatureViewCustomTabs } =
React.useContext(TabsRegistryContext);

return genericCustomTabRoutes(
StreamFeatureViewCustomTabs || [],
StreamFeatureViewCustomTabLoadingWrapper
);
};

const useFeatureServiceCustomTabRoutes = () => {
const { FeatureServiceCustomTabs } = React.useContext(TabsRegistryContext);

Expand Down Expand Up @@ -264,6 +287,7 @@ export {
// Navigation
useRegularFeatureViewCustomTabs,
useOnDemandFeatureViewCustomTabs,
useStreamFeatureViewCustomTabs,
useFeatureServiceCustomTabs,
useFeatureCustomTabs,
useDataSourceCustomTabs,
Expand All @@ -272,6 +296,7 @@ export {
// Routes
useRegularFeatureViewCustomTabRoutes,
useOnDemandFeatureViewCustomTabRoutes,
useStreamFeatureViewCustomTabRoutes,
useFeatureServiceCustomTabRoutes,
useFeatureCustomTabRoutes,
useDataSourceCustomTabRoutes,
Expand Down
85 changes: 85 additions & 0 deletions ui/src/custom-tabs/stream-fv-demo-tab/DemoCustomTab.tsx
@@ -0,0 +1,85 @@
import React from "react";

import {
// Feature View Custom Tabs will get these props
StreamFeatureViewCustomTabProps,
} from "../types";

import {
EuiLoadingContent,
EuiEmptyPrompt,
EuiFlexGroup,
EuiFlexItem,
EuiCode,
EuiSpacer,
} from "@elastic/eui";

// Separating out the query is not required,
// but encouraged for code readability
import useDemoQuery from "./useDemoQuery";

const DemoCustomTab = ({
id,
feastObjectQuery,
}: StreamFeatureViewCustomTabProps) => {
// Use React Query to fetch data
// that is custom to this tab.
// See: https://react-query.tanstack.com/guides/queries
const { isLoading, isError, isSuccess, data } = useDemoQuery({
featureView: id,
});

if (isLoading) {
// Handle Loading State
// https://elastic.github.io/eui/#/display/loading
return <EuiLoadingContent lines={3} />;
}

if (isError) {
// Handle Data Fetching Error
// https://elastic.github.io/eui/#/display/empty-prompt
return (
<EuiEmptyPrompt
iconType="alert"
color="danger"
title={<h2>Unable to load your demo page</h2>}
body={
<p>
There was an error loading the Dashboard application. Contact your
administrator for help.
</p>
}
/>
);
}

// Feast UI uses the Elastic UI component system.
// <EuiFlexGroup> and <EuiFlexItem> are particularly
// useful for layouts.
return (
<React.Fragment>
<EuiFlexGroup>
<EuiFlexItem grow={1}>
<p>Hello World. The following is fetched data.</p>
<EuiSpacer />
{isSuccess && data && (
<EuiCode>
<pre>{JSON.stringify(data, null, 2)}</pre>
</EuiCode>
)}
</EuiFlexItem>
<EuiFlexItem grow={2}>
<p>... and this is data from Feast UI&rsquo;s own query.</p>
<EuiSpacer />
{feastObjectQuery.isSuccess && feastObjectQuery.data && (
<EuiCode>
<pre>{JSON.stringify(feastObjectQuery.data, null, 2)}</pre>
</EuiCode>
)}
</EuiFlexItem>
</EuiFlexGroup>
</React.Fragment>
);
};

export default DemoCustomTab;
44 changes: 44 additions & 0 deletions ui/src/custom-tabs/stream-fv-demo-tab/useDemoQuery.tsx
@@ -0,0 +1,44 @@
import { useQuery } from "react-query";
import { z } from "zod";

// Use Zod to check the shape of the
// json object being loaded
const demoSchema = z.object({
hello: z.string(),
name: z.string().optional(),
});

// Make the type of the object available
type DemoDataType = z.infer<typeof demoSchema>;

interface DemoQueryInterface {
featureView: string | undefined;
}

const useDemoQuery = ({ featureView }: DemoQueryInterface) => {
// React Query manages caching for you based on query keys
// See: https://react-query.tanstack.com/guides/query-keys
const queryKey = `demo-tab-namespace:${featureView}`;

// Pass the type to useQuery
// so that components consuming the
// result gets nice type hints
// on the other side.
return useQuery<DemoDataType>(
queryKey,
() => {
// Customizing the URL based on your needs
const url = `/demo-custom-tabs/demo.json`;

return fetch(url)
.then((res) => res.json())
.then((data) => demoSchema.parse(data)); // Use zod to parse results
},
{
enabled: !!featureView, // Only start the query when the variable is not undefined
}
);
};

export default useDemoQuery;
export type { DemoDataType };
21 changes: 21 additions & 0 deletions ui/src/custom-tabs/types.ts
@@ -1,5 +1,6 @@
import {
useLoadOnDemandFeatureView,
useLoadStreamFeatureView,
useLoadRegularFeatureView,
} from "../pages/feature-views/useLoadFeatureView";
import useLoadFeature from "../pages/features/useLoadFeature";
Expand Down Expand Up @@ -48,6 +49,23 @@ interface OnDemandFeatureViewCustomTabRegistrationInterface
}: OnDemandFeatureViewCustomTabProps) => JSX.Element;
}

// Type for Stream Feature View Custom Tabs
type StreamFeatureViewQueryReturnType = ReturnType<
typeof useLoadStreamFeatureView
>;
interface StreamFeatureViewCustomTabProps {
id: string | undefined;
feastObjectQuery: StreamFeatureViewQueryReturnType;
}
interface StreamFeatureViewCustomTabRegistrationInterface
extends CustomTabRegistrationInterface {
Component: ({
id,
feastObjectQuery,
...args
}: StreamFeatureViewCustomTabProps) => JSX.Element;
}

// Type for Entity Custom Tabs
interface EntityCustomTabProps {
id: string | undefined;
Expand Down Expand Up @@ -127,6 +145,9 @@ export type {
OnDemandFeatureViewQueryReturnType,
OnDemandFeatureViewCustomTabProps,
OnDemandFeatureViewCustomTabRegistrationInterface,
StreamFeatureViewQueryReturnType,
StreamFeatureViewCustomTabProps,
StreamFeatureViewCustomTabRegistrationInterface,
FeatureServiceCustomTabRegistrationInterface,
FeatureServiceCustomTabProps,
DataSourceCustomTabRegistrationInterface,
Expand Down
10 changes: 9 additions & 1 deletion ui/src/index.tsx
Expand Up @@ -18,6 +18,7 @@ import FeastUI from "./FeastUI";
import DataTab from "./custom-tabs/data-tab/DataTab";
import RFVDemoCustomTab from "./custom-tabs/reguar-fv-demo-tab/DemoCustomTab";
import ODFVDemoCustomTab from "./custom-tabs/ondemand-fv-demo-tab/DemoCustomTab";
import SFVDemoCustomTab from "./custom-tabs/stream-fv-demo-tab/DemoCustomTab";
import FSDemoCustomTab from "./custom-tabs/feature-service-demo-tab/DemoCustomTab";
import DSDemoCustomTab from "./custom-tabs/data-source-demo-tab/DemoCustomTab";
import EntDemoCustomTab from "./custom-tabs/entity-demo-tab/DemoCustomTab";
Expand Down Expand Up @@ -46,6 +47,13 @@ const tabsRegistry = {
Component: ODFVDemoCustomTab,
},
],
StreamFeatureViewCustomTabs: [
{
label: "Custom Tab Demo",
path: "demo-tab",
Component: SFVDemoCustomTab,
},
],
FeatureServiceCustomTabs: [
{
label: "Custom Tab Demo",
Expand Down Expand Up @@ -93,4 +101,4 @@ ReactDOM.render(
/>
</React.StrictMode>,
document.getElementById("root")
);
);
8 changes: 8 additions & 0 deletions ui/src/pages/feature-views/FeatureViewInstance.tsx
Expand Up @@ -7,8 +7,11 @@ import { FeastFeatureViewType } from "../../parsers/feastFeatureViews";
import RegularFeatureInstance from "./RegularFeatureViewInstance";
import { FEAST_FV_TYPES } from "../../parsers/mergedFVTypes";
import { FeastODFVType } from "../../parsers/feastODFVS";
import { FeastSFVType } from "../../parsers/feastSFVS";
import useLoadFeatureView from "./useLoadFeatureView";
import OnDemandFeatureInstance from "./OnDemandFeatureViewInstance";
import StreamFeatureInstance from "./StreamFeatureViewInstance";


const FeatureViewInstance = () => {
const { featureViewName } = useParams();
Expand Down Expand Up @@ -45,6 +48,11 @@ const FeatureViewInstance = () => {

return <OnDemandFeatureInstance data={odfv} />;
}
if (data.type === FEAST_FV_TYPES.stream) {
const sfv: FeastSFVType = data.object;

return <StreamFeatureInstance data={sfv} />;
}
}

return <p>No Data So Sad</p>;
Expand Down
2 changes: 1 addition & 1 deletion ui/src/pages/feature-views/FeatureViewListingTable.tsx
Expand Up @@ -35,7 +35,7 @@ const FeatureViewListingTable = ({
href={`/p/${projectName}/feature-view/${name}`}
to={`/p/${projectName}/feature-view/${name}`}
>
{name} {item.type === "ondemand" && <EuiBadge>ondemand</EuiBadge>}
{name} {(item.type === "ondemand" && <EuiBadge>ondemand</EuiBadge>) || (item.type === "stream" && <EuiBadge>stream</EuiBadge>)}
</EuiCustomLink>
);
},
Expand Down

0 comments on commit 1f70b3a

Please sign in to comment.