\n", + "To setup authentication use the ```ads.set_auth(\"resource_principal\")``` or ```ads.set_auth(\"api_key\")```." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "419755bd", + "metadata": { + "ExecuteTime": { + "start_time": "2023-05-24T08:26:08.577504Z" + }, + "is_executing": true, + "pycharm": { + "is_executing": true, + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import ads\n", + "ads.set_auth(auth=\"resource_principal\", client_kwargs={\"fs_service_endpoint\": \"https://{api_gateway}/20230101\"})" + ] + }, + { + "cell_type": "markdown", + "id": "869ab9aa", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "\n", + "### 2.4. Variables\n", + "To run this notebook, you must provide some information about your tenancy configuration. To create and run a feature store, you must specify a `
\n",
+ " ![](\"https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/_images/feature_group_flights.gif\")
\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cbeed679",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_flights = (\n",
+ " FeatureGroup()\n",
+ " .with_feature_store_id(feature_store.id)\n",
+ " .with_primary_keys([\"FLIGHT_NUMBER\"])\n",
+ " .with_name(\"flights_feature_group\")\n",
+ " .with_entity_id(entity.id)\n",
+ " .with_compartment_id(compartment_id)\n",
+ " .with_schema_details_from_dataframe(flights_df)\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fa811b52",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_flights.create()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e0da6f0d",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_flights.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8da122b1",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_flights.materialise(flights_df)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "67016236",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "##### Airport Feature Group\n",
+ "\n",
+ "Create feature group for airport"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5fabe3f2",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "expectation_suite_airports = ExpectationSuite(\n",
+ " expectation_suite_name=\"test_airports_df\"\n",
+ ")\n",
+ "expectation_suite_airports.add_expectation(\n",
+ " ExpectationConfiguration(\n",
+ " expectation_type=\"expect_column_values_to_not_be_null\",\n",
+ " kwargs={\"column\": \"IATA_CODE\"},\n",
+ " )\n",
+ ")\n",
+ "expectation_suite_airports.add_expectation(\n",
+ " ExpectationConfiguration(\n",
+ " expectation_type=\"expect_column_values_to_be_between\",\n",
+ " kwargs={\"column\": \"LATITUDE\", \"min_value\": -1.0, \"max_value\": 1.0},\n",
+ " )\n",
+ ")\n",
+ "\n",
+ "expectation_suite_airports.add_expectation(\n",
+ " ExpectationConfiguration(\n",
+ " expectation_type=\"expect_column_values_to_be_between\",\n",
+ " kwargs={\"column\": \"LONGITUDE\", \"min_value\": -1.0, \"max_value\": 1.0},\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "99294b1c",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airports = (\n",
+ " FeatureGroup()\n",
+ " .with_feature_store_id(feature_store.id)\n",
+ " .with_primary_keys([\"IATA_CODE\"])\n",
+ " .with_name(\"airport_feature_group\")\n",
+ " .with_entity_id(entity.id)\n",
+ " .with_compartment_id(compartment_id)\n",
+ " .with_schema_details_from_dataframe(airports_df)\n",
+ " .with_expectation_suite(\n",
+ " expectation_suite=expectation_suite_airports,\n",
+ " expectation_type=ExpectationType.LENIENT,\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "91301811",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airports.create()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "767ce780",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airports.materialise(airports_df)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1fce8a51",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airports.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "15ac3504",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "##### Airlines Feature Group\n",
+ "\n",
+ "Create feature group for airlines\n",
+ "\n",
+ "\n",
+ "\n",
+ " ![](\"https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/_images/feature_groups.gif\")
\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "192ad5cd",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a95c8833",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "expectation_suite_airlines = ExpectationSuite(\n",
+ " expectation_suite_name=\"test_airlines_df\"\n",
+ ")\n",
+ "expectation_suite_airlines.add_expectation(\n",
+ " ExpectationConfiguration(\n",
+ " expectation_type=\"expect_column_values_to_not_be_null\",\n",
+ " kwargs={\"column\": \"IATA_CODE\"},\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0976a401",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airlines = (\n",
+ " FeatureGroup()\n",
+ " .with_feature_store_id(feature_store.id)\n",
+ " .with_primary_keys([\"IATA_CODE\"])\n",
+ " .with_name(\"airlines_feature_group\")\n",
+ " .with_entity_id(entity.id)\n",
+ " .with_compartment_id(compartment_id)\n",
+ " .with_schema_details_from_dataframe(airlines_df)\n",
+ " .with_expectation_suite(\n",
+ " expectation_suite=expectation_suite_airlines,\n",
+ " expectation_type=ExpectationType.STRICT,\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b9d7ef56",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.create()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b926675d",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.materialise(airlines_df)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "707c723d",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "683f2b6f",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "### 3.3. Explore feature groups"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d06d4dbc",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_flights.get_features_df()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5fb0ed95",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airports.get_features_df()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "edd5c063",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.get_features_df()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "647a3818",
+ "metadata": {},
+ "source": [
+ "You can retrieve feature data in a DataFrame, that can either be used to train models."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3484d5af",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_flights.select().show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "53111598",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airports.select().show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1947ac2b",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.select().show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5ff4ffc6",
+ "metadata": {},
+ "source": [
+ "You can call the `get_statistics()` method of the feature group to fetch statistics for a specific ingestion job.You can use `to_pandas()` or `to_json()` to view the statistics.\n",
+ "You can visualize feature statistics with `to_viz()`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "30d69581",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.get_statistics().to_pandas()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "02bec075",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_flights.get_statistics().to_pandas()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e31148f6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.get_statistics().to_viz()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "11f3a879",
+ "metadata": {},
+ "source": [
+ "You can call the `get_validation_output()` method of the FeatureGroup instance to fetch validation results for a specific ingestion job."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d382ff25",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.get_validation_output().to_pandas()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "442d4462",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.get_validation_output().to_summary()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e3ded350",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "### 3.4. Select subset of features"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e11b7184",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.select(['IATA_CODE']).show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "16561536",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "### 3.5. Filter feature groups"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1a251d97",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airlines.filter(feature_group_airlines.IATA_CODE == \"EV\").show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2944b0e7",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "### 3.6. Apply joins on feature groups\n",
+ "As in Pandas, if the feature has the same name on both feature groups, then you can use the `on=[]` paramter. If they have different names, then you can use the `left_on=[]` and `right_on=[]` paramters:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "56bedaff",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "from ads.feature_store.common.enums import JoinType\n",
+ "\n",
+ "query = (\n",
+ " feature_group_flights.select()\n",
+ " .join(feature_group_airlines.select(), left_on=['ORIGIN_AIRPORT'], right_on=['IATA_CODE'], join_type=JoinType.LEFT)\n",
+ " .join(feature_group_airports.select(), left_on=['AIRLINE'], right_on=['IATA_CODE'], join_type=JoinType.LEFT)\n",
+ ")\n",
+ "query.show(5)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5d77bcde",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "query.to_string()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b018652b",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "### 3.7. Create dataset from multiple or one feature group\n",
+ "A dataset is a collection of feature snapshots that are joined together to either train a model or perform model inference.\n",
+ "\n",
+ "\n",
+ "\n",
+ " ![](\"https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/_images/dataset.gif\")
\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1df69889",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "dataset = (\n",
+ " Dataset()\n",
+ " .with_description(\"Combined dataset for flights\")\n",
+ " .with_compartment_id(compartment_id)\n",
+ " .with_name(\"flights_dataset\")\n",
+ " .with_entity_id(entity.id)\n",
+ " .with_feature_store_id(feature_store.id)\n",
+ " .with_query(query.to_string())\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5bb3d9ff",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "##### Create Dataset\n",
+ "\n",
+ "Call the ```.create()``` method of the Dataset instance to create a dataset."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b5dd4e45",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "dataset.create()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fc64c019",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "dataset.materialise()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6fabd82e",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "#### Interoperability with model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "14b04a7c",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "model_details = ModelDetails().with_items([\"ocid1.modelcatalog.oc1.unique_ocid\"])\n",
+ "dataset.add_models(model_details)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2305f112",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "#### Visualise lineage\n",
+ "\n",
+ "Use the ```.show()``` method on the Dataset instance to visualize the lineage of the dataset."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bad948c4",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "dataset.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f3125aa8",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "dataset.profile().show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "840a47fa",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "dataset.as_of(version_number=0).show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8b3ed236",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "### 3.8. Freeform SQL query\n",
+ "Feature store provides a way to query feature store using free flow query. User need to mention `entity id` as the database name and `feature group name` as the table name to query feature store. This functionality can be useful if you need to express more complex queries for your use case"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5d38518f",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "entity_id = entity.id\n",
+ "\n",
+ "sql = (f\"SELECT flights_feature_group.*, airport_feature_group.IATA_CODE \"\n",
+ " f\"FROM `{entity_id}`.flights_feature_group flights_feature_group \"\n",
+ " f\"LEFT JOIN `{entity_id}`.airport_feature_group airport_feature_group \"\n",
+ " f\"ON flights_feature_group.ORIGIN_AIRPORT=airport_feature_group.IATA_CODE\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "425b900b",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_store.sql(sql).show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1962f56e",
+ "metadata": {},
+ "source": [
+ "\n",
+ "### 3.9. Feature store Entities using YAML\n",
+ "In an ADS feature store module, you can either use the Python programmatic interface or YAML to define feature store entities. Below section describes how to create feature store entities using YAML as an interface."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2734866a",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_store_yaml = \"\"\"\n",
+ "apiVersion: v1\n",
+ "kind: featureStore\n",
+ "spec:\n",
+ " displayName: Flights feature store\n",
+ " compartmentId: \"ocid1.tenancy.oc1..aaaaaaaa462hfhplpx652b32ix62xrdijppq2c7okwcqjlgrbknhgtj2kofa\"\n",
+ " offlineConfig:\n",
+ " metastoreId: \"ocid1.datacatalogmetastore.oc1.iad.amaaaaaabiudgxyap7tizm4gscwz7amu7dixz7ml3mtesqzzwwg3urvvdgua\"\n",
+ "\n",
+ " entity: &flights_entity\n",
+ " - kind: entity\n",
+ " spec:\n",
+ " name: Flights\n",
+ "\n",
+ " featureGroup:\n",
+ " - kind: featureGroup\n",
+ " spec:\n",
+ " entity: *flights_entity\n",
+ " name: flights_feature_group\n",
+ " primaryKeys:\n",
+ " - IATA_CODE\n",
+ " inputFeatureDetails:\n",
+ " - featureType: STRING\n",
+ " name: IATA_CODE\n",
+ " orderNumber: 1\n",
+ " - featureType: STRING\n",
+ " name: AIRPORT\n",
+ " orderNumber: 2\n",
+ " - featureType: STRING\n",
+ " name: CITY\n",
+ " orderNumber: 3\n",
+ " - featureType: STRING\n",
+ " name: STATE\n",
+ " orderNumber: 4\n",
+ " - featureType: STRING\n",
+ " name: COUNTRY\n",
+ " orderNumber: 5\n",
+ " - featureType: FLOAT\n",
+ " name: LATITUDE\n",
+ " orderNumber: 6\n",
+ " - featureType: FLOAT\n",
+ " name: LONGITUDE\n",
+ " orderNumber: 7\n",
+ " - kind: featureGroup\n",
+ " spec:\n",
+ " entity: *flights_entity\n",
+ " name: airlines_feature_group\n",
+ " primaryKeys:\n",
+ " - IATA_CODE\n",
+ " inputFeatureDetails:\n",
+ " - featureType: STRING\n",
+ " name: IATA_CODE\n",
+ " orderNumber: 1\n",
+ " - featureType: STRING\n",
+ " name: AIRPORT\n",
+ " orderNumber: 2\n",
+ " - featureType: STRING\n",
+ " name: CITY\n",
+ " orderNumber: 3\n",
+ " - featureType: STRING\n",
+ " name: STATE\n",
+ " orderNumber: 4\n",
+ " - featureType: STRING\n",
+ " name: COUNTRY\n",
+ " orderNumber: 5\n",
+ " - featureType: FLOAT\n",
+ " name: LATITUDE\n",
+ " orderNumber: 6\n",
+ " - featureType: FLOAT\n",
+ " name: LONGITUDE\n",
+ " orderNumber: 7\n",
+ "\n",
+ " - kind: featureGroup\n",
+ " spec:\n",
+ " entity: *flights_entity\n",
+ " name: airport_feature_group\n",
+ " primaryKeys:\n",
+ " - IATA_CODE\n",
+ " inputFeatureDetails:\n",
+ " - featureType: STRING\n",
+ " name: IATA_CODE\n",
+ " orderNumber: 1\n",
+ " - featureType: STRING\n",
+ " name: AIRLINE\n",
+ " orderNumber: 2\n",
+ " dataset:\n",
+ " - kind: dataset\n",
+ " spec:\n",
+ " name: flights_dataset\n",
+ " entity: *flights_entity\n",
+ " description: \"Dataset for flights\"\n",
+ " query: 'SELECT flight.IATA_CODE, flight.AIRPORT FROM flights_feature_group flight'\n",
+ "\"\"\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b988a15a",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "registrar = FeatureStoreRegistrar.from_yaml(yaml_string=feature_store_yaml)\n",
+ "registrar.create()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9fee36b0",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "# 4. References\n",
+ "\n",
+ "- [Feature Store Documentation](https://feature-store-accelerated-data-science.readthedocs.io/en/latest/overview.html)\n",
+ "- [ADS Library Documentation](https://accelerated-data-science.readthedocs.io/en/latest/index.html)\n",
+ "- [Data Science YouTube Videos](https://www.youtube.com/playlist?list=PLKCk3OyNwIzv6CWMhvqSB_8MLJIZdO80L)\n",
+ "- [OCI Data Science Documentation](https://docs.cloud.oracle.com/en-us/iaas/data-science/using/data-science.htm)\n",
+ "- [Oracle Data & AI Blog](https://blogs.oracle.com/datascience/)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a9c7006c",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python [conda env:fspyspark32_p38_cpu_v1]",
+ "language": "python",
+ "name": "conda-env-fspyspark32_p38_cpu_v1-py"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.17"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebook_examples/feature_store_quickstart.ipynb b/notebook_examples/feature_store_quickstart.ipynb
new file mode 100644
index 00000000..26db2c76
--- /dev/null
+++ b/notebook_examples/feature_store_quickstart.ipynb
@@ -0,0 +1,870 @@
+{
+ "cells": [
+ {
+ "cell_type": "raw",
+ "id": "63f5fcad",
+ "metadata": {},
+ "source": [
+ "@notebook{feature_store-quickstart.ipynb,\n",
+ " title: Using feature store for feature ingestion and feature querying,\n",
+ " summary: Introduction to the Oracle Cloud Infrastructure Feature Store.Use feature store for feature ingestion and feature querying,\n",
+ " developed_on: fspyspark32_p38_cpu_v1,\n",
+ " keywords: feature store,\n",
+ " license: Universal Permissive License v 1.0\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e4664bc7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!odsc conda install -s fspyspark32_p38_cpu_v1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "60881fb2",
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Upgrade Oracle ADS to pick up latest features and maintain compatibility with Oracle Cloud Infrastructure.\n",
+ "!pip install --pre --no-deps oracle-ads==2.9.0rc0"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "526f6c48",
+ "metadata": {},
+ "source": [
+ "Oracle Data Science service sample notebook.\n",
+ "\n",
+ "Copyright (c) 2022, 2023 Oracle, Inc. All rights reserved. Licensed under the [Universal Permissive License v 1.0](https://oss.oracle.com/licenses/upl).\n",
+ "\n",
+ "***\n",
+ "\n",
+ "# Feature store quickstart\n",
+ "\n",
+ "by the Oracle Cloud Infrastructure Data Science Service.
\n", + "\n", + "---\n", + "# Overview:\n", + "---\n", + "Managing many datasets, datasources and transformations for machine learning is complex and costly. Poorly cleaned data, data issues, bugs in transformations, data drift and training serving skew all lead to increased model development time and worse model performance. Feature store can be used to solve many of the problems becuase it provides a centralised way to transform and access data for training and serving time. Feature store helps define a standardised pipeline for ingestion of data and querying of data.This notebook demonstrates how to use feature store using a notebook spark session.\n", + "\n", + "Compatible conda pack: [PySpark 3.2 and Feature Store](https://docs.oracle.com/iaas/data-science/using/conda-pyspark-fam.htm) for CPU on Python 3.8 (version 1.0)\n", + "\n", + "## Contents:\n", + "\n", + "- 1. Introduction\n", + "- 2. Pre-requisites to Running this Notebook\n", + " - 2.1 Setup\n", + " - 2.2 Policies\n", + " - 2.3 Authentication\n", + " - 2.4 Variables\n", + "- 3. Feature store quickstart using APIs\n", + " - 3.1 Exploration of data\n", + " - 3.2 Create feature store logical entities\n", + " - 3.2.1 Feature store\n", + " - 3.2.2 Entity\n", + " - 3.2.3 Transformation\n", + " - 3.2.4 Feature group \n", + " - 3.3 Explore feature groups\n", + " - 3.4 Create dataset\n", + " - 3.3 Explore dataset\n", + " - 4. Feature store quickstart using YAML\n", + " - 5. References\n", + "\n", + "---\n", + "\n", + "**Important:**\n", + "\n", + "Placeholder text for required values are surrounded by angle brackets that must be removed when adding the indicated content. For example, when adding a database name to `database_name = \"\n", + "To setup authentication use the ```ads.set_auth(\"resource_principal\")``` or ```ads.set_auth(\"api_key\")```. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dae3ada6", + "metadata": {}, + "outputs": [], + "source": [ + "import ads\n", + "ads.set_auth(auth=\"resource_principal\", client_kwargs={\"fs_service_endpoint\": \"http://{api_gateway}/20230101\"})" + ] + }, + { + "cell_type": "markdown", + "id": "e05054be", + "metadata": {}, + "source": [ + "\n", + "### 2.4. Variables\n", + "To run this notebook, you must provide some information about your tenancy configuration. To create and run a feature store, you must specify a `
by the Oracle Cloud Infrastructure Data Science Service.
\n", + "\n", + "---\n", + "# Overview:\n", + "---\n", + "Managing many datasets, data sources and transformations for machine learning is complex and costly. Poorly cleaned data, data issues, bugs in transformations, data drift, and training serving skew all lead to increased model development time and poor model performance. Feature store can be used to solve many of the problems becuase it provides a centralised way to transform and access data for training and serving time. Feature store helps define a standardised pipeline for ingestion of data and querying of data. This notebook shows how schema enforcement and schema evolution are carried out in Feature Store\n", + "\n", + "Compatible conda pack: [PySpark 3.2 and Feature store](https://docs.oracle.com/iaas/data-science/using/conda-pyspark-fam.htm) for CPU on Python 3.8\n", + "\n", + "\n",
+ " ![](\"https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/_images/overview-roles.png\")
\n",
+ "
\n",
+ "\n",
+ "## Contents:\n",
+ "\n",
+ "- 1. Introduction\n",
+ "- 2. Pre-requisites to Running this Notebook\n",
+ " - 2.1. Setup\n",
+ " - 2.2. Policies\n",
+ " - 2.3. Authentication\n",
+ " - 2.4. Variables\n",
+ "- 3. Schema enforcement and schema evolution\n",
+ " - 3.1. Exploration of data in feature store\n",
+ " - 3.2. Create feature store logical entities\n",
+ " - 3.3. Schema enforcement\n",
+ " - 3.4. Schema evolution\n",
+ " - 3.5. Ingestion Modes\n",
+ " - 3.5.1. Append\n",
+ " - 3.5.2. Overwrite\n",
+ " - 3.5.3. Upsert\n",
+ " - 3.6. Viewing Feature Group History\n",
+ " - 3.7. Time travel Queries on Feature Group \n",
+ "- 4. References\n",
+ "\n",
+ "---\n",
+ "\n",
+ "**Important:**\n",
+ "\n",
+ "Placeholder text for required values are surrounded by angle brackets that must be removed when adding the indicated content. For example, when adding a database name to `database_name = \"\n",
+ "\n", + "To setup authentication use the ```ads.set_auth(\"resource_principal\")``` or ```ads.set_auth(\"api_key\")```." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24963829", + "metadata": { + "ExecuteTime": { + "start_time": "2023-05-24T08:26:08.577504Z" + }, + "is_executing": true, + "pycharm": { + "is_executing": true + } + }, + "outputs": [], + "source": [ + "import ads\n", + "ads.set_auth(auth=\"resource_principal\", client_kwargs={\"fs_service_endpoint\": \"https://{api_gateway}/20230101\"})" + ] + }, + { + "cell_type": "markdown", + "id": "b0c5e0fd", + "metadata": {}, + "source": [ + "\n", + "### 2.4. Variables\n", + "To run this notebook, you must provide some information about your tenancy configuration. To create and run a feature store, you must specify a `
\n",
+ " ![](\"https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/_images/feature_store_demo.jpg\")
\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f43e2ef0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "flights_df = pd.read_csv(\"https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/data/flights/flights.csv\")[['YEAR', 'MONTH', 'DAY', 'DAY_OF_WEEK', 'AIRLINE', 'FLIGHT_NUMBER', 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT']]\n",
+ "flights_df = flights_df.head(100)\n",
+ "flights_df.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "82430a0d",
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [],
+ "source": [
+ "columns = ['IATA_CODE', 'AIRPORT', 'CITY', 'STATE', 'LATITUDE', 'LONGITUDE']\n",
+ "airports_df = pd.read_csv(\"https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/data/flights/airports.csv\")[columns]\n",
+ "airports_df.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "143c3b29",
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [],
+ "source": [
+ "airlines_df = pd.read_csv(\"https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/data/flights/airlines.csv\")\n",
+ "airlines_df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "00083134",
+ "metadata": {},
+ "source": [
+ "\n",
+ "### 3.2. Create feature store logical entities"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4f99ae87",
+ "metadata": {},
+ "source": [
+ "#### 3.2.1. Feature Store\n",
+ "Feature store is the top level entity for feature store service"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ca0b8bfd",
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_store_resource = (\n",
+ " FeatureStore().\n",
+ " with_description(\"Data consisting of flights\").\n",
+ " with_compartment_id(compartment_id).\n",
+ " with_display_name(\"flights details\").\n",
+ " with_offline_config(metastore_id=metastore_id)\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9704fa85",
+ "metadata": {},
+ "source": [
+ "\n",
+ "##### Create Feature Store\n",
+ "\n",
+ "Call the ```.create()``` method of the Feature store instance to create a feature store."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "de4b205d",
+ "metadata": {
+ "pycharm": {
+ "is_executing": true
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_store = feature_store_resource.create()\n",
+ "feature_store"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "473f6677",
+ "metadata": {},
+ "source": [
+ "#### 3.2.2. Entity\n",
+ "An entity is a group of semantically related features."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3dcf22bf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "entity = feature_store.create_entity(\n",
+ " display_name=\"Flight details schema evolution/enforcement\",\n",
+ " description=\"description for flight details\"\n",
+ ")\n",
+ "entity"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "80c9c3be",
+ "metadata": {},
+ "source": [
+ "\n",
+ "#### 3.2.3. Feature Group\n",
+ "\n",
+ "Create feature group for airport"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "970161e6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from great_expectations.core import ExpectationSuite, ExpectationConfiguration\n",
+ "\n",
+ "expectation_suite_airports = ExpectationSuite(\n",
+ " expectation_suite_name=\"test_airports_df\"\n",
+ ")\n",
+ "expectation_suite_airports.add_expectation(\n",
+ " ExpectationConfiguration(\n",
+ " expectation_type=\"expect_column_values_to_not_be_null\",\n",
+ " kwargs={\"column\": \"IATA_CODE\"},\n",
+ " )\n",
+ ")\n",
+ "expectation_suite_airports.add_expectation(\n",
+ " ExpectationConfiguration(\n",
+ " expectation_type=\"expect_column_values_to_be_between\",\n",
+ " kwargs={\"column\": \"LATITUDE\", \"min_value\": -1.0, \"max_value\": 1.0},\n",
+ " )\n",
+ ")\n",
+ "\n",
+ "expectation_suite_airports.add_expectation(\n",
+ " ExpectationConfiguration(\n",
+ " expectation_type=\"expect_column_values_to_be_between\",\n",
+ " kwargs={\"column\": \"LONGITUDE\", \"min_value\": -1.0, \"max_value\": 1.0},\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bc323dd5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airports = (\n",
+ " FeatureGroup()\n",
+ " .with_feature_store_id(feature_store.id)\n",
+ " .with_primary_keys([\"IATA_CODE\"])\n",
+ " .with_name(\"airport_feature_group\")\n",
+ " .with_entity_id(entity.id)\n",
+ " .with_compartment_id(compartment_id)\n",
+ " .with_schema_details_from_dataframe(airports_df)\n",
+ " .with_expectation_suite(\n",
+ " expectation_suite=expectation_suite_airports,\n",
+ " expectation_type=ExpectationType.LENIENT,\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2e437522",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "feature_group_airports.create()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3fc98501",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airports.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ec22e95c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airports.materialise(airports_df)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ed7c012e",
+ "metadata": {},
+ "source": [
+ "\n",
+ "### 3.3. Schema enforcement\n",
+ "\n",
+ "Schema enforcement, also known as schema validation, is a safeguard in Delta Lake that ensures data quality by rejecting writes to a table that do not match the table's schema. For example, a front desk manager at a busy restaurant that only accepts reservations, the schema enforcement checks to see whether each column in the data inserted into the table is in the list of expected columns. Meaning each one has a \"reservation\", and rejects any writes with columns that aren't on the list."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "eef566cc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "columns = ['IATA_CODE', 'AIRPORT', 'CITY', 'STATE', 'LATITUDE', 'LONGITUDE', 'COUNTRY']\n",
+ "airports_df = pd.read_csv(\"https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/data/flights/airports.csv\")[columns]\n",
+ "airports_df.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d620cedf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airports.with_schema_details_from_dataframe(airports_df)\n",
+ "feature_group_airports.update()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c62e82c9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airports.materialise(airports_df)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e8cbab63",
+ "metadata": {},
+ "source": [
+ "\n",
+ "### 3.4. Schema evolution\n",
+ "\n",
+ "Schema evolution allows you to change a table's current schema to accommodate data that is changing over time. Typically, it's used when performing an append or overwrite operation to automatically adapt the schema to include one or more new columns."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d69f3378",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from ads.feature_store.feature_option_details import FeatureOptionDetails\n",
+ "feature_option_details = FeatureOptionDetails().with_feature_option_write_config_details(merge_schema=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "794597c9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airports.materialise(\n",
+ " input_dataframe=airports_df,\n",
+ " feature_option_details=feature_option_details\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bcaa552c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airports"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b4eca757",
+ "metadata": {},
+ "source": [
+ "\n",
+ "### 3.5. Ingestion modes"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7ae4c0e5",
+ "metadata": {},
+ "source": [
+ "\n",
+ "#### 3.5.1. Append\n",
+ "\n",
+ "In ``append`` mode, new data is added to the existing table. If the table already exists, the new data is appended to it, extending the dataset. This mode is suitable for scenarios where you want to continuously add new records without modifying or deleting existing data. It preserves the existing data and only appends the new data to the end of the table."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "24acf8e6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from ads.feature_store.feature_group_job import IngestionMode\n",
+ "feature_group_airports.materialise(airports_df, ingestion_mode=IngestionMode.APPEND)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "91ed98b3",
+ "metadata": {},
+ "source": [
+ "\n",
+ "#### 3.5.2. Overwrite\n",
+ "In ``overwrite`` mode, the existing table is replaced entirely with the new data being saved. If the table already exists, it is dropped and a new table is created with the new data. This mode is useful when you want to completely refresh the data in the table with the latest data and discard all previous records."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "690a6136",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from ads.feature_store.feature_group_job import IngestionMode\n",
+ "feature_group_airports.materialise(airports_df, ingestion_mode=IngestionMode.OVERWRITE)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1866795c",
+ "metadata": {},
+ "source": [
+ "\n",
+ "#### 3.5.3. Upsert\n",
+ "``Upsert`` mode (merge mode) is used to update existing records in the table based on a primary key or a specified condition. If a record with the same key exists, it is updated with the new data. Otherwise, a new record is inserted. This mode is useful for maintaining and synchronizing data between the source and destination tables while avoiding duplicates."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b2ddd858",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from ads.feature_store.feature_group_job import IngestionMode\n",
+ "feature_group_airports.materialise(airports_df, ingestion_mode=IngestionMode.UPSERT)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5495e3cf",
+ "metadata": {},
+ "source": [
+ "\n",
+ "### 3.6. Viewing Feature Group History\n",
+ "You can call the ``history()`` method of the FeatureGroup instance to show history of the feature group."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "feb1762f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airports.history().toPandas()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1fb57d2c",
+ "metadata": {},
+ "source": [
+ "\n",
+ "### 3.7. Time travel Queries on Feature Group\n",
+ "\n",
+ "You can call the ``as_of()`` method of the FeatureGroup instance to get specified point in time and time traveled data.\n",
+ "The ``.as_of()`` method takes the following optional parameter:\n",
+ "\n",
+ "- commit_timestamp: date-time. Commit timestamp for feature group\n",
+ "- version_number: int. Version number for feature group"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1ec4cc00",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airports.as_of(version_number = 0).show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4ce013ab",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "feature_group_airports.as_of(version_number = 1).show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1abcc338",
+ "metadata": {},
+ "source": [
+ "\n",
+ "# 4. References\n",
+ "- [Feature Store Documentation](https://feature-store-accelerated-data-science.readthedocs.io/en/latest/overview.html)\n",
+ "- [ADS Library Documentation](https://accelerated-data-science.readthedocs.io/en/latest/index.html)\n",
+ "- [Data Science YouTube Videos](https://www.youtube.com/playlist?list=PLKCk3OyNwIzv6CWMhvqSB_8MLJIZdO80L)\n",
+ "- [OCI Data Science Documentation](https://docs.cloud.oracle.com/en-us/iaas/data-science/using/data-science.htm)\n",
+ "- [Oracle Data & AI Blog](https://blogs.oracle.com/datascience/)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1157840c",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python [conda env:fspyspark32_p38_cpu_v1]",
+ "language": "python",
+ "name": "conda-env-fspyspark32_p38_cpu_v1-py"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.17"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebook_examples/feature_store_spark_magic.ipynb b/notebook_examples/feature_store_spark_magic.ipynb
new file mode 100644
index 00000000..98d2cb40
--- /dev/null
+++ b/notebook_examples/feature_store_spark_magic.ipynb
@@ -0,0 +1,687 @@
+{
+ "cells": [
+ {
+ "cell_type": "raw",
+ "id": "8ce4f16c",
+ "metadata": {},
+ "source": [
+ "qweews@notebook{feature_store_spark_magic.ipynb,\n",
+ " title: Data Flow Studio : Big Data Operations in Feature Store.,\n",
+ " summary: Run Feature Store on interactive Spark workloads on a long lasting Data Flow Cluster.,\n",
+ " developed_on: fspyspark32_p38_cpu_v1,\n",
+ " keywords: feature store, querying,spark magic,data flow\n",
+ " license: Universal Permissive License v 1.0\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "55da3909",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!odsc conda install -s fspyspark32_p38_cpu_v1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5c24e9f2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Upgrade Oracle ADS to pick up the latest preview version to maintain compatibility with Oracle Cloud Infrastructure.\n",
+ "!pip install --pre --no-deps oracle-ads==2.9.0rc0"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "fe5598bd",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "Oracle Data Science service sample notebook.\n",
+ "\n",
+ "Copyright (c) 2022, 2023 Oracle, Inc. All rights reserved. Licensed under the [Universal Permissive License v 1.0](https://oss.oracle.com/licenses/upl).\n",
+ "***\n",
+ "\n",
+ "# Data Flow Studio: Big Data Operations in Feature Store\n",
+ "\n",
+ "by the Oracle Cloud Infrastructure Data Science Service.
\n", + "\n", + "---\n", + "# Overview:\n", + "\n", + "This notebook demonstrates how to run Feature Store on interactive Spark workloads on a long lasting [Oracle Cloud Infrastructure Data Flow](https://docs.oracle.com/en-us/iaas/data-flow/using/home.htm) cluster through [Apache Livy](https://livy.apache.org/) integration. **Data Flow Spark Magic** is used for interactively working with remote Spark clusters using Livy (a Spark REST server) in Jupyter notebooks. Data Flow Spark Magic includes a set of magic commands for interactively running Spark code.\n", + "\n", + "\n", + "\n", + "## Contents:\n", + "\n", + "- 1. Introduction\n", + "- 2. 2. Pre-requisites to Running this Notebook\n", + " - 2.1 Policies\n", + " - 2.2 Helpers\n", + " - 2.3 Authentication\n", + " - 2.4 Variables\n", + "- 3. Data Flow Spark Magic\n", + " - 3.1. Load Spark Magic Commands and Getting Help\n", + " - 3.2. Create DataFlow Session\n", + " - 3.3. Data exploration\n", + " - 3.4. Create Feature Store Logical Entities\n", + " - 3.4.1 Creating a feature store\n", + " - 3.4.2 Creating an entity\n", + " - 3.4.3 Creating a feature group\n", + " - 3.4.4 Materialising a Feature Group\n", + " - 3.4.5 Querying a Feature group\n", + "- 4. References\n", + "\n", + "---\n", + "\n", + "\n", + "Compatible conda pack: [PySpark 3.2 and Feature Store](https://docs.oracle.com/iaas/data-science/using/conda-pyspark-fam.htm) for CPU on Python 3.8 (version 1.0)\n" + ] + }, + { + "cell_type": "markdown", + "id": "cd84d936", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "\n", + "# 1. Introduction\n", + "\n", + "OCI Data Science feature store is a stack-based API solution that's deployed using OCI Resource Manager in your tenancy.\n", + "\n", + "Review the following key terms to understand the Data Science feature store:\n", + "\n", + "\n", + "* **Feature Vector**: Set of feature values for any one primary or identifier key. For example, all or a subset of features of customer id ‘2536’ can be called as one feature vector.\n", + "\n", + "* **Feature**: A feature is an individual measurable property or characteristic of a phenomenon being observed.\n", + "\n", + "* **Entity**: An entity is a group of semantically related features. The first step a consumer of features would typically do when accessing the feature store service is to list the entities and the entities associated features. Or, an entity is an object or concept that is described by its features. Examples of entities are customer, product, transaction, review, image, document, and so on.\n", + "\n", + "* **Feature Group**: A feature group in a feature store is a collection of related features that are often used together in machine learning (ML) models. It serves as an organizational unit within the feature store for you to manage, version, and share features across different ML projects. By organizing features into groups, data scientists and ML engineers can efficiently discover, reuse, and collaborate on features reducing the redundant work and ensuring consistency in feature engineering.\n", + "\n", + "* **Feature Group Job**: A feature group job is the processing instance of a feature group. Each feature group job includes validation and statistics results.\n", + "\n", + "* **Dataset**: A dataset is a collection of features that are used together to either train a model or perform model inference.\n", + "\n", + "* **Dataset Job**: A dataset job is the processing instance of a dataset. Each dataset job includes validation and statistics results." + ] + }, + { + "cell_type": "markdown", + "id": "76acf33b", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "\n", + "# 2. Pre-requisites to Running this Notebook\n", + "\n", + "Data Flow Sessions are accessible using the PySpark 3.2 and Feature Store Python 3.8 (fspyspark32_p38_cpu_v1) conda environment.\n", + "\n", + "The [Data Catalog Hive Metastore](https://docs.oracle.com/en-us/iaas/data-catalog/using/metastore.htm) provides schema definitions for objects in structured and unstructured data assets. The Metastore is the central metadata repository to understand tables backed by files on object storage. You can customize `fs_pyspark32_p38_cpu_v1`, publish it, and use it as a runtime environment for a Data Flow session cluster. The metastore id of hive metastore is tied to feature store construct of feature store service." + ] + }, + { + "cell_type": "markdown", + "id": "f1e2b6a1", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "\n", + "## 2.1. Policies\n", + "This section covers the creation of dynamic groups and policies needed to use the service.\n", + "\n", + "* [Data Flow Policies](https://docs.oracle.com/iaas/data-flow/using/policies.htm)\n", + "* [Getting Started with Data Flow](https://docs.oracle.com/iaas/data-flow/using/dfs_getting_started.htm)\n", + "* [About Data Science Policies](https://docs.oracle.com/iaas/data-science/using/policies.htm)\n", + "* [Data Catalog Metastore](https://docs.oracle.com/en-us/iaas/data-catalog/using/metastore.htm)" + ] + }, + { + "cell_type": "markdown", + "id": "2207bfb3", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "\n", + "## 2.2 Helpers\n", + "This helper method is used across the notebook to prepare arguments for the magic commands. This function is particularly useful when you want to pass Python variables as arguments to the spark magic commands." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32894857", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import json\n", + "\n", + "\n", + "def prepare_command(command: dict) -> str:\n", + " \"\"\"Converts dictionary command to the string formatted commands.\"\"\"\n", + " return f\"'{json.dumps(command)}'\"" + ] + }, + { + "cell_type": "markdown", + "id": "4b610391", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "\n", + "## 2.3. Authentication\n", + "The [Oracle Accelerated Data Science SDK (ADS)](https://docs.oracle.com/iaas/tools/ads-sdk/latest/index.html) controls the authentication mechanism with the Data Flow Session Spark cluster.\n", + "To setup authentication use the ```ads.set_auth(\"resource_principal\")``` or ```ads.set_auth(\"api_key\")```. For example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae1080f3", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import ads\n", + "\n", + "ads.set_auth(\"resource_principal\") # Supported values: resource_principal, api_key" + ] + }, + { + "cell_type": "markdown", + "id": "3aa57661", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "\n", + "## 2.4. Variables\n", + "To run this notebook, you must provide some information about your tenancy configuration. To connect to the HIVE metastore, replace `
\n", + "After the extension is activated, you can use the `%help` command to view the list of supported commands." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d61b5fa", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "%load_ext dataflow.magics" + ] + }, + { + "cell_type": "markdown", + "id": "ec076494", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "\n", + "### 3.2. Create DataFlow Session.\n", + "Create a new Data Flow cluster session using the `%create_session` magic command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1aba2243", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "command = prepare_command(\n", + " {\n", + " \"compartmentId\": compartment_id,\n", + " \"displayName\": \"spark_session_via_notebook\",\n", + " \"language\": \"PYTHON\",\n", + " \"sparkVersion\": \"3.2.1\",\n", + " \"numExecutors\": 8,\n", + " \"metastoreId\": metastore_id,\n", + " \"driverShape\": \"VM.Standard2.1\",\n", + " \"executorShape\": \"VM.Standard2.1\",\n", + " \"driverShapeConfig\": {\"ocpus\": 2, \"memoryInGBs\": 16},\n", + " \"executorShapeConfig\": {\"ocpus\": 2, \"memoryInGBs\": 16},\n", + " \"type\": \"SESSION\",\n", + " \"logsBucketUri\": logs_bucket_uri,\n", + " \"configuration\": {\n", + " \"spark.archives\": custom_conda_environment_uri,\n", + " \"fs.oci.client.hostname\": \"https://objectstorage.us-ashburn-1.oraclecloud.com\"\n", + " },\n", + " }\n", + ")\n", + "\n", + "%create_session -l python -c $command" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0aad36e2", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "%%spark\n", + "from great_expectations.core import ExpectationSuite, ExpectationConfiguration\n", + "\n", + "import ads\n", + "from ads.feature_store.entity import Entity\n", + "from ads.feature_store.feature_group import FeatureGroup\n", + "from ads.feature_store.feature_group_expectation import ExpectationType\n", + "from ads.feature_store.feature_store import FeatureStore\n", + "from ads.feature_store.input_feature_detail import FeatureDetail, FeatureType\n", + "from ads.feature_store.statistics_config import StatisticsConfig\n", + "from ads.feature_store.transformation import TransformationMode\n", + "import os\n", + "\n", + "# Set the Authentications for the feature store operations\n", + "ads.set_auth(auth=\"resource_principal\", client_kwargs={\"fs_service_endpoint\": \"https://{api_gateway}/20230101\"})\n", + "\n", + "# Variables\n", + "compartment_id = \"