sdk/python/sample/demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Install the _Kubeflow-metadata_ library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To use the latest publish `kubeflow-metadata` library, you can run:\n",
    "!pip install kubeflow-metadata --user\n",
    "# Install other packages:\n",
    "!pip install pandas --user\n",
    "# Then restart the Notebook kernel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "from kubeflow.metadata import metadata\n",
    "from datetime import datetime\n",
    "from uuid import uuid4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "METADATA_STORE_HOST = \"metadata-grpc-service.kubeflow\" # default DNS of Kubeflow Metadata gRPC serivce.\n",
    "METADATA_STORE_PORT = 8080"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ws1 = metadata.Workspace(\n",
    "    # Connect to metadata service in namespace kubeflow in k8s cluster.\n",
    "    store=metadata.Store(grpc_host=METADATA_STORE_HOST, grpc_port=METADATA_STORE_PORT),\n",
    "    name=\"workspace_1\",\n",
    "    description=\"a workspace for testing\",\n",
    "    labels={\"n1\": \"v1\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a run in a workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = metadata.Run(\n",
    "    workspace=ws1,\n",
    "    name=\"run-\" + datetime.utcnow().isoformat(\"T\") ,\n",
    "    description=\"a run in ws_1\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create an execution in a run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "An execution was created with id 290\n"
    }
   ],
   "source": [
    "exec = metadata.Execution(\n",
    "    name = \"execution\" + datetime.utcnow().isoformat(\"T\") ,\n",
    "    workspace=ws1,\n",
    "    run=r,\n",
    "    description=\"execution example\",\n",
    ")\n",
    "print(\"An execution was created with id %s\" % exec.id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log a data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Data set id is 171 with version 'data_set_version_cbebc757-0d76-4e1e-bbd9-02b065e4c3ea'\n"
    }
   ],
   "source": [
    "date_set_version = \"data_set_version_\" + str(uuid4())\n",
    "data_set = exec.log_input(\n",
    "        metadata.DataSet(\n",
    "            description=\"an example data\",\n",
    "            name=\"mytable-dump\",\n",
    "            owner=\"owner@my-company.org\",\n",
    "            uri=\"file://path/to/dataset\",\n",
    "            version=date_set_version,\n",
    "            query=\"SELECT * FROM mytable\"))\n",
    "print(\"Data set id is {0.id} with version '{0.version}'\".format(data_set))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "kubeflow.metadata.metadata.Model(workspace=None, name='MNIST', description='model to recognize handwritten digits', owner='someone@kubeflow.org', uri='gcs://my-bucket/mnist', version='model_version_50b419e2-af69-4c0e-a251-78246d4c0578', model_type='neural network', training_framework={'name': 'tensorflow', 'version': 'v1.0'}, hyperparameters={'learning_rate': 0.5, 'layers': [10, 3, 1], 'early_stop': True}, labels={'mylabel': 'l1'}, id=172, create_time='2019-12-04T00:44:49.444411Z', kwargs={})\n\nModel id is 172 and version is model_version_50b419e2-af69-4c0e-a251-78246d4c0578\n"
    }
   ],
   "source": [
    "model_version = \"model_version_\" + str(uuid4())\n",
    "model = exec.log_output(\n",
    "    metadata.Model(\n",
    "            name=\"MNIST\",\n",
    "            description=\"model to recognize handwritten digits\",\n",
    "            owner=\"someone@kubeflow.org\",\n",
    "            uri=\"gcs://my-bucket/mnist\",\n",
    "            model_type=\"neural network\",\n",
    "            training_framework={\n",
    "                \"name\": \"tensorflow\",\n",
    "                \"version\": \"v1.0\"\n",
    "            },\n",
    "            hyperparameters={\n",
    "                \"learning_rate\": 0.5,\n",
    "                \"layers\": [10, 3, 1],\n",
    "                \"early_stop\": True\n",
    "            },\n",
    "            version=model_version,\n",
    "            labels={\"mylabel\": \"l1\"}))\n",
    "print(model)\n",
    "print(\"\\nModel id is {0.id} and version is {0.version}\".format(model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log the evaluation of a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Metrics id is 173\n"
    }
   ],
   "source": [
    "metrics = exec.log_output(\n",
    "    metadata.Metrics(\n",
    "            name=\"MNIST-evaluation\",\n",
    "            description=\"validating the MNIST model to recognize handwritten digits\",\n",
    "            owner=\"someone@kubeflow.org\",\n",
    "            uri=\"gcs://my-bucket/mnist-eval.csv\",\n",
    "            data_set_id=str(data_set.id),\n",
    "            model_id=str(model.id),\n",
    "            metrics_type=metadata.Metrics.VALIDATION,\n",
    "            values={\"accuracy\": 0.95},\n",
    "            labels={\"mylabel\": \"l1\"}))\n",
    "print(\"Metrics id is %s\" % metrics.id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add Metadata for serving the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found the mode with id 172 and version 'model_version_50b419e2-af69-4c0e-a251-78246d4c0578'.\n"
     ]
    }
   ],
   "source": [
    "serving_application = metadata.Execution(\n",
    "    name=\"serving model\",\n",
    "    workspace=ws1,\n",
    "    description=\"an execution to represent model serving component\",\n",
    ")\n",
    "# Noticed we use model name, version, uri to uniquely identify existing model.\n",
    "served_model = metadata.Model(\n",
    "    name=\"MNIST\",\n",
    "    uri=\"gcs://my-bucket/mnist\",\n",
    "    version=model.version,\n",
    ")\n",
    "m=serving_application.log_input(served_model)\n",
    "print(\"Found the mode with id {0.id} and version '{0.version}'.\".format(m))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### List all models in the workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>workspace</th>\n",
       "      <th>run</th>\n",
       "      <th>create_time</th>\n",
       "      <th>version</th>\n",
       "      <th>owner</th>\n",
       "      <th>description</th>\n",
       "      <th>name</th>\n",
       "      <th>model_type</th>\n",
       "      <th>uri</th>\n",
       "      <th>training_framework</th>\n",
       "      <th>hyperparameters</th>\n",
       "      <th>labels</th>\n",
       "      <th>kwargs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>152</td>\n",
       "      <td>workspace_1</td>\n",
       "      <td>run-2019-12-04T00:10:29.797245</td>\n",
       "      <td>2019-12-04T00:10:30.516009Z</td>\n",
       "      <td>model_version_68fd3ccd-bb61-496a-b6b3-b9a7ab7a...</td>\n",
       "      <td>someone@kubeflow.org</td>\n",
       "      <td>model to recognize handwritten digits</td>\n",
       "      <td>MNIST</td>\n",
       "      <td>neural network</td>\n",
       "      <td>gcs://my-bucket/mnist</td>\n",
       "      <td>{'name': 'tensorflow', 'version': 'v1.0'}</td>\n",
       "      <td>{'learning_rate': 0.5, 'layers': [10, 3, 1], '...</td>\n",
       "      <td>{'mylabel': 'l1'}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>154</td>\n",
       "      <td>workspace_1</td>\n",
       "      <td>run-2019-12-04T00:10:29.797245</td>\n",
       "      <td>2019-12-04T00:14:44.401822Z</td>\n",
       "      <td>model_version_4757fb07-20f5-4443-8000-ba0ad5ca...</td>\n",
       "      <td>someone@kubeflow.org</td>\n",
       "      <td>model to recognize handwritten digits</td>\n",
       "      <td>MNIST</td>\n",
       "      <td>neural network</td>\n",
       "      <td>gcs://my-bucket/mnist</td>\n",
       "      <td>{'name': 'tensorflow', 'version': 'v1.0'}</td>\n",
       "      <td>{'learning_rate': 0.5, 'layers': [10, 3, 1], '...</td>\n",
       "      <td>{'mylabel': 'l1'}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>156</td>\n",
       "      <td>workspace_1</td>\n",
       "      <td>run-2019-12-04T00:10:29.797245</td>\n",
       "      <td>2019-12-04T00:14:56.893195Z</td>\n",
       "      <td>model_version_aa162457-85e4-473d-9f20-8fe72637...</td>\n",
       "      <td>someone@kubeflow.org</td>\n",
       "      <td>model to recognize handwritten digits</td>\n",
       "      <td>MNIST</td>\n",
       "      <td>neural network</td>\n",
       "      <td>gcs://my-bucket/mnist</td>\n",
       "      <td>{'name': 'tensorflow', 'version': 'v1.0'}</td>\n",
       "      <td>{'learning_rate': 0.5, 'layers': [10, 3, 1], '...</td>\n",
       "      <td>{'mylabel': 'l1'}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>160</td>\n",
       "      <td>workspace_1</td>\n",
       "      <td>run-2019-12-04T00:15:41.059380</td>\n",
       "      <td>2019-12-04T00:15:41.766812Z</td>\n",
       "      <td>model_version_7af7615c-ec8d-4e30-b765-14dd1d81...</td>\n",
       "      <td>someone@kubeflow.org</td>\n",
       "      <td>model to recognize handwritten digits</td>\n",
       "      <td>MNIST</td>\n",
       "      <td>neural network</td>\n",
       "      <td>gcs://my-bucket/mnist</td>\n",
       "      <td>{'name': 'tensorflow', 'version': 'v1.0'}</td>\n",
       "      <td>{'learning_rate': 0.5, 'layers': [10, 3, 1], '...</td>\n",
       "      <td>{'mylabel': 'l1'}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>163</td>\n",
       "      <td>workspace_1</td>\n",
       "      <td>run-2019-12-04T00:43:16.177845</td>\n",
       "      <td>2019-12-04T00:43:16.907629Z</td>\n",
       "      <td>model_version_3d8540f1-f74f-4660-83dc-2b5a886d...</td>\n",
       "      <td>someone@kubeflow.org</td>\n",
       "      <td>model to recognize handwritten digits</td>\n",
       "      <td>MNIST</td>\n",
       "      <td>neural network</td>\n",
       "      <td>gcs://my-bucket/mnist</td>\n",
       "      <td>{'name': 'tensorflow', 'version': 'v1.0'}</td>\n",
       "      <td>{'learning_rate': 0.5, 'layers': [10, 3, 1], '...</td>\n",
       "      <td>{'mylabel': 'l1'}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>166</td>\n",
       "      <td>workspace_1</td>\n",
       "      <td>run-2019-12-04T00:43:23.426877</td>\n",
       "      <td>2019-12-04T00:43:24.138877Z</td>\n",
       "      <td>model_version_9987cd2f-d354-49b5-955e-952f21ff...</td>\n",
       "      <td>someone@kubeflow.org</td>\n",
       "      <td>model to recognize handwritten digits</td>\n",
       "      <td>MNIST</td>\n",
       "      <td>neural network</td>\n",
       "      <td>gcs://my-bucket/mnist</td>\n",
       "      <td>{'name': 'tensorflow', 'version': 'v1.0'}</td>\n",
       "      <td>{'learning_rate': 0.5, 'layers': [10, 3, 1], '...</td>\n",
       "      <td>{'mylabel': 'l1'}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>169</td>\n",
       "      <td>workspace_1</td>\n",
       "      <td>run-2019-12-04T00:43:47.136103</td>\n",
       "      <td>2019-12-04T00:43:47.851783Z</td>\n",
       "      <td>model_version_123141d7-a3ce-4ac4-ade9-e750ad18...</td>\n",
       "      <td>someone@kubeflow.org</td>\n",
       "      <td>model to recognize handwritten digits</td>\n",
       "      <td>MNIST</td>\n",
       "      <td>neural network</td>\n",
       "      <td>gcs://my-bucket/mnist</td>\n",
       "      <td>{'name': 'tensorflow', 'version': 'v1.0'}</td>\n",
       "      <td>{'learning_rate': 0.5, 'layers': [10, 3, 1], '...</td>\n",
       "      <td>{'mylabel': 'l1'}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>172</td>\n",
       "      <td>workspace_1</td>\n",
       "      <td>run-2019-12-04T00:44:48.687078</td>\n",
       "      <td>2019-12-04T00:44:49.444411Z</td>\n",
       "      <td>model_version_50b419e2-af69-4c0e-a251-78246d4c...</td>\n",
       "      <td>someone@kubeflow.org</td>\n",
       "      <td>model to recognize handwritten digits</td>\n",
       "      <td>MNIST</td>\n",
       "      <td>neural network</td>\n",
       "      <td>gcs://my-bucket/mnist</td>\n",
       "      <td>{'name': 'tensorflow', 'version': 'v1.0'}</td>\n",
       "      <td>{'learning_rate': 0.5, 'layers': [10, 3, 1], '...</td>\n",
       "      <td>{'mylabel': 'l1'}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id    workspace                             run  \\\n",
       "0  152  workspace_1  run-2019-12-04T00:10:29.797245   \n",
       "1  154  workspace_1  run-2019-12-04T00:10:29.797245   \n",
       "2  156  workspace_1  run-2019-12-04T00:10:29.797245   \n",
       "3  160  workspace_1  run-2019-12-04T00:15:41.059380   \n",
       "4  163  workspace_1  run-2019-12-04T00:43:16.177845   \n",
       "5  166  workspace_1  run-2019-12-04T00:43:23.426877   \n",
       "6  169  workspace_1  run-2019-12-04T00:43:47.136103   \n",
       "7  172  workspace_1  run-2019-12-04T00:44:48.687078   \n",
       "\n",
       "                   create_time  \\\n",
       "0  2019-12-04T00:10:30.516009Z   \n",
       "1  2019-12-04T00:14:44.401822Z   \n",
       "2  2019-12-04T00:14:56.893195Z   \n",
       "3  2019-12-04T00:15:41.766812Z   \n",
       "4  2019-12-04T00:43:16.907629Z   \n",
       "5  2019-12-04T00:43:24.138877Z   \n",
       "6  2019-12-04T00:43:47.851783Z   \n",
       "7  2019-12-04T00:44:49.444411Z   \n",
       "\n",
       "                                             version                 owner  \\\n",
       "0  model_version_68fd3ccd-bb61-496a-b6b3-b9a7ab7a...  someone@kubeflow.org   \n",
       "1  model_version_4757fb07-20f5-4443-8000-ba0ad5ca...  someone@kubeflow.org   \n",
       "2  model_version_aa162457-85e4-473d-9f20-8fe72637...  someone@kubeflow.org   \n",
       "3  model_version_7af7615c-ec8d-4e30-b765-14dd1d81...  someone@kubeflow.org   \n",
       "4  model_version_3d8540f1-f74f-4660-83dc-2b5a886d...  someone@kubeflow.org   \n",
       "5  model_version_9987cd2f-d354-49b5-955e-952f21ff...  someone@kubeflow.org   \n",
       "6  model_version_123141d7-a3ce-4ac4-ade9-e750ad18...  someone@kubeflow.org   \n",
       "7  model_version_50b419e2-af69-4c0e-a251-78246d4c...  someone@kubeflow.org   \n",
       "\n",
       "                             description   name      model_type  \\\n",
       "0  model to recognize handwritten digits  MNIST  neural network   \n",
       "1  model to recognize handwritten digits  MNIST  neural network   \n",
       "2  model to recognize handwritten digits  MNIST  neural network   \n",
       "3  model to recognize handwritten digits  MNIST  neural network   \n",
       "4  model to recognize handwritten digits  MNIST  neural network   \n",
       "5  model to recognize handwritten digits  MNIST  neural network   \n",
       "6  model to recognize handwritten digits  MNIST  neural network   \n",
       "7  model to recognize handwritten digits  MNIST  neural network   \n",
       "\n",
       "                     uri                         training_framework  \\\n",
       "0  gcs://my-bucket/mnist  {'name': 'tensorflow', 'version': 'v1.0'}   \n",
       "1  gcs://my-bucket/mnist  {'name': 'tensorflow', 'version': 'v1.0'}   \n",
       "2  gcs://my-bucket/mnist  {'name': 'tensorflow', 'version': 'v1.0'}   \n",
       "3  gcs://my-bucket/mnist  {'name': 'tensorflow', 'version': 'v1.0'}   \n",
       "4  gcs://my-bucket/mnist  {'name': 'tensorflow', 'version': 'v1.0'}   \n",
       "5  gcs://my-bucket/mnist  {'name': 'tensorflow', 'version': 'v1.0'}   \n",
       "6  gcs://my-bucket/mnist  {'name': 'tensorflow', 'version': 'v1.0'}   \n",
       "7  gcs://my-bucket/mnist  {'name': 'tensorflow', 'version': 'v1.0'}   \n",
       "\n",
       "                                     hyperparameters             labels kwargs  \n",
       "0  {'learning_rate': 0.5, 'layers': [10, 3, 1], '...  {'mylabel': 'l1'}     {}  \n",
       "1  {'learning_rate': 0.5, 'layers': [10, 3, 1], '...  {'mylabel': 'l1'}     {}  \n",
       "2  {'learning_rate': 0.5, 'layers': [10, 3, 1], '...  {'mylabel': 'l1'}     {}  \n",
       "3  {'learning_rate': 0.5, 'layers': [10, 3, 1], '...  {'mylabel': 'l1'}     {}  \n",
       "4  {'learning_rate': 0.5, 'layers': [10, 3, 1], '...  {'mylabel': 'l1'}     {}  \n",
       "5  {'learning_rate': 0.5, 'layers': [10, 3, 1], '...  {'mylabel': 'l1'}     {}  \n",
       "6  {'learning_rate': 0.5, 'layers': [10, 3, 1], '...  {'mylabel': 'l1'}     {}  \n",
       "7  {'learning_rate': 0.5, 'layers': [10, 3, 1], '...  {'mylabel': 'l1'}     {}  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pandas.DataFrame.from_dict(ws1.list(metadata.Model.ARTIFACT_TYPE_NAME))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic Lineage Tracking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Model id is 172\n\nAll executions related to the model are {290, 291}\nAll artifacts related to the training event are {171, 172, 173}\n"
    }
   ],
   "source": [
    "print(\"Model id is %s\\n\" % model.id)\n",
    "    \n",
    "model_events = ws1.store.get_events_by_artifact_ids([model.id])\n",
    "\n",
    "execution_ids = set(e.execution_id for e in model_events)\n",
    "print(\"All executions related to the model are {}\".format(execution_ids))\n",
    "# assert execution_ids == set([serving_application.id, exec.id])\n",
    "\n",
    "trainer_events = ws1.store.get_events_by_execution_ids([exec.id])\n",
    "artifact_ids = set(e.artifact_id for e in trainer_events)\n",
    "print(\"All artifacts related to the training event are {}\".format(artifact_ids))# assert artifact_ids == set([model.id, metrics.id, data_set.id])"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Tags",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}