notebooks/labeled_stream_creator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Labeled Stream Creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nuclio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%nuclio cmd -c python -m pip install v3io --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%nuclio: setting 'METRICS_TABLE' environment variable\n",
      "%nuclio: setting 'PREDICTIONS_TABLE' environment variable\n",
      "%nuclio: setting 'OUTPUT_STREAM' environment variable\n",
      "%nuclio: setting 'prediction_col' environment variable\n",
      "%nuclio: setting 'label_col' environment variable\n",
      "%nuclio: setting 'output_stream_shards' environment variable\n"
     ]
    }
   ],
   "source": [
    "%%nuclio env\n",
    "METRICS_TABLE = /User/demo-network-operations/streaming/metrics\n",
    "PREDICTIONS_TABLE = /User/demo-network-operations/streaming/predictions\n",
    "OUTPUT_STREAM = /users/admin/demo-network-operations/streaming/labels_stream_v1\n",
    "prediction_col = predictions\n",
    "label_col = is_error\n",
    "output_stream_shards = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nuclio: start-code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import json\n",
    "import v3io\n",
    "import v3io.dataplane\n",
    "import socket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_path(mntpath=''):\n",
    "    if mntpath[0] == '/':\n",
    "        mntpath = mntpath[1:]\n",
    "    paths = mntpath.split('/')\n",
    "    container = paths[0]\n",
    "    subpath = ''\n",
    "    if len(paths) > 1:\n",
    "        subpath = mntpath[len(container):]\n",
    "    return container, subpath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_stream(context, path, shards=1):\n",
    "    # create a stream w/8 shards\n",
    "    container, stream_path = split_path(path)\n",
    "    context.logger.info(f'Creating stream in Container: {container} & Path {stream_path}')\n",
    "    response = context.v3io_client.create_stream(container=container,\n",
    "                                        path=stream_path, \n",
    "                                        shard_count=shards,\n",
    "                                        raise_for_status=v3io.dataplane.RaiseForStatus.never)\n",
    "    response.raise_for_status([409, 204])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def push_to_stream(context, stream_path, data):\n",
    "    def restructure_stream_event(context, event):\n",
    "        instances = [dict()]\n",
    "        for key in data.keys():\n",
    "            if key not in ['when', 'class', 'model', 'worker', 'hostname', context.prediction_col]:\n",
    "                instances[0].update({key: event.pop(key)})\n",
    "        event['request'] = {'instances': instances}\n",
    "        event['resp'] = [int(event.pop(context.prediction_col))]\n",
    "        return event\n",
    "    \n",
    "    records = json.loads(data.to_json(orient='records'))\n",
    "    records = [{'data': json.dumps(restructure_stream_event(context, record))} for record in records]\n",
    "    context.logger.info(f'Logging {len(records)} records, Record example: {records[0]}')\n",
    "    container, stream_path = split_path(stream_path)\n",
    "    # batch\n",
    "    step = 10\n",
    "    for idx in range(0, len(records), step):\n",
    "        response = context.v3io_client.put_records(container=container,\n",
    "                                                   path=stream_path, \n",
    "                                                   records=records[idx:idx+step])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_parquet(table, files_to_select=1):\n",
    "    mpath = [os.path.join(table, file) for file in os.listdir(table) if file.endswith(('parquet', 'pq'))]\n",
    "    files_by_updated = sorted(mpath, key=os.path.getmtime, reverse=False)\n",
    "    context.logger.debug_with('Input', input_files=files_by_updated[:files_to_select])\n",
    "    dfs = pd.concat([pd.read_parquet(file) for file in files_by_updated[:files_to_select]])\n",
    "    return dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_context(context):\n",
    "    setattr(context, 'metrics_table', os.environ['METRICS_TABLE'])\n",
    "    setattr(context, 'predictions_table', os.environ['PREDICTIONS_TABLE'])\n",
    "    setattr(context, 'output_stream', os.environ['OUTPUT_STREAM'])\n",
    "    setattr(context, 'timestamp_col', os.getenv('timestamp_col', 'when'))\n",
    "    setattr(context, 'orig_timestamp_col', os.getenv('orig_timestamp_col', 'timestamp'))\n",
    "    \n",
    "    v3io_client = v3io.dataplane.Client(endpoint='http://v3io-webapi:8081', logger_verbosity='DEBUG', transport_verbosity='DEBUG')\n",
    "    setattr(context, 'v3io_client', v3io_client)\n",
    "    if hasattr(context, 'trigger') and context.trigger.kind == 'cron':\n",
    "        create_stream(context, context.output_stream)\n",
    "    \n",
    "    setattr(context, 'label_col', os.environ['label_col'])\n",
    "    setattr(context, 'prediction_col', os.environ['prediction_col'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handler(context, event):\n",
    "    metrics = get_data_parquet(context.metrics_table, 2).loc[:, context.label_col].astype('int')\n",
    "    metrics.index.names = list([name if name != context.orig_timestamp_col else context.timestamp_col for name in metrics.index.names])\n",
    "    predictions = get_data_parquet(context.predictions_table, 2)\n",
    "    context.logger.debug(f'Labeling metrics ({metrics.shape}) and predictions ({predictions.shape})')\n",
    "    context.logger.debug_with('Indexes', metrics_index=metrics.index.names, predictions_index=predictions.index.names)\n",
    "    \n",
    "    full_df = pd.merge(left=predictions, right=metrics, left_on=list(metrics.index.names), left_index=True, right_index=True)\n",
    "    full_df = full_df.reset_index()\n",
    "    context.logger.debug(f'Fully labeled batch size is {full_df.shape}')\n",
    "    context.logger.debug(f'Indexes: {list(full_df.index.names)}')\n",
    "    context.logger.debug(f'Columns: {full_df.columns}')\n",
    "    context.logger.debug_with('sample', full_df=full_df.head(1))\n",
    "    full_df = full_df.loc[:10]\n",
    "    \n",
    "    push_to_stream(context, context.output_stream, full_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nuclio: end-code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "context.set_logger_level(False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_context(context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "event = nuclio.Event(body='')\n",
    "out = handler(context, event)\n",
    "out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stream test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from v3io.dataplane import Client\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "v3io_client = Client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# v3io_client.delete_stream(container='users', path='/admin/demo-network-operations/streaming/labeled_stream')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_stream(path, shard='0', seek_type='EARLIEST', last=100):\n",
    "    # seek the shard to the first record in it\n",
    "    container, stream_path = split_path(path)\n",
    "    shard_path = os.path.join(stream_path, shard)\n",
    "    response = v3io_client.seek_shard(container=container,\n",
    "                                      path=shard_path, \n",
    "                                      seek_type=seek_type)\n",
    "    response.raise_for_status()\n",
    "\n",
    "    # get records, starting from the location we got from seek\n",
    "    response = v3io_client.get_records(container=container,\n",
    "                                       path=shard_path, \n",
    "                                       location=response.output.location)\n",
    "    response.raise_for_status()\n",
    "    \n",
    "    models = ['pagehinkley', 'eddm', 'ddm']\n",
    "    result_record = response.output.records\n",
    "    records = [json.loads(record.data) for record in result_record[:last]]\n",
    "    pprint(records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'class': 'RandomForestClassifier',\n",
      "  'hostname': 'jupyter-558bf7fbc8-sq5kd',\n",
      "  'model': 'netops_predictor_v1',\n",
      "  'request': {'instances': [{'company': 'Wilson_LLC',\n",
      "                             'cpu_utilization': 66.9391393542,\n",
      "                             'data_center': 'Zachary_Drives',\n",
      "                             'device': '6001003522699',\n",
      "                             'is_error': 0,\n",
      "                             'latency': 0.5372793066,\n",
      "                             'packet_loss': 0.0,\n",
      "                             'throughput': 256.4821896882}]},\n",
      "  'resp': [0],\n",
      "  'when': 1593499337454,\n",
      "  'worker': None},\n",
      " {'class': 'RandomForestClassifier',\n",
      "  'hostname': 'jupyter-558bf7fbc8-sq5kd',\n",
      "  'model': 'netops_predictor_v1',\n",
      "  'request': {'instances': [{'company': 'Wilson_LLC',\n",
      "                             'cpu_utilization': 72.4927066691,\n",
      "                             'data_center': 'Obrien_Mountain',\n",
      "                             'device': '0966571261270',\n",
      "                             'is_error': 0,\n",
      "                             'latency': 0.0,\n",
      "                             'packet_loss': 4.961307962,\n",
      "                             'throughput': 264.1226480676}]},\n",
      "  'resp': [0],\n",
      "  'when': 1593499337454,\n",
      "  'worker': None}]\n"
     ]
    }
   ],
   "source": [
    "print_stream(context.output_stream, seek_type='EARLIEST', last=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deploy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlrun import code_to_function, mount_v3io"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<mlrun.runtimes.function.RemoteRuntime at 0x7fb3d7effcc0>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = code_to_function('labeled-stream-creator',\n",
    "                      kind='nuclio',\n",
    "                      project='network-operations')\n",
    "fn.spec.base_spec['spec']['build']['baseImage'] = 'mlrun/ml-models'\n",
    "fn.apply(mount_v3io())\n",
    "fn.add_trigger('cron', nuclio.triggers.CronTrigger(interval='1m'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2020-08-11 09:28:55,659 [info] function spec saved to path: ../src/labeled_stream_creator.yaml\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<mlrun.runtimes.function.RemoteRuntime at 0x7fb3d7effcc0>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn.save()\n",
    "fn.export('../src/labeled_stream_creator.yaml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2020-08-10 13:51:08,258 [info] deploy started\n",
      "[nuclio] 2020-08-10 13:54:42,507 (info) Build complete\n",
      "[nuclio] 2020-08-10 13:54:52,646 (info) Function deploy complete\n",
      "[nuclio] 2020-08-10 13:54:52,655 done creating network-operations-labeled-stream-creator, function address: 192.168.224.209:31059\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'http://192.168.224.209:31059'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn.deploy(project='network-operations')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>cpu_utilization</th>\n",
       "      <th>latency</th>\n",
       "      <th>packet_loss</th>\n",
       "      <th>throughput</th>\n",
       "      <th>predictions</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>when</th>\n",
       "      <th>company</th>\n",
       "      <th>data_center</th>\n",
       "      <th>device</th>\n",
       "      <th>model</th>\n",
       "      <th>class</th>\n",
       "      <th>worker</th>\n",
       "      <th>hostname</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2020-06-30 06:42:17.454</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">Wilson_LLC</th>\n",
       "      <th>Zachary_Drives</th>\n",
       "      <th>6001003522699</th>\n",
       "      <th>netops_predictor_v1</th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>NaN</th>\n",
       "      <th>jupyter-558bf7fbc8-sq5kd</th>\n",
       "      <td>66.939139</td>\n",
       "      <td>0.537279</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>256.482190</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Obrien_Mountain</th>\n",
       "      <th>0966571261270</th>\n",
       "      <th>netops_predictor_v1</th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>NaN</th>\n",
       "      <th>jupyter-558bf7fbc8-sq5kd</th>\n",
       "      <td>72.492707</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.961308</td>\n",
       "      <td>264.122648</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8069812479542</th>\n",
       "      <th>netops_predictor_v1</th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>NaN</th>\n",
       "      <th>jupyter-558bf7fbc8-sq5kd</th>\n",
       "      <td>69.116878</td>\n",
       "      <td>2.606934</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>263.528599</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Bennett__Delacruz_and_Walls</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Natasha_Harbors</th>\n",
       "      <th>5863502247054</th>\n",
       "      <th>netops_predictor_v1</th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>NaN</th>\n",
       "      <th>jupyter-558bf7fbc8-sq5kd</th>\n",
       "      <td>64.944107</td>\n",
       "      <td>1.571046</td>\n",
       "      <td>0.172451</td>\n",
       "      <td>241.149554</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4285071567351</th>\n",
       "      <th>netops_predictor_v1</th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>NaN</th>\n",
       "      <th>jupyter-558bf7fbc8-sq5kd</th>\n",
       "      <td>78.641128</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>263.688823</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2020-06-30 07:42:12.454</th>\n",
       "      <th>Wilson_LLC</th>\n",
       "      <th>Obrien_Mountain</th>\n",
       "      <th>8069812479542</th>\n",
       "      <th>netops_predictor_v1</th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>NaN</th>\n",
       "      <th>jupyter-558bf7fbc8-sq5kd</th>\n",
       "      <td>59.574487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>269.816306</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">Bennett__Delacruz_and_Walls</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Natasha_Harbors</th>\n",
       "      <th>5863502247054</th>\n",
       "      <th>netops_predictor_v1</th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>NaN</th>\n",
       "      <th>jupyter-558bf7fbc8-sq5kd</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4285071567351</th>\n",
       "      <th>netops_predictor_v1</th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>NaN</th>\n",
       "      <th>jupyter-558bf7fbc8-sq5kd</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Dominique_Branch</th>\n",
       "      <th>4579248894449</th>\n",
       "      <th>netops_predictor_v1</th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>NaN</th>\n",
       "      <th>jupyter-558bf7fbc8-sq5kd</th>\n",
       "      <td>69.053014</td>\n",
       "      <td>0.064657</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>255.943689</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7731097392608</th>\n",
       "      <th>netops_predictor_v1</th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>NaN</th>\n",
       "      <th>jupyter-558bf7fbc8-sq5kd</th>\n",
       "      <td>69.629988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.153888</td>\n",
       "      <td>238.072156</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5759 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                               cpu_utilization  \\\n",
       "when                    company                     data_center      device        model               class                  worker hostname                                    \n",
       "2020-06-30 06:42:17.454 Wilson_LLC                  Zachary_Drives   6001003522699 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        66.939139   \n",
       "                                                    Obrien_Mountain  0966571261270 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        72.492707   \n",
       "                                                                     8069812479542 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        69.116878   \n",
       "                        Bennett__Delacruz_and_Walls Natasha_Harbors  5863502247054 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        64.944107   \n",
       "                                                                     4285071567351 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        78.641128   \n",
       "...                                                                                                                                                                        ...   \n",
       "2020-06-30 07:42:12.454 Wilson_LLC                  Obrien_Mountain  8069812479542 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        59.574487   \n",
       "                        Bennett__Delacruz_and_Walls Natasha_Harbors  5863502247054 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd       100.000000   \n",
       "                                                                     4285071567351 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd       100.000000   \n",
       "                                                    Dominique_Branch 4579248894449 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        69.053014   \n",
       "                                                                     7731097392608 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        69.629988   \n",
       "\n",
       "                                                                                                                                                                  latency  \\\n",
       "when                    company                     data_center      device        model               class                  worker hostname                               \n",
       "2020-06-30 06:42:17.454 Wilson_LLC                  Zachary_Drives   6001003522699 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    0.537279   \n",
       "                                                    Obrien_Mountain  0966571261270 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    0.000000   \n",
       "                                                                     8069812479542 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    2.606934   \n",
       "                        Bennett__Delacruz_and_Walls Natasha_Harbors  5863502247054 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    1.571046   \n",
       "                                                                     4285071567351 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    0.000000   \n",
       "...                                                                                                                                                                   ...   \n",
       "2020-06-30 07:42:12.454 Wilson_LLC                  Obrien_Mountain  8069812479542 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    0.000000   \n",
       "                        Bennett__Delacruz_and_Walls Natasha_Harbors  5863502247054 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd  100.000000   \n",
       "                                                                     4285071567351 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd  100.000000   \n",
       "                                                    Dominique_Branch 4579248894449 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    0.064657   \n",
       "                                                                     7731097392608 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    0.000000   \n",
       "\n",
       "                                                                                                                                                               packet_loss  \\\n",
       "when                    company                     data_center      device        model               class                  worker hostname                                \n",
       "2020-06-30 06:42:17.454 Wilson_LLC                  Zachary_Drives   6001003522699 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd     0.000000   \n",
       "                                                    Obrien_Mountain  0966571261270 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd     4.961308   \n",
       "                                                                     8069812479542 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd     0.000000   \n",
       "                        Bennett__Delacruz_and_Walls Natasha_Harbors  5863502247054 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd     0.172451   \n",
       "                                                                     4285071567351 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd     0.000000   \n",
       "...                                                                                                                                                                    ...   \n",
       "2020-06-30 07:42:12.454 Wilson_LLC                  Obrien_Mountain  8069812479542 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd     0.000000   \n",
       "                        Bennett__Delacruz_and_Walls Natasha_Harbors  5863502247054 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    50.000000   \n",
       "                                                                     4285071567351 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    50.000000   \n",
       "                                                    Dominique_Branch 4579248894449 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd     0.000000   \n",
       "                                                                     7731097392608 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd     1.153888   \n",
       "\n",
       "                                                                                                                                                               throughput  \\\n",
       "when                    company                     data_center      device        model               class                  worker hostname                               \n",
       "2020-06-30 06:42:17.454 Wilson_LLC                  Zachary_Drives   6001003522699 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd  256.482190   \n",
       "                                                    Obrien_Mountain  0966571261270 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd  264.122648   \n",
       "                                                                     8069812479542 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd  263.528599   \n",
       "                        Bennett__Delacruz_and_Walls Natasha_Harbors  5863502247054 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd  241.149554   \n",
       "                                                                     4285071567351 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd  263.688823   \n",
       "...                                                                                                                                                                   ...   \n",
       "2020-06-30 07:42:12.454 Wilson_LLC                  Obrien_Mountain  8069812479542 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd  269.816306   \n",
       "                        Bennett__Delacruz_and_Walls Natasha_Harbors  5863502247054 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    0.000000   \n",
       "                                                                     4285071567351 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd    0.000000   \n",
       "                                                    Dominique_Branch 4579248894449 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd  255.943689   \n",
       "                                                                     7731097392608 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd  238.072156   \n",
       "\n",
       "                                                                                                                                                               predictions  \n",
       "when                    company                     data_center      device        model               class                  worker hostname                               \n",
       "2020-06-30 06:42:17.454 Wilson_LLC                  Zachary_Drives   6001003522699 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        False  \n",
       "                                                    Obrien_Mountain  0966571261270 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        False  \n",
       "                                                                     8069812479542 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        False  \n",
       "                        Bennett__Delacruz_and_Walls Natasha_Harbors  5863502247054 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        False  \n",
       "                                                                     4285071567351 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        False  \n",
       "...                                                                                                                                                                    ...  \n",
       "2020-06-30 07:42:12.454 Wilson_LLC                  Obrien_Mountain  8069812479542 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        False  \n",
       "                        Bennett__Delacruz_and_Walls Natasha_Harbors  5863502247054 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd         True  \n",
       "                                                                     4285071567351 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd         True  \n",
       "                                                    Dominique_Branch 4579248894449 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        False  \n",
       "                                                                     7731097392608 netops_predictor_v1 RandomForestClassifier NaN    jupyter-558bf7fbc8-sq5kd        False  \n",
       "\n",
       "[5759 rows x 5 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = pd.read_parquet('/User/demo-network-operations/streaming/predictions/20200630T064217-20200630T074212.parquet')\n",
    "predictions"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}