Formant/Formant_Query_Device_Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "latin-packing",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-23T14:22:16.610471Z",
     "iopub.status.busy": "2021-02-23T14:22:16.610129Z",
     "iopub.status.idle": "2021-02-23T14:22:16.627784Z",
     "shell.execute_reply": "2021-02-23T14:22:16.626866Z",
     "shell.execute_reply.started": "2021-02-23T14:22:16.610384Z"
    },
    "papermill": {},
    "tags": []
   },
   "source": [
    "<img width=\"8%\" alt=\"Formant.png\" src=\"https://raw.githubusercontent.com/jupyter-naas/awesome-notebooks/master/.github/assets/logos/Formant.png\" style=\"border-radius: 15%\">"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "compressed-wilson",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "# Formant - Query Device Network\n",
    "<a href=\"https://bit.ly/3JyWIk6\">Give Feedback</a> | <a href=\"https://github.com/jupyter-naas/awesome-notebooks/issues/new?assignees=&labels=bug&template=bug_report.md&title=Formant+-+Query+Device+Network:+Error+short+description\">Bug report</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "religious-programmer",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "**Tags:** #formant #matplotlib #notification #email #image"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fe9f56e-561c-4f52-aef8-b861c9462107",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "**Author:** [Nicolas Binford](https://www.linkedin.com/in/nicolasbinford)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7dcb55ac-d3c2-47de-8540-4a205d2680ed",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "**Last update:** 2023-05-25 (Created: 2023-05-25)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31ea7cdb-e10d-43fc-b026-f69249a59736",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "**Description:** This notebook queries network data over a period of time from a Formant device, graphs it, and emails the images."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "857d2611-00a5-47ed-96d2-96ccab3993f1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-12T14:44:15.678097Z",
     "iopub.status.busy": "2023-05-12T14:44:15.677874Z",
     "iopub.status.idle": "2023-05-12T14:44:15.685354Z",
     "shell.execute_reply": "2023-05-12T14:44:15.684538Z",
     "shell.execute_reply.started": "2023-05-12T14:44:15.678072Z"
    },
    "papermill": {},
    "tags": []
   },
   "source": [
    "**Reference:** \n",
    "- https://docs.formant.io/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distinguished-truth",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "## Input"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "install_cell",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "### Install packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "prescription-railway",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install formant"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "numeric-mediterranean",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "potential-surfing",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "outputs": [],
   "source": [
    "from datetime import datetime, timedelta\n",
    "import dateutil.parser as parser\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as mplt\n",
    "import os\n",
    "\n",
    "# Comment out this and Part 3 if running outside of Naas\n",
    "import naas\n",
    "\n",
    "from formant.sdk.cloud.v2 import Client as FormantClient\n",
    "from formant.sdk.cloud.v2.formant_admin_api_client.models.device_query import (\n",
    "    DeviceQuery,\n",
    ")\n",
    "from formant.sdk.cloud.v2.formant_admin_api_client.models.event_query import (\n",
    "    EventQuery,\n",
    "    EventQueryEventTypesItem,\n",
    "    EventQuerySeveritiesItem,\n",
    ")\n",
    "from formant.sdk.cloud.v2.formant_admin_api_client.models.event_list_response import (\n",
    "    EventListResponse,\n",
    ")\n",
    "from formant.sdk.cloud.v2.formant_query_api_client.models.query import Query"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aggressive-trustee",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "### Setup Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "continuous-melbourne",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Service account with \"viewer\" role\n",
    "FORMANT_EMAIL_DEMO = \"naas-demo@0d29f656-cc1c-4b9e-baad-199cfa1fcced.iam.formant.io\"\n",
    "FORMANT_PASSWORD_DEMO = \"prHHR_nnpzre3ZYTEZJ-NFGfS6OKRiEXw8xiJqtmlFuAT1Jx6yFyVQLmFe_ig9ei\"\n",
    "\n",
    "# 10 hours of data\n",
    "START_TIME = \"2023-03-10T00:00:00.000Z\"\n",
    "END_TIME = \"2023-03-10T00:10:00.000Z\"\n",
    "\n",
    "# Put your email address here\n",
    "EMAIL_TO=[\"nicolas@formant.io\", \"jeremy@naas.ai\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "registered-showcase",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tested-astrology",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "### Query and work with Formant data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "crude-louisville",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Part 1: Query from Formant\n",
    "\"\"\"\n",
    "\n",
    "# Names of devices you want to query\n",
    "my_device_names = [\"walter\"]\n",
    "\n",
    "# Streams that you want to query\n",
    "streams_of_interest = [\n",
    "    \"$.host.network\",\n",
    "]\n",
    "\n",
    "# Set up API clients\n",
    "fclient = FormantClient(email=FORMANT_EMAIL_DEMO, password=FORMANT_PASSWORD_DEMO)\n",
    "admin_api = fclient.admin\n",
    "query_api = fclient.query\n",
    "\n",
    "# Get a list of all available devices\n",
    "# Store device_id of desired devices, which is what is used for querying\n",
    "device_ids_to_query = []\n",
    "device_query = DeviceQuery(enabled=True)\n",
    "response = admin_api.devices.query(device_query).parsed\n",
    "for device in response.items:\n",
    "    if device.name in my_device_names:\n",
    "        device_ids_to_query.append(device.id)\n",
    "\n",
    "# Specify timerange for which script should pull data\n",
    "start = parser.isoparse(START_TIME)\n",
    "end = parser.isoparse(END_TIME)\n",
    "\n",
    "# The final list with the data from each device\n",
    "device_data_dicts = []\n",
    "\n",
    "for device_id in device_ids_to_query:\n",
    "    # Dict structure:\n",
    "    # {device: {data_label_0: [values], data_label_1: [values],...}}\n",
    "    device_data_dict = {}\n",
    "    data_values = {}\n",
    "\n",
    "    # Query stream data for this device\n",
    "    for stream_name in streams_of_interest:\n",
    "        query_model = Query(\n",
    "            start=start,\n",
    "            end=end,\n",
    "            device_ids=[device_id],\n",
    "            names=[stream_name],\n",
    "        )\n",
    "        response = query_api.queries.query(query_model)\n",
    "        response_items = response.parsed.items\n",
    "\n",
    "        # Labels specific to your desired stream\n",
    "        event_data_labels = [\"transmit\", \"receive\"]\n",
    "        for label in event_data_labels:\n",
    "            data_values[label] = []\n",
    "\n",
    "        for item in response_items:\n",
    "            # Some streams have 1 \"item\" with multiple \"points\", some have\n",
    "            # multiple \"items\" with multiple \"points\"\n",
    "            item_points = item.points\n",
    "            for point in item_points:\n",
    "                # This the entire datapoint with multiple labels and values\n",
    "                event_data = point.additional_properties[\"value\"][1]\n",
    "                for datapoint_specific in event_data:\n",
    "                    if datapoint_specific[\"label\"] in data_values.keys():\n",
    "                         datapoint_specific_label = datapoint_specific[\"label\"]\n",
    "                         datapoint_specific_value = datapoint_specific[\"value\"]\n",
    "                         data_values[datapoint_specific_label].append(datapoint_specific_value)\n",
    "\n",
    "    # Build the final dict structure\n",
    "    device_data_dict[device_id] = data_values\n",
    "    # Add the dict to the final list\n",
    "    device_data_dicts.append(device_data_dict)\n",
    "\n",
    "\"\"\"\n",
    "Part 2: Working with the data\n",
    "\"\"\"\n",
    "attachments = []\n",
    "#attachments = \"\"\n",
    "attachment_number = 0\n",
    "\n",
    "for device_data_dict in device_data_dicts:\n",
    "    # This is the device\n",
    "    device_id = list(device_data_dict.keys())[0]\n",
    "    # This is all of the data for that device\n",
    "    for data_values in device_data_dict.values():\n",
    "        # This is each type of event label in that datapoint\n",
    "        for datapoint_specific_label in data_values:\n",
    "\n",
    "            # This is the list of data from each stream\n",
    "            y_axis = np.asarray(data_values[datapoint_specific_label])\n",
    "            num_datapoints = len(y_axis)\n",
    "            # Create time increments for X-axis\n",
    "            x_axis = []\n",
    "            increment = (end - start) / num_datapoints\n",
    "            for i in range(0, num_datapoints):\n",
    "                x_axis.append(start + (increment * i))\n",
    "\n",
    "            # Plot the data\n",
    "            mplt.figure(f\"Formant Query Results {attachment_number}\", figsize=(14, 10))\n",
    "            mplt.plot(x_axis, y_axis)\n",
    "            # Hardcode some things for the labels text, but could make more generic\n",
    "            title = (f\"Device: {device_id}\\n\" +\n",
    "                    f\"Network {datapoint_specific_label} (Mbps)\\n\" +\n",
    "                    f\"{start} to {end}\")\n",
    "            mplt.title(title)\n",
    "            max_value = np.amax(y_axis)\n",
    "            avg_value = np.average(y_axis)\n",
    "            mplt.xlabel(f\"\\nMax value: {max_value}\\nAverage value: {avg_value}\", fontsize=14)\n",
    "            outfile_name = f\"figure_{attachment_number}.png\"\n",
    "            mplt.savefig(outfile_name)\n",
    "            outfile_path = os.path.join(os.getcwd(), outfile_name)\n",
    "            attachments.append(outfile_path)\n",
    "            attachment_number += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lonely-pacific",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-02T23:32:10.789097Z",
     "iopub.status.busy": "2021-07-02T23:32:10.788829Z",
     "iopub.status.idle": "2021-07-02T23:32:10.796900Z",
     "shell.execute_reply": "2021-07-02T23:32:10.796358Z",
     "shell.execute_reply.started": "2021-07-02T23:32:10.789033Z"
    },
    "papermill": {},
    "tags": []
   },
   "source": [
    "## Output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "890f7c86-b7bb-4f5d-9a1b-e492dd9580fd",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "source": [
    "### Send results to email"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c4e3b7b-6440-4844-8054-265f1aec65eb",
   "metadata": {
    "papermill": {},
    "tags": []
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Part 3: Exporting the data\n",
    "\"\"\"\n",
    "email_subject = f\"Formant Query Result, {datetime.now()}\"\n",
    "naas.notification.send(\n",
    "    email_to=EMAIL_TO,\n",
    "    subject=email_subject,\n",
    "    html=\"<p>Hello from Formant! Here is the data you requested.<p>\",\n",
    "    files=attachments\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "naas": {
   "notebook_id": "25c79c072793ed9689fec890977052fa9dd7977df1e8fb6e9832aa77df17f10f",
   "notebook_path": "Formant/Formant_Query_Device_Network.ipynb"
  },
  "papermill": {
   "default_parameters": {},
   "environment_variables": {},
   "parameters": {},
   "version": "2.3.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}