tests/Sensor/multi_record_Demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "multi_file_Demo.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyN4lBX/vZLtDQdbbRxa1NXt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aid4mh/QPrism/blob/main/tests/Sensor/multi_record_Demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LhSe9vDWxoC5",
        "outputId": "71277376-b6bc-41e8-909f-57a758602602"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pip in /usr/local/lib/python3.7/dist-packages (22.2.2)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "! python3 -m pip install --upgrade pip"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Install the dependencies and QPrism"
      ],
      "metadata": {
        "id": "tqzHWTZzEX2j"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install -r https://raw.githubusercontent.com/aid4mh/QPrism/main/requirements.txt"
      ],
      "metadata": {
        "id": "VLXUliwbx84K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install --no-deps QPrism"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s-xdWa0Qx-Mi",
        "outputId": "31d31ad1-fdc9-4be9-d45c-cde17d943d65"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: QPrism in /usr/local/lib/python3.7/dist-packages (0.1.9)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from QPrism.Sensor.DQM import DQM_multiple_record\n",
        "from QPrism.Sensor.load_data import *"
      ],
      "metadata": {
        "id": "k2ECvMlJyFFt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Load a list of sufficient json records and demonstrate the structure of the loaded data. For the original json file data structure, check the provided test data under data/sensor/json in the repo. modules/Sensor/load_data.py provides examples of how to load a json/csv file into this particular structure locally."
      ],
      "metadata": {
        "id": "DdHGaUsbydWM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from urllib.request import urlopen\n",
        "import json\n",
        "import pandas as pd"
      ],
      "metadata": {
        "id": "6wkzLOoizZxM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "url_list = ['https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000000.json', \n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000001.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000002.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000003.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000004.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000005.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000006.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000007.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000008.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000009.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000010.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000011.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000012.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000013.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000014.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000015.json',\n",
        "            'https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/json/SUBJ00001_Accelerometer_REC000016.json']"
      ],
      "metadata": {
        "id": "mGwLXJEFy3V1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_list = []\n",
        "for url in url_list:\n",
        "  response = urlopen(url)\n",
        "  json_data = response.read().decode('utf-8', 'replace')\n",
        "  try:\n",
        "    json_file = json.loads(json_data)\n",
        "  except:\n",
        "    df_list.append(pd.DataFrame([]))\n",
        "    continue\n",
        "  json_file = json.dumps(list(json_file['records']))\n",
        "  df = (pd.read_json(json_file))\n",
        "  df[list(df.keys())[0]] = df[list(df.keys())[0]].apply(lambda x: x.value)\n",
        "  df_list.append(df)\n",
        "print(df_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p1uPRVDvzT9Y",
        "outputId": "704c0989-2279-4c27-cc5c-9ebc47bb833c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[               timestamp         x         y          z\n",
            "0    1593954057754000000  2.229002  6.114791   5.324705\n",
            "1    1593954058125000000  2.135628  4.283227   6.833052\n",
            "2    1593954058506000000 -1.721431  5.561730  10.201694\n",
            "3    1593954058729000000 -0.220267  5.365406   7.668628\n",
            "4    1593954058937000000 -0.177171  5.130774   8.346188\n",
            "..                   ...       ...       ...        ...\n",
            "440  1593954154639000000 -0.258574  6.212954   7.388506\n",
            "441  1593954154858000000 -0.301669  6.212954   7.378930\n",
            "442  1593954155076000000 -0.275333  6.193800   7.407660\n",
            "443  1593954155276000000 -0.385466  6.205771   7.474698\n",
            "444  1593954155491000000 -0.385466  6.215348   7.129933\n",
            "\n",
            "[445 rows x 4 columns],                timestamp         x         y         z\n",
            "0    1593961300097000000 -2.176329  8.549694 -4.436456\n",
            "1    1593961300302000000 -1.776498  8.360553 -4.797980\n",
            "2    1593961300506000000 -1.826776  8.434772 -4.697424\n",
            "3    1593961300713000000 -2.130840  8.477868 -4.601655\n",
            "4    1593961300920000000 -1.977611  8.463503 -4.762067\n",
            "..                   ...       ...       ...       ...\n",
            "200  1593961341850000000 -2.267309  8.640674 -4.256891\n",
            "201  1593961342054000000 -2.147599  8.645462 -4.010288\n",
            "202  1593961342261000000 -2.348712  8.640674 -4.340688\n",
            "203  1593961342468000000 -2.262521  8.635885 -4.118027\n",
            "204  1593961342674000000 -2.669535  8.643068 -4.814740\n",
            "\n",
            "[205 rows x 4 columns],                timestamp         x         y         z\n",
            "0    1593961300097000000 -2.176329  8.549694 -4.436456\n",
            "1    1593961300302000000 -1.776498  8.360553 -4.797980\n",
            "2    1593961300506000000 -1.826776  8.434772 -4.697424\n",
            "3    1593961300713000000 -2.130840  8.477868 -4.601655\n",
            "4    1593961300920000000 -1.977611  8.463503 -4.762067\n",
            "..                   ...       ...       ...       ...\n",
            "200  1593961341850000000 -2.267309  8.640674 -4.256891\n",
            "201  1593961342054000000 -2.147599  8.645462 -4.010288\n",
            "202  1593961342261000000 -2.348712  8.640674 -4.340688\n",
            "203  1593961342468000000 -2.262521  8.635885 -4.118027\n",
            "204  1593961342674000000 -2.669535  8.643068 -4.814740\n",
            "\n",
            "[205 rows x 4 columns],                 timestamp         x         y         z\n",
            "0     1593963206555000000  2.944868  9.244013 -1.522712\n",
            "1     1593963206769000000  3.038242  9.196129 -1.829170\n",
            "2     1593963206973000000  3.059790  9.200917 -1.587356\n",
            "3     1593963207179000000  3.040636  9.203311 -1.539472\n",
            "4     1593963207385000000  3.014300  9.208099 -1.553837\n",
            "...                   ...       ...       ...       ...\n",
            "1062  1593963608699000000 -0.543484  3.928885  8.767567\n",
            "1063  1593963611106000000 -0.454898  3.940856  8.834604\n",
            "1064  1593963611310000000 -0.562637  3.653552  8.968679\n",
            "1065  1593963611518000000 -0.505177  3.612850  8.923190\n",
            "1066  1593963611733000000 -0.445322  3.598485  8.894460\n",
            "\n",
            "[1067 rows x 4 columns],                 timestamp         x         y         z\n",
            "0     1593961343290000000 -1.484405  8.662222 -4.252102\n",
            "1     1593961343495000000 -1.345541  8.561666 -4.793192\n",
            "2     1593961343708000000 -2.006341  8.595184 -4.841076\n",
            "3     1593961343912000000 -1.855506  8.528147 -4.848259\n",
            "4     1593961344114000000 -0.919373  8.396465 -4.754885\n",
            "...                   ...       ...       ...       ...\n",
            "5237  1593962661392000000  0.883460  5.846641  7.740454\n",
            "5238  1593962661598000000  0.905008  5.786786  7.757214\n",
            "5239  1593962661804000000  0.859518  5.870583  7.539341\n",
            "5240  1593962662009000000  0.864307  5.930438  7.671022\n",
            "5241  1593962662217000000  0.890643  5.872977  7.893683\n",
            "\n",
            "[5242 rows x 4 columns],                 timestamp         x         y         z\n",
            "0     1593965108713000000 -0.418985  8.166622 -5.542577\n",
            "1     1593965108914000000 -0.469264  8.145074 -5.604826\n",
            "2     1593965109115000000 -0.490811  8.154651 -5.590461\n",
            "3     1593965109335000000 -0.502782  8.183381 -5.499481\n",
            "4     1593965109539000000 -0.505177  8.190564 -5.379771\n",
            "...                   ...       ...       ...       ...\n",
            "1957  1593965596247000000 -1.824382  9.028535 -3.433285\n",
            "1958  1593965596467000000 -1.769315  9.021352 -3.402161\n",
            "1959  1593965596687000000 -1.704672  9.006987 -3.603274\n",
            "1960  1593965596898000000 -1.460463  9.047688 -3.129222\n",
            "1961  1593965597115000000 -1.726219  9.009381 -3.411737\n",
            "\n",
            "[1962 rows x 4 columns], Empty DataFrame\n",
            "Columns: []\n",
            "Index: [],                 timestamp         x         y         z\n",
            "0     1593961343290000000 -1.484405  8.662222 -4.252102\n",
            "1     1593961343495000000 -1.345541  8.561666 -4.793192\n",
            "2     1593961343708000000 -2.006341  8.595184 -4.841076\n",
            "3     1593961343912000000 -1.855506  8.528147 -4.848259\n",
            "4     1593961344114000000 -0.919373  8.396465 -4.754885\n",
            "...                   ...       ...       ...       ...\n",
            "5237  1593962661392000000  0.883460  5.846641  7.740454\n",
            "5238  1593962661598000000  0.905008  5.786786  7.757214\n",
            "5239  1593962661804000000  0.859518  5.870583  7.539341\n",
            "5240  1593962662009000000  0.864307  5.930438  7.671022\n",
            "5241  1593962662217000000  0.890643  5.872977  7.893683\n",
            "\n",
            "[5242 rows x 4 columns],                 timestamp         x         y         z\n",
            "0     1593965108713000000 -0.418985  8.166622 -5.542577\n",
            "1     1593965108914000000 -0.469264  8.145074 -5.604826\n",
            "2     1593965109115000000 -0.490811  8.154651 -5.590461\n",
            "3     1593965109335000000 -0.502782  8.183381 -5.499481\n",
            "4     1593965109539000000 -0.505177  8.190564 -5.379771\n",
            "...                   ...       ...       ...       ...\n",
            "1957  1593965596247000000 -1.824382  9.028535 -3.433285\n",
            "1958  1593965596467000000 -1.769315  9.021352 -3.402161\n",
            "1959  1593965596687000000 -1.704672  9.006987 -3.603274\n",
            "1960  1593965596898000000 -1.460463  9.047688 -3.129222\n",
            "1961  1593965597115000000 -1.726219  9.009381 -3.411737\n",
            "\n",
            "[1962 rows x 4 columns],                 timestamp          x         y          z\n",
            "0     1593968276953000000  -0.387861  3.294421   8.212111\n",
            "1     1593968277172000000  -0.706290  5.561730   6.098032\n",
            "2     1593968277378000000  -0.629675  5.595249   6.926425\n",
            "3     1593968277591000000  -5.078102  5.631162  13.709199\n",
            "4     1593968278539000000   1.314417  4.661511   9.356541\n",
            "...                   ...        ...       ...        ...\n",
            "1949  1593968831358000000  10.591949  9.016563   6.121974\n",
            "1950  1593968831573000000   2.521094  3.014300   1.242591\n",
            "1951  1593968831801000000   4.175488  8.130709   4.465186\n",
            "1952  1593968832010000000   1.754950  6.301539   2.896984\n",
            "1953  1593968832229000000   6.387730  8.485051   6.354211\n",
            "\n",
            "[1954 rows x 4 columns],                timestamp         x         y         z\n",
            "0    1593971073415000000 -0.313640  5.863400  7.857770\n",
            "1    1593971073622000000 -0.227449  5.676652  6.912060\n",
            "2    1593971073844000000 -0.763750  5.719748  8.248025\n",
            "3    1593971074044000000  0.454898  5.968745  7.955932\n",
            "4    1593971074256000000 -0.155623  5.899313  8.166622\n",
            "..                   ...       ...       ...       ...\n",
            "189  1593971241944000000 -0.859518  5.631162  7.769185\n",
            "190  1593971242161000000  0.124498  5.882554  8.413225\n",
            "191  1593971242364000000 -0.696713  6.198588  7.290344\n",
            "192  1593971243556000000 -0.028730  5.916072  8.719683\n",
            "193  1593971243766000000 -0.368707  5.750873  7.833828\n",
            "\n",
            "[194 rows x 4 columns],                 timestamp          x         y          z\n",
            "0     1593968276953000000  -0.387861  3.294421   8.212111\n",
            "1     1593968277172000000  -0.706290  5.561730   6.098032\n",
            "2     1593968277378000000  -0.629675  5.595249   6.926425\n",
            "3     1593968277591000000  -5.078102  5.631162  13.709199\n",
            "4     1593968278539000000   1.314417  4.661511   9.356541\n",
            "...                   ...        ...       ...        ...\n",
            "1949  1593968831358000000  10.591949  9.016563   6.121974\n",
            "1950  1593968831573000000   2.521094  3.014300   1.242591\n",
            "1951  1593968831801000000   4.175488  8.130709   4.465186\n",
            "1952  1593968832010000000   1.754950  6.301539   2.896984\n",
            "1953  1593968832229000000   6.387730  8.485051   6.354211\n",
            "\n",
            "[1954 rows x 4 columns],                 timestamp          x         y          z\n",
            "0     1593968276953000000  -0.387861  3.294421   8.212111\n",
            "1     1593968277172000000  -0.706290  5.561730   6.098032\n",
            "2     1593968277378000000  -0.629675  5.595249   6.926425\n",
            "3     1593968277591000000  -5.078102  5.631162  13.709199\n",
            "4     1593968278539000000   1.314417  4.661511   9.356541\n",
            "...                   ...        ...       ...        ...\n",
            "1949  1593968831358000000  10.591949  9.016563   6.121974\n",
            "1950  1593968831573000000   2.521094  3.014300   1.242591\n",
            "1951  1593968831801000000   4.175488  8.130709   4.465186\n",
            "1952  1593968832010000000   1.754950  6.301539   2.896984\n",
            "1953  1593968832229000000   6.387730  8.485051   6.354211\n",
            "\n",
            "[1954 rows x 4 columns],                timestamp         x         y         z\n",
            "0    1593971073415000000 -0.313640  5.863400  7.857770\n",
            "1    1593971073622000000 -0.227449  5.676652  6.912060\n",
            "2    1593971073844000000 -0.763750  5.719748  8.248025\n",
            "3    1593971074044000000  0.454898  5.968745  7.955932\n",
            "4    1593971074256000000 -0.155623  5.899313  8.166622\n",
            "..                   ...       ...       ...       ...\n",
            "189  1593971241944000000 -0.859518  5.631162  7.769185\n",
            "190  1593971242161000000  0.124498  5.882554  8.413225\n",
            "191  1593971242364000000 -0.696713  6.198588  7.290344\n",
            "192  1593971243556000000 -0.028730  5.916072  8.719683\n",
            "193  1593971243766000000 -0.368707  5.750873  7.833828\n",
            "\n",
            "[194 rows x 4 columns],                 timestamp         x          y         z\n",
            "0     1593971244424000000 -0.311246   5.712565  7.802703\n",
            "1     1593971244624000000 -0.421380   5.643134  7.836222\n",
            "2     1593971244840000000 -0.287304   5.714960  8.183381\n",
            "3     1593971245058000000 -0.373495   5.659893  8.149862\n",
            "4     1593971245271000000 -0.227449   5.755661  7.829040\n",
            "...                   ...       ...        ...       ...\n",
            "2782  1593971855221000000 -1.242591   4.376601  6.536170\n",
            "2783  1593971855437000000  4.522647   7.220912  4.960786\n",
            "2784  1593971855655000000  3.794810  10.065224  6.179435\n",
            "2785  1593971855874000000  3.177106   7.934384  8.779537\n",
            "2786  1593971856092000000 -0.691924   4.752491  2.928109\n",
            "\n",
            "[2787 rows x 4 columns],                 timestamp         x          y          z\n",
            "0     1593971874412000000 -1.843535   7.187394   7.443573\n",
            "1     1593971874627000000 -9.962274   4.807557   3.069367\n",
            "2     1593971874846000000 -6.813898   3.995923   4.012682\n",
            "3     1593971875064000000 -2.753332   6.452374   6.744466\n",
            "4     1593971875282000000 -3.347094  10.807426   6.095637\n",
            "...                   ...       ...        ...        ...\n",
            "2639  1593972450041000000  3.385401   7.275979   9.368511\n",
            "2640  1593972450257000000  0.284910   4.630386  12.423512\n",
            "2641  1593972450462000000  2.142811   5.279215   3.742137\n",
            "2642  1593972450675000000  1.970428   7.038953   7.481880\n",
            "2643  1593972450893000000  6.313510   5.204995   6.220136\n",
            "\n",
            "[2644 rows x 4 columns],                 timestamp         x         y         z\n",
            "0     1593972468340000000 -1.288081  0.746991  5.501875\n",
            "1     1593972468574000000  0.890643  4.254497  7.196970\n",
            "2     1593972468790000000  3.150770  1.999158  7.814674\n",
            "3     1593972468991000000  1.498770  3.818752  3.342306\n",
            "4     1593972469213000000 -0.581791  5.944803  9.967062\n",
            "...                   ...       ...       ...       ...\n",
            "2061  1593974165350000000 -0.090980  0.337582  9.617508\n",
            "2062  1593974165572000000 -0.074220  0.323217  9.622296\n",
            "2063  1593974165772000000 -0.086191  0.318429  9.619903\n",
            "2064  1593974165983000000 -0.081403  0.318429  9.627085\n",
            "2065  1593974166197000000 -0.079009  0.325611  9.627085\n",
            "\n",
            "[2066 rows x 4 columns]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Create the DQM_multiple_file object, set the input data frame, then compute the averaged DQM for all records"
      ],
      "metadata": {
        "id": "VKTVjvCUz7M4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm = DQM_multiple_record()\n",
        "multi_json_file_dqm.set_input_data(df_list)\n",
        "multi_json_file_dqm.compute_avg_DQM()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yEmNHaGGz8ry",
        "outputId": "e3632e83-bd3b-4ee9-87dc-5b3fdeb7e8dd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Start computing the DQM... This may take a long time if the dataset is large\n",
            "The total time for computing the DQM is: 21.35908317565918 seconds.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Get the fields of computed averaged DQM"
      ],
      "metadata": {
        "id": "TOR8uNh40AE8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_avg_fields()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EpdAZetp0AuW",
        "outputId": "5cd09300-602c-473e-996e-1bd02e02de49"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['IRLR', 'SNR', 'SCR', 'RLC', 'VRC', 'SRC', 'MDR', 'APD', 'VDR']"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Get the computed averaged DQM as a list, each element in the list represents a metric with the same order in the fields list."
      ],
      "metadata": {
        "id": "vUNd5aXT0NcD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_avg_DQM()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7eg2m1Ch0Tsi",
        "outputId": "58a96dd2-c242-412d-8510-6453d558ad01"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['0.9411764705882353',\n",
              " '-2.2764849187263625',\n",
              " '1.0',\n",
              " '0.6111766701110624',\n",
              " '0.6713937097996874',\n",
              " '0.42181793198733025',\n",
              " '0.2447423162562039',\n",
              " '0.013750579814819692',\n",
              " '1.0']"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Get each single metric of the computed averaged DQM as a str"
      ],
      "metadata": {
        "id": "-Li7lq3u0due"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_IRLR()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "PGugEKvD0f1k",
        "outputId": "8b839425-0a13-4bec-84e4-bdf8b11e1b83"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'0.9411764705882353'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_SNR()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "Fmuh4Stz0hrt",
        "outputId": "099627d2-485b-4ba4-bc12-8cabcf91515f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'-2.2764849187263625'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_SCR()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "pvnpZKc40i5W",
        "outputId": "96de22be-0b51-4708-a149-8063efffccb7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'1.0'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_SRC()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "NIrAFvc10j82",
        "outputId": "559c3e74-47c0-40bf-e95d-e38cb61ed15e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'0.42181793198733025'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_MDR()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "TDXbZFhq0lAi",
        "outputId": "1fd0d39c-c708-47df-c65e-ad9352fc3ee2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'0.2447423162562039'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_APD()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "sZObzd590mVm",
        "outputId": "a998f677-962b-4a02-d73a-7e6cfcc3c1ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'0.013750579814819692'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_VRC()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "fAYbdD_k0nZK",
        "outputId": "43f90ba6-83e7-4770-dac9-eacf3a61f59e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'0.6713937097996874'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "multi_json_file_dqm.get_RLC()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "oOq571RG0ob_",
        "outputId": "ee115f0e-16db-436b-8ad6-dc64c6a7b13e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'0.6111766701110624'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Save the computed averaged DQM as a csv file at given path."
      ],
      "metadata": {
        "id": "WCJvivJd0qt1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UIu5IJgD0qJQ",
        "outputId": "cf0839b0-86a8-40e5-fa31-aa94e9235fb9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.save_avg_DQM_to_file('/content/gdrive/MyDrive/CAMH/JOSS/Sensor_result/demo_result_multi_file.csv')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p9YIb86S0x2G",
        "outputId": "b60e5dfd-0185-41d6-d10a-6e72492da1aa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Data successfuly saved.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Display the saved csv file."
      ],
      "metadata": {
        "id": "aTnqtOod039H"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"/content/gdrive/MyDrive/CAMH/JOSS/Sensor_result/demo_result_multi_file.csv\", \"r\") as DQM_csv:\n",
        "    for row in DQM_csv:\n",
        "        print(row)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sQhdqCR20-zA",
        "outputId": "c9df8f0b-1a37-401f-d10f-8a75bf42d422"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "IRLR,SNR,SCR,RLC,VRC,SRC,MDR,APD,VDR\n",
            "\n",
            "0.9411764705882353,-2.2764849187263625,1.0,0.6111766701110624,0.6713937097996874,0.42181793198733025,0.2447423162562039,0.013750579814819692,1.0\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Now we retrieve the individual DQMs for all input records."
      ],
      "metadata": {
        "id": "P3BXuGxl1FP8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.compute_individual_DQM()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D5yWtBst1JcI",
        "outputId": "c8a308e4-4964-4222-88d7-cf724a548934"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Start computing the DQM... This may take a long time if the dataset is large\n",
            "The total time for computing the DQM is: 19.200064659118652 seconds.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_individual_fields()"
      ],
      "metadata": {
        "id": "Zj69nKla1LT3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "105571a2-7e0e-4474-db96-3fce409c337d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['IRLR', 'SNR', 'SRC', 'MDR', 'APD', 'VDR']"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Now the returned list represents a list of DQMs, each element for a corresponding record in the input data list."
      ],
      "metadata": {
        "id": "GlTD04x-1M74"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "multi_json_file_dqm.get_individual_DQM()"
      ],
      "metadata": {
        "id": "2IAOHHIq1Nd_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "34059e0f-f04e-40ff-f189-58d2c2c29285"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['1',\n",
              "  '-3.096340603605768',\n",
              "  '1.0',\n",
              "  '0.9011216710642995',\n",
              "  '0.013303769401330377',\n",
              "  '0.020224719101123594'],\n",
              " ['1',\n",
              "  '-2.5639205648692864',\n",
              "  '1.0',\n",
              "  '0.9096993348886553',\n",
              "  '0.00966183574879227',\n",
              "  '0.00975609756097561'],\n",
              " ['1',\n",
              "  '-2.5639205648692864',\n",
              "  '1.0',\n",
              "  '0.9096993348886553',\n",
              "  '0.00966183574879227',\n",
              "  '0.014634146341463415'],\n",
              " ['1',\n",
              "  '-2.5443294444725577',\n",
              "  '1.0',\n",
              "  '0.21887748986956956',\n",
              "  '0.42696025778732544',\n",
              "  '0.0018744142455482662'],\n",
              " ['1',\n",
              "  '-1.8544629444980452',\n",
              "  '1.0',\n",
              "  '0.011961611519057058',\n",
              "  '0.1642219387755102',\n",
              "  '0.002098435711560473'],\n",
              " ['1',\n",
              "  '-2.1372237098572295',\n",
              "  '1.0',\n",
              "  '0.6597869045768825',\n",
              "  '0.11221719457013575',\n",
              "  '0.011722731906218144'],\n",
              " ['1',\n",
              "  '-1.8544629444980452',\n",
              "  '1.0',\n",
              "  '0.011961611519057058',\n",
              "  '0.1642219387755102',\n",
              "  '0.002098435711560473'],\n",
              " ['1',\n",
              "  '-2.1372237098572295',\n",
              "  '1.0',\n",
              "  '0.6597869045768825',\n",
              "  '0.11221719457013575',\n",
              "  '0.016819571865443424'],\n",
              " ['1',\n",
              "  '-2.425568247528492',\n",
              "  '1.0',\n",
              "  '0.22998553798698063',\n",
              "  '0.21399839098954143',\n",
              "  '0.01330603889457523'],\n",
              " ['1',\n",
              "  '-1.6358899797539834',\n",
              "  '1.0',\n",
              "  '0.0008434478296908132',\n",
              "  '0.7531806615776081',\n",
              "  '0.030927835051546393'],\n",
              " ['1',\n",
              "  '-2.425568247528492',\n",
              "  '1.0',\n",
              "  '0.22998553798698063',\n",
              "  '0.21399839098954143',\n",
              "  '0.0127942681678608'],\n",
              " ['1',\n",
              "  '-2.425568247528492',\n",
              "  '1.0',\n",
              "  '0.22998553798698063',\n",
              "  '0.21399839098954143',\n",
              "  '0.01023541453428864'],\n",
              " ['1',\n",
              "  '-1.6358899797539834',\n",
              "  '1.0',\n",
              "  '0.0008434478296908132',\n",
              "  '0.7531806615776081',\n",
              "  '0.02577319587628866'],\n",
              " ['1',\n",
              "  '-2.6345095072869413',\n",
              "  '1.0',\n",
              "  '0.853564399886102',\n",
              "  '0.014149274849663955',\n",
              "  '0.022246142805884463'],\n",
              " ['1',\n",
              "  '-2.623577265481869',\n",
              "  '1.0',\n",
              "  '0.9209837701144623',\n",
              "  '0.006015037593984963',\n",
              "  '0.0208018154311649'],\n",
              " ['1',\n",
              "  '-1.865302738232099',\n",
              "  '1.0',\n",
              "  '3.6927333790792716e-07',\n",
              "  '0.734890286154241',\n",
              "  '0.008228460793804453']]"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Save the list of DQMs to csv file."
      ],
      "metadata": {
        "id": "pYHcHHgu1Qg4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.save_individual_DQM_to_file(\"/content/gdrive/MyDrive/CAMH/JOSS/Sensor_result/demo_result_multi_file_individual.csv\")"
      ],
      "metadata": {
        "id": "9iLqLPQZ1WX5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "571fcabe-3431-41e8-e2fc-6c08f659e2c9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Data successfuly saved.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Display the saved csv file."
      ],
      "metadata": {
        "id": "b6YYJ-Zf1fWY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"/content/gdrive/MyDrive/CAMH/JOSS/Sensor_result/demo_result_multi_file_individual.csv\", \"r\") as DQM_csv:\n",
        "    for row in DQM_csv:\n",
        "        print(row)"
      ],
      "metadata": {
        "id": "MitX93Py1f31",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8a4d337f-9d23-4608-c815-afedb79c193a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "IRLR,SNR,SRC,MDR,APD,VDR\n",
            "\n",
            "1,-3.096340603605768,1.0,0.9011216710642995,0.013303769401330377,0.020224719101123594\n",
            "\n",
            "1,-2.5639205648692864,1.0,0.9096993348886553,0.00966183574879227,0.00975609756097561\n",
            "\n",
            "1,-2.5639205648692864,1.0,0.9096993348886553,0.00966183574879227,0.014634146341463415\n",
            "\n",
            "1,-2.5443294444725577,1.0,0.21887748986956956,0.42696025778732544,0.0018744142455482662\n",
            "\n",
            "1,-1.8544629444980452,1.0,0.011961611519057058,0.1642219387755102,0.002098435711560473\n",
            "\n",
            "1,-2.1372237098572295,1.0,0.6597869045768825,0.11221719457013575,0.011722731906218144\n",
            "\n",
            "1,-1.8544629444980452,1.0,0.011961611519057058,0.1642219387755102,0.002098435711560473\n",
            "\n",
            "1,-2.1372237098572295,1.0,0.6597869045768825,0.11221719457013575,0.016819571865443424\n",
            "\n",
            "1,-2.425568247528492,1.0,0.22998553798698063,0.21399839098954143,0.01330603889457523\n",
            "\n",
            "1,-1.6358899797539834,1.0,0.0008434478296908132,0.7531806615776081,0.030927835051546393\n",
            "\n",
            "1,-2.425568247528492,1.0,0.22998553798698063,0.21399839098954143,0.0127942681678608\n",
            "\n",
            "1,-2.425568247528492,1.0,0.22998553798698063,0.21399839098954143,0.01023541453428864\n",
            "\n",
            "1,-1.6358899797539834,1.0,0.0008434478296908132,0.7531806615776081,0.02577319587628866\n",
            "\n",
            "1,-2.6345095072869413,1.0,0.853564399886102,0.014149274849663955,0.022246142805884463\n",
            "\n",
            "1,-2.623577265481869,1.0,0.9209837701144623,0.006015037593984963,0.0208018154311649\n",
            "\n",
            "1,-1.865302738232099,1.0,3.6927333790792716e-07,0.734890286154241,0.008228460793804453\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Set SNR to be not included in DQM"
      ],
      "metadata": {
        "id": "20Zyowv-1mYD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.set_SNR(False)"
      ],
      "metadata": {
        "id": "uLZ0FLTF1nvR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Compute and get the updated averaged DQM, now SNR should not be included in the DQM."
      ],
      "metadata": {
        "id": "DmgGowtV1pED"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.compute_avg_DQM()"
      ],
      "metadata": {
        "id": "AQcjFA7z1ss7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6c326034-e4d7-4dc8-b46c-0fe55925a701"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Start computing the DQM... This may take a long time if the dataset is large\n",
            "The total time for computing the DQM is: 21.256505966186523 seconds.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_avg_fields()"
      ],
      "metadata": {
        "id": "CZ0KiQGT1vD5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "52cf5df3-5d7a-4f59-ed08-e6bce0303f0c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['IRLR', 'SCR', 'RLC', 'VRC', 'SRC', 'MDR', 'APD', 'VDR']"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_avg_DQM()"
      ],
      "metadata": {
        "id": "UNDRa30S1v-7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f248f17c-9ff2-4fa0-9ff4-cf2ad3fb7f62"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['1.0',\n",
              " '1.0',\n",
              " '0.6111766701110624',\n",
              " '0.6713937097996874',\n",
              " '0.42181793198733025',\n",
              " '0.2447423162562039',\n",
              " '0.013534474260714848',\n",
              " '1.0']"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Now get_SNR should not be called"
      ],
      "metadata": {
        "id": "VYgdBavZ1w5T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_SNR()"
      ],
      "metadata": {
        "id": "l7MqTxYX1yAY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "4d27bc51-eaf4-4b86-86fa-c326ad24b0b2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'SNR is not computed according to the configuration'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Compute and get the updated individual DQM, not SNR should not be included in the DQM."
      ],
      "metadata": {
        "id": "nqIRMD5o16Y4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.compute_individual_DQM()"
      ],
      "metadata": {
        "id": "2rpSB1Qi2A8r",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b71f2d34-3217-4d3c-90d2-3f0206c1d834"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Start computing the DQM... This may take a long time if the dataset is large\n",
            "The total time for computing the DQM is: 19.505983114242554 seconds.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_individual_fields()"
      ],
      "metadata": {
        "id": "gESOSdeN2FDD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "24349fcc-8326-4020-9ac3-092ab01fc172"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['IRLR', 'SRC', 'MDR', 'APD', 'VDR']"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_json_file_dqm.get_individual_DQM()"
      ],
      "metadata": {
        "id": "5NAA3ODI2HSM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "06260b20-e0fb-4586-fd06-d598c60f6200"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['1',\n",
              "  '1.0',\n",
              "  '0.9011216710642995',\n",
              "  '0.013303769401330377',\n",
              "  '0.02696629213483146'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.9096993348886553',\n",
              "  '0.00966183574879227',\n",
              "  '0.00975609756097561'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.9096993348886553',\n",
              "  '0.00966183574879227',\n",
              "  '0.00975609756097561'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.21887748986956956',\n",
              "  '0.42696025778732544',\n",
              "  '0.004686035613870665'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.011961611519057058',\n",
              "  '0.1642219387755102',\n",
              "  '0.01755055322396032'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.6597869045768825',\n",
              "  '0.11221719457013575',\n",
              "  '0.008664627930682976'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.011961611519057058',\n",
              "  '0.1642219387755102',\n",
              "  '0.002098435711560473'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.6597869045768825',\n",
              "  '0.11221719457013575',\n",
              "  '0.010193679918450561'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.22998553798698063',\n",
              "  '0.21399839098954143',\n",
              "  '0.012282497441146366'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.0008434478296908132',\n",
              "  '0.7531806615776081',\n",
              "  '0.020618556701030927'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.22998553798698063',\n",
              "  '0.21399839098954143',\n",
              "  '0.009211873080859774'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.22998553798698063',\n",
              "  '0.21399839098954143',\n",
              "  '0.012282497441146366'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.0008434478296908132',\n",
              "  '0.7531806615776081',\n",
              "  '0.020618556701030927'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.853564399886102',\n",
              "  '0.014149274849663955',\n",
              "  '0.021887334050950842'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '0.9209837701144623',\n",
              "  '0.006015037593984963',\n",
              "  '0.02042360060514372'],\n",
              " ['1',\n",
              "  '1.0',\n",
              "  '3.6927333790792716e-07',\n",
              "  '0.734890286154241',\n",
              "  '0.008712487899322363']]"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Note that you can set any metrics except IRLR to be not included as above.\n",
        "\n",
        "#For an empty record that cannot be loaded without error, load it as an empty data frame instead.\n",
        "\n",
        "#Now load a list csv sensor data, similarly, you can check modules/Sensor/load_data.py for the code that loads the csv into this structure locally."
      ],
      "metadata": {
        "id": "ZYZxEl-w2dgA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "csv_url_list = [\"https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/csv/id9603e9c3.csv\",\n",
        "                \"https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/csv/idabd0c53c.csv\",\n",
        "                \"https://raw.githubusercontent.com/aid4mh/QPrism/main/data/sensor/csv/idd80ac2b4.csv\"]\n",
        "csv_df_list = []\n",
        "for url in csv_url_list:\n",
        "  csv_df_list.append(load_data_csv_one_file(url))\n",
        "for record in csv_df_list:\n",
        "  print(record)"
      ],
      "metadata": {
        "id": "BePJfEh-2nIz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "965fcd35-a1c3-4a2d-9af8-26a516e0084d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "         time_s   lw_x   lw_y   lw_z   lh_x   lh_y   lh_z   la_x   la_y  \\\n",
            "0          0.01 -0.004  0.945 -0.461  0.055  0.992 -0.199  0.168  1.418   \n",
            "1          0.02 -0.406 -0.379  1.031 -0.320 -0.102  0.844 -0.422  0.559   \n",
            "2          0.03 -0.340 -0.918  1.160 -0.152 -0.676  1.160 -0.500  1.004   \n",
            "3          0.04 -0.207 -1.023  0.344  0.059 -0.797  0.453 -0.363  1.363   \n",
            "4          0.05 -0.152 -1.098 -0.012  0.133 -0.875  0.133 -0.301  1.582   \n",
            "...         ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
            "111295  1112.96  0.000  0.000  0.000  0.246  0.012  0.949  0.453  0.102   \n",
            "111296  1112.97  0.000  0.000  0.000  0.246  0.012  0.949  0.449  0.102   \n",
            "111297  1112.98  0.000  0.000  0.000  0.246  0.012  0.949  0.449  0.102   \n",
            "111298  1112.99  0.000  0.000  0.000  0.246  0.012  0.949  0.449  0.102   \n",
            "111299  1113.00  0.000  0.000  0.000  0.246  0.012  0.949  0.453  0.102   \n",
            "\n",
            "         la_z   ra_x   ra_y   ra_z  \n",
            "0      -0.070 -0.105  1.000  0.125  \n",
            "1       0.891 -0.352 -0.086  0.984  \n",
            "2       1.168 -0.105 -0.602  1.438  \n",
            "3       0.523  0.109 -0.797  0.793  \n",
            "4       0.305  0.203 -0.992  0.512  \n",
            "...       ...    ...    ...    ...  \n",
            "111295  0.879 -0.059  1.027 -0.043  \n",
            "111296  0.879 -0.059  1.027 -0.043  \n",
            "111297  0.871 -0.059  1.027 -0.043  \n",
            "111298  0.875 -0.059  1.027 -0.043  \n",
            "111299  0.875 -0.059  1.027 -0.043  \n",
            "\n",
            "[111300 rows x 13 columns]\n",
            "       time_s   lw_x   lw_y   lw_z   lh_x   lh_y   lh_z   la_x   la_y   la_z  \\\n",
            "0        0.01 -0.004  0.949 -0.426  0.000  0.000  0.000  1.000  0.359 -0.191   \n",
            "1        0.02 -0.477 -0.117  0.758 -0.398 -0.047  1.180 -0.055 -0.441  0.926   \n",
            "2        0.03 -0.547 -0.637  1.016 -0.543  0.449  1.484 -0.254 -0.828  1.332   \n",
            "3        0.04 -0.445 -0.742  0.301 -0.500  0.730  0.801 -0.195 -0.875  0.660   \n",
            "4        0.05 -0.406 -0.801  0.000 -0.461  0.836  0.512 -0.176 -0.910  0.363   \n",
            "...       ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
            "94195  941.96 -0.348 -0.832 -0.109  0.000  0.000  0.000 -0.012 -0.988  0.094   \n",
            "94196  941.97 -0.348 -0.871 -0.109  0.000  0.000  0.000 -0.012 -0.992  0.094   \n",
            "94197  941.98 -0.348 -0.898 -0.109  0.000  0.000  0.000 -0.008 -0.988  0.102   \n",
            "94198  941.99 -0.363 -0.910 -0.176  0.000  0.000  0.000 -0.008 -0.977  0.109   \n",
            "94199  942.00 -0.465 -0.930 -0.250  0.000  0.000  0.000 -0.008 -0.973  0.117   \n",
            "\n",
            "        ra_x   ra_y   ra_z  \n",
            "0      0.090  0.992 -0.098  \n",
            "1     -0.348  0.348  0.773  \n",
            "2     -0.336  0.676  1.078  \n",
            "3     -0.219  0.906  0.395  \n",
            "4     -0.172  0.977  0.098  \n",
            "...      ...    ...    ...  \n",
            "94195 -0.172  0.996 -0.195  \n",
            "94196 -0.176  0.992 -0.203  \n",
            "94197 -0.176  0.988 -0.195  \n",
            "94198 -0.180  0.992 -0.203  \n",
            "94199 -0.180  0.992 -0.211  \n",
            "\n",
            "[94200 rows x 13 columns]\n",
            "         time_s   lw_x   lw_y   lw_z   lh_x   lh_y   lh_z   la_x   la_y  \\\n",
            "0          0.01  0.020  0.953 -0.367  0.012  0.938 -0.453 -0.035  1.000   \n",
            "1          0.02 -0.652  0.004  1.176 -0.336  0.223  1.156 -0.461 -0.180   \n",
            "2          0.03 -1.000  0.035  1.398 -0.184  0.551  1.555 -0.379 -0.730   \n",
            "3          0.04 -1.012  0.160  0.621  0.008  0.785  0.820 -0.211 -0.848   \n",
            "4          0.05 -1.008  0.188  0.277  0.078  0.879  0.508 -0.137 -0.914   \n",
            "...         ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
            "100795  1007.96  0.000  0.000  0.000 -0.055  0.906  0.426  0.152  0.031   \n",
            "100796  1007.97  0.000  0.000  0.000 -0.055  0.906  0.426  0.180  0.047   \n",
            "100797  1007.98  0.000  0.000  0.000 -0.055  0.906  0.426  0.180  0.105   \n",
            "100798  1007.99  0.000  0.000  0.000 -0.055  0.910  0.430  0.148  0.117   \n",
            "100799  1008.00  0.000  0.000  0.000 -0.059  0.914  0.430  0.137  0.078   \n",
            "\n",
            "         la_z   ra_x   ra_y   ra_z  \n",
            "0      -0.059 -0.941  0.723  0.449  \n",
            "1       0.973 -0.660 -0.176  0.418  \n",
            "2       1.305 -0.441  0.445  0.898  \n",
            "3       0.594 -0.246  0.820  0.363  \n",
            "4       0.273 -0.164  0.945  0.129  \n",
            "...       ...    ...    ...    ...  \n",
            "100795 -0.980  0.684  0.242 -0.734  \n",
            "100796 -1.008  0.684  0.234 -0.738  \n",
            "100797 -0.996  0.684  0.230 -0.730  \n",
            "100798 -0.996  0.688  0.227 -0.730  \n",
            "100799 -0.984  0.684  0.227 -0.734  \n",
            "\n",
            "[100800 rows x 13 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Compute the averaged DQM"
      ],
      "metadata": {
        "id": "lWPmQ4aE3krH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_csv_file_dqm = DQM_multiple_record()\n",
        "multi_csv_file_dqm.set_input_data(csv_df_list)\n",
        "multi_csv_file_dqm.compute_avg_DQM()"
      ],
      "metadata": {
        "id": "JpC_HUKW3nWr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7120c2f4-da7a-45ab-dd22-005da6495bbf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Start computing the DQM... This may take a long time if the dataset is large\n",
            "The total time for computing the DQM is: 553.0278580188751 seconds.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Display the results"
      ],
      "metadata": {
        "id": "jpntSx4L3sas"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_csv_file_dqm.get_avg_fields()"
      ],
      "metadata": {
        "id": "W78oNtTf3uPq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ecb628c9-2763-45ab-e7cc-191ee32afce0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['IRLR', 'SNR', 'SCR', 'RLC', 'VRC', 'SRC', 'MDR', 'APD', 'VDR']"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_csv_file_dqm.get_avg_DQM()"
      ],
      "metadata": {
        "id": "Sv57AayR3waH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c8d1af83-7e2b-421e-8bf9-d73abcc6aa62"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['1.0',\n",
              " '-3.3257782415997195',\n",
              " '1.0',\n",
              " '0.9655312574226851',\n",
              " '0.9742128674915318',\n",
              " '0.999999999998742',\n",
              " '0.0',\n",
              " '0.011856376802747431',\n",
              " '1.0']"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Compute the individual DQM for each record"
      ],
      "metadata": {
        "id": "QOZlP6Al3zsL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_csv_file_dqm.compute_individual_DQM()"
      ],
      "metadata": {
        "id": "e38qDAjn4Dqz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc13d443-6ddc-4809-f8e8-66266bb4cba1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Start computing the DQM... This may take a long time if the dataset is large\n",
            "The total time for computing the DQM is: 554.7631962299347 seconds.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Display the results"
      ],
      "metadata": {
        "id": "K4y43b_Q4G6P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "multi_csv_file_dqm.get_individual_fields()"
      ],
      "metadata": {
        "id": "_mVsehLM4J-r",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6921eab2-127c-4a2b-c21b-880226a613bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['IRLR', 'SNR', 'SRC', 'MDR', 'APD', 'VDR']"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_csv_file_dqm.get_individual_DQM()"
      ],
      "metadata": {
        "id": "K1t0lREV4NTb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "be8d9260-df16-4896-b57e-2c3b33ac6839"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['1',\n",
              "  '-3.436961042075442',\n",
              "  '1.0',\n",
              "  '0.9999999999986589',\n",
              "  '0.0',\n",
              "  '0.001949685534591195'],\n",
              " ['1',\n",
              "  '-3.390101218959695',\n",
              "  '1.0',\n",
              "  '0.9999999999987961',\n",
              "  '0.0',\n",
              "  '0.01611464968152866'],\n",
              " ['1',\n",
              "  '-3.1502724637640216',\n",
              "  '1.0',\n",
              "  '0.9999999999987712',\n",
              "  '0.0',\n",
              "  '0.017192460317460318']]"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    }
  ]
}