HDBSCAN_OutlierLabel_assign.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HDBSCAN_OutlierLabel_assign",
      "provenance": [],
      "authorship_tag": "ABX9TyMcnzU+KHqxTJ6sNxVDDGuA",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/NeuralClassifier/OutlierDetect/blob/main/HDBSCAN_OutlierLabel_assign.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "5jKpCNSrSk84"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "from IPython.display import clear_output\n",
        "import matplotlib.pyplot as plt\n",
        "import os as oo"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip /content/reduced_synthetic_data_local_global.zip"
      ],
      "metadata": {
        "id": "i3INQQvLTAQr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip /content/outlierScore_perData_withNoise.zip"
      ],
      "metadata": {
        "id": "do3rpxIoxEfS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "backbone_path = '/content/reduced_synthetic_data_local_global/'\n",
        "redc = [30,50,70,90]\n",
        "over = [1,2,3,4]\n",
        "dim = [2,10,20,40,80,100]\n",
        "for r in redc:\n",
        "  redc_path = backbone_path + str(r)+'p/outlierScore_perData/refined/'\n",
        "  for o in over:\n",
        "    over_path = redc_path + str(o)+'/'\n",
        "    for d in dim:\n",
        "      clear_output(wait=True)\n",
        "      final_path = over_path+'OS_EachPt_mpts_o'+str(o)+'d'+str(d)+'r'+str(r)+'_refined.xlsx'\n",
        "      print('Reduction: ',r,', Overlap: ',o,', Dimension: ',d)\n",
        "      df = pd.read_excel(final_path)\n",
        "      df.drop(['Unnamed: 0'],axis=1,inplace=True)\n",
        "      #thres = int(input('Enter the threshold: '))\n",
        "      min_pts = list(df.mpts.unique())\n",
        "      for mp in min_pts:\n",
        "        df_temp = df[df['mpts']==mp]\n",
        "        os = np.array(df_temp['outlier score'])\n",
        "        threshold = pd.Series(os).quantile(0.92)\n",
        "        sns.distplot(os[np.isfinite(os)], rug=True)\n",
        "        plt.show()\n",
        "        lbs = []\n",
        "        for i in range(len(df_temp)):\n",
        "          if df_temp.iloc[i,-1] > threshold:\n",
        "            lbs.append(1)\n",
        "          else:\n",
        "            lbs.append(0)\n",
        "        lbs=pd.DataFrame(np.array(lbs),columns=['HDBSCAN_outliers'])\n",
        "        df_temp=pd.concat([df_temp,lbs],axis=1)\n",
        "        if mp == 2:\n",
        "          df_final = df_temp\n",
        "        else:\n",
        "          df_final = pd.concat([df_final,df_temp],axis=0)\n",
        "      df_final.to_excel('/content/HDBSCAN_outlier/'+str(r)+'p/'+str(o)+'/'+'OS_EachPt_mpts_o'+str(o)+'d'+str(d)+'r'+str(r)+'_hdbLabls.xlsx')\n"
      ],
      "metadata": {
        "id": "G5mCG8lbru3E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r /content/HDBSCAN_outlier.zip /content/HDBSCAN_outlier"
      ],
      "metadata": {
        "id": "PW4qxdkd1GK4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf /content/HDBSCAN_outlier"
      ],
      "metadata": {
        "id": "6Br0YWBgOEUl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "mR22p5MRhZVq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "8e6vUvARxPl4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "hG1hX1DexPpO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def seg_RealData(df,lim):\n",
        "  for i in range(lim+1):\n",
        "    df_temp = df[df['data index']==i]\n",
        "    if i ==0:\n",
        "      df_final = df_temp \n",
        "    else:\n",
        "      df_final = pd.concat([df_final,df_temp],axis=0)\n",
        "  return df_final\n",
        "\n",
        "backbone_path = '/content/outlierScore_perData_withNoise/'\n",
        "redc = [70]\n",
        "over = [4]\n",
        "dim = [2,10,20,40,80,100]\n",
        "noise = [20,30,40,50]\n",
        "\n",
        "oo.mkdir('/content/HDBSCAN_outlier_noise/')\n",
        "\n",
        "for r in redc:\n",
        "  #redc_path = backbone_path + str(r)+'p/outlierScore_perData/refined/'\n",
        "  for o in over:\n",
        "    #over_path = redc_path + str(o)+'/'\n",
        "    for d in dim:\n",
        "      for n in noise:\n",
        "        clear_output(wait=True)\n",
        "        final_path = backbone_path+'OS_EachPt_mpts_od'+str(d)+'r'+str(r)+'n'+str(n)+'.xlsx'\n",
        "        print('Reduction: ',r,', Overlap: ',o,', Dimension: ',d,', Noise: ',n)\n",
        "        df = pd.read_excel(final_path)\n",
        "        df.drop(['Unnamed: 0'],axis=1,inplace=True)\n",
        "        df = seg_RealData(df,339)\n",
        "        df.reset_index(drop=True, inplace=True)\n",
        "        #thres = int(input('Enter the threshold: '))\n",
        "        min_pts = list(df.mpts.unique())\n",
        "        for mp in min_pts:\n",
        "          df_temp = df[df['mpts']==mp]\n",
        "          df_temp.reset_index(drop=True, inplace=True)\n",
        "          print(df_temp)\n",
        "          os = np.array(df_temp['outlier score'])\n",
        "          threshold = pd.Series(os).quantile(0.92)\n",
        "          #sns.distplot(os[np.isfinite(os)], rug=True)\n",
        "          #plt.show()\n",
        "          lbs = []\n",
        "          for i in range(len(df_temp)):\n",
        "            if df_temp.iloc[i,-1] > threshold:\n",
        "              lbs.append(1)\n",
        "            else:\n",
        "              lbs.append(0)\n",
        "          lbs=pd.DataFrame(np.array(lbs),columns=['HDBSCAN_outliers'])\n",
        "          df_temp=pd.concat([df_temp,lbs],axis=1)\n",
        "          if mp == 2:\n",
        "            df_final = df_temp\n",
        "          else:\n",
        "            df_final = pd.concat([df_final,df_temp],axis=0)\n",
        "            df_final.reset_index(drop=True, inplace=True)\n",
        "          print(df_final)\n",
        "        df_final.to_excel('/content/HDBSCAN_outlier_noise/'+'OS_EachPt_mpts_o'+str(o)+'d'+str(d)+'r'+str(r)+'n'+str(n)+'_hdbLabls.xlsx')\n"
      ],
      "metadata": {
        "id": "FOHCKLwSxPsV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r /content/HDBSCAN_outlier_noise.zip /content/HDBSCAN_outlier_noise"
      ],
      "metadata": {
        "id": "ejbZ2L3rzgLH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_final"
      ],
      "metadata": {
        "id": "UJ5-P7oBXz0H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "117JX6Qic14R"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}