Initial_Screening_Layer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Monirah_Code (1).ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uKuI8m7p6o8n"
      },
      "source": [
        "# Initial Screening Layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "47n_LTYl6s0n"
      },
      "source": [
        "import pandas as pd\n",
        "jf=pd.read_csv('/content/Jobs.csv',encoding = \"ISO-8859-1\")\n",
        "cf=pd.read_csv('/content/Candidate.csv',encoding = \"ISO-8859-1\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0S4SEsMFWiMK",
        "outputId": "5d88f072-f980-4fd0-add2-c54f9da11966"
      },
      "source": [
        "jf.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 2002 entries, 0 to 2001\n",
            "Data columns (total 12 columns):\n",
            " #   Column            Non-Null Count  Dtype  \n",
            "---  ------            --------------  -----  \n",
            " 0   Job ID            0 non-null      float64\n",
            " 1   Date              0 non-null      float64\n",
            " 2   Company Name      1989 non-null   object \n",
            " 3   Post              1989 non-null   object \n",
            " 4   Location          1989 non-null   object \n",
            " 5   Description       1989 non-null   object \n",
            " 6   Level             1989 non-null   object \n",
            " 7   Type              1989 non-null   object \n",
            " 8   Function          1989 non-null   object \n",
            " 9   Industry          1989 non-null   object \n",
            " 10  Responsibilities  1989 non-null   object \n",
            " 11  Qualifications    1989 non-null   object \n",
            "dtypes: float64(2), object(10)\n",
            "memory usage: 187.8+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p8q9hOiSWuaG",
        "outputId": "96623bec-d32f-4689-a061-f9d129ac4388"
      },
      "source": [
        "cf.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 16170 entries, 0 to 16169\n",
            "Data columns (total 12 columns):\n",
            " #   Column           Non-Null Count  Dtype  \n",
            "---  ------           --------------  -----  \n",
            " 0   Unnamed: 0       16170 non-null  int64  \n",
            " 1   ID               16170 non-null  int64  \n",
            " 2   About            16080 non-null  object \n",
            " 3   Accomplishments  11870 non-null  object \n",
            " 4   Education        15900 non-null  object \n",
            " 5   Experience       15800 non-null  object \n",
            " 6   Industry         0 non-null      float64\n",
            " 7   Level            16170 non-null  object \n",
            " 8   Location         16170 non-null  object \n",
            " 9   Skills           15860 non-null  object \n",
            " 10  Type             16170 non-null  object \n",
            " 11  Certification    8020 non-null   object \n",
            "dtypes: float64(1), int64(2), object(9)\n",
            "memory usage: 1.5+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cdPMpDeqWw2s"
      },
      "source": [
        "# dropping duplicates\n",
        "#jf=jf.drop_duplicates(keep='first',subset='Description')\n",
        "#cf=cf.drop_duplicates(keep='first',subset='Education')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KnAsTOE9MtlE"
      },
      "source": [
       
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EeQW0yLnX4id"
      },
      "source": [
        "# Creating courpus for generating TFIDF vectors (Stacking of jobs and candidates together)\n",
        "corpus=[]\n",
        "J_ids=[]\n",
        "C_ids=[]\n",
        "for i in range(len(jf)):\n",
        "  corpus.append(str(jf.iloc[i,4])+' '+str(jf.iloc[i,6])+' '+str(jf.iloc[i,7]))      # taking collumns of level, type and location\n",
        "  J_ids.append('J'+' '+str(i))\n",
        "for i in range(len(cf)):\n",
        "  corpus.append(str(cf.iloc[i,7])+' '+str(cf.iloc[i,8])+' '+str(cf.iloc[i,10]))      # taking collumns of level, type and location\n",
        "  C_ids.append('C'+' '+str(i))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s1kUT4ZgOXhr"
      },
      "source": [
   
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6jFO48CGXdvG",
        "outputId": "99249bbc-32b4-46bf-f766-4a9199e5e0c5"
      },
      "source": [
        "# Code for TFIDF vectorization, these vectors will be fed in birch algorithm\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "vectorizer = TfidfVectorizer()\n",
        "X = vectorizer.fit_transform(corpus)\n",
        "print(vectorizer.get_feature_names())\n",
        "print(X.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['abha', 'al', 'applicable', 'associate', 'contract', 'dammam', 'director', 'entry', 'executive', 'full', 'internship', 'jeddah', 'jizan', 'jubail', 'khobar', 'level', 'madinah', 'makkah', 'mid', 'nan', 'non', 'not', 'riyadh', 'senior', 'tabuk', 'time']\n",
            "(18172, 26)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mMSbV1i3ZCzb"
      },
      "source": [
        "#Code for birch clustering\n",
        "from sklearn.cluster import Birch\n",
        "brc = Birch(n_clusters=None) # n_cluster is a hyperparameter, which needs to be decded on basis of size of corpus\n",
        "brc.fit(X)                   # set n_clusters=None to return subclusters as it is, change n_clusters to some number to perform final clustering step\n",
        "i_cluster=brc.predict(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k3rDYnXkXdaw"
      },
      "source": [
        "ids=J_ids+C_ids"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "D2YPgtpnd1Gq",
        "outputId": "94e42b1a-6004-4935-8067-c90429a26d57"
      },
      "source": [
        "# output dataset, contating job/candidate id and their cluster number.\n",
        "cluster_dataframe=pd.DataFrame(list(zip(ids,i_cluster)),columns=['ID','Cluster number'])\n",
        "cluster_dataframe"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>Cluster number</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>J 0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>J 1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>J 2</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>J 3</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>J 4</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18167</th>\n",
              "      <td>C 16165</td>\n",
              "      <td>29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18168</th>\n",
              "      <td>C 16166</td>\n",
              "      <td>29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18169</th>\n",
              "      <td>C 16167</td>\n",
              "      <td>29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18170</th>\n",
              "      <td>C 16168</td>\n",
              "      <td>29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18171</th>\n",
              "      <td>C 16169</td>\n",
              "      <td>29</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>18172 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            ID  Cluster number\n",
              "0          J 0               0\n",
              "1          J 1               0\n",
              "2          J 2               0\n",
              "3          J 3               0\n",
              "4          J 4               0\n",
              "...        ...             ...\n",
              "18167  C 16165              29\n",
              "18168  C 16166              29\n",
              "18169  C 16167              29\n",
              "18170  C 16168              29\n",
              "18171  C 16169              29\n",
              "\n",
              "[18172 rows x 2 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F6Fdhb7ZXg1q"
      },
      "source": [
        "cluster_dataframe.to_excel('Clustered_dataframe.xlsx')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E3PGc2pWXo68"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}