data_investigate.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "data_investigate.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6CxZK_3TZt9s",
        "colab_type": "text"
      },
      "source": [
        "## Task 1: **Basic Industry Analysis**\n",
        "\n",
        "Analyze data from industry level and year trend"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CLIpl_f6vRSH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import re\n",
        "import sys\n",
        "from collections import defaultdict\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from scipy.stats import norm\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f2G8LsX6wZR_",
        "colab_type": "code",
        "outputId": "c84448d8-be6a-4fdb-c318-85d75aba1715",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lVueRCc0UJLM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def read_data(root_dir, data_folder, csv_file):\n",
        "    print(\"\\nReading data from \" + csv_file)\n",
        "    file_dir = os.path.join(root_dir, data_folder, csv_file)\n",
        "    return pd.read_csv(file_dir)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MPAMWapBUOL-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def compute_corelation(columns, src_dataframe):\n",
        "    print(\"Computing Correlation matrix\")\n",
        "    return np.corrcoef(src_dataframe[columns].to_numpy(), rowvar = False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yRs_dQj0UShC",
        "colab_type": "code",
        "outputId": "ad28ddce-7a8d-483a-c541-adef8e8a36ee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "'''\n",
        "Basic environ params\n",
        "'''\n",
        "# root_dir = \"\\\\\".join(os.path.dirname(__file__).split('\\\\')[:-1])\n",
        "root_dir = '/content/drive/My Drive/Penn Inequality Project'\n",
        "data_folder = \"reduced_paynet_data\"\n",
        "task_folder = \"task1\"\n",
        "print('----------Analysis Starts----------\\n')\n",
        "\n",
        "data_dir = os.path.join(root_dir, data_folder)\n",
        "all_df = None\n",
        "all_annual_df = None\n",
        "all_annual_plus_df = None\n",
        "all_data_df = None\n",
        "cash_df = None\n",
        "\n",
        "'''\n",
        "Adjustable params\n",
        "'''\n",
        "total_analyzed = False\n",
        "year_analyzed = False\n",
        "sector_analyzed = False\n",
        "sector_annual_analyzed = False"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------Analysis Starts----------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fnK7hobQYCXT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''\n",
        "Data import\n",
        "'''\n",
        "all_df = pd.read_csv(os.path.join(root_dir, 'reclassified_all_data.csv'))\n",
        "all_data_df = all_df.drop(columns = ['Benefit Values', 'Fixed Annual Remuneration', 'Total Earnings', 'Long Term Incentive Values', 'Short Term Variable Payments', 'Target Incentive Payment (%)']).dropna()\n",
        "all_annual_df = all_df[['CalendarYear','IndustryName','Total Annual Remuneration']].dropna()\n",
        "all_annual_plus_df = all_df[['CalendarYear','IndustryName','Total Remuneration Plus']].dropna()\n",
        "cash_df = all_df[['CalendarYear','IndustryName','Total Cash']].dropna()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "npljg6T_VG1F",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "bd63c6f4-b511-4ab1-feb1-dce29269e37f"
      },
      "source": [
        "'''\n",
        "Total analysis\n",
        "'''\n",
        "if not total_analyzed:\n",
        "# correlation analysis\n",
        "    all_columns = ['Base Salary', 'Total Annual Remuneration', 'Total Cash', 'Total Direct Compensation', 'Total Remuneration Plus']\n",
        "    corelation_matrix = compute_corelation(all_columns, all_data_df)\n",
        "    print(corelation_matrix)\n",
        "    np.save(os.path.join(root_dir, task_folder, 'cor_mat.npy'), corelation_matrix)\n",
        "\n",
        "    # integrety analysis\n",
        "    temp = 'Total Annual Remuneration'\n",
        "    plt.title(temp + \" for all integrated data\")\n",
        "    plt.ylabel(\"Percentage per range\")\n",
        "    plt.xlabel('Range in logarithmic dollar')\n",
        "    logit_data = np.log10(all_data_df[temp].tolist())\n",
        "    logit_mean = np.mean(logit_data)\n",
        "    logit_std = np.std(logit_data)\n",
        "    n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "    y = norm.pdf(bins, logit_mean, logit_std)\n",
        "    plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "    plt.xlim(left = 0, right = 10)\n",
        "    plt.grid(True)\n",
        "    plt.legend()\n",
        "    # plt.show()\n",
        "    plt.savefig(os.path.join(root_dir, task_folder, \"all_data_\" + temp))\n",
        "    plt.clf()\n",
        "\n",
        "\n",
        "    temp = 'Total Remuneration Plus'\n",
        "    plt.title(temp + \" for all integrated data\")\n",
        "    plt.ylabel(\"Percentage per range\")\n",
        "    plt.xlabel('Range in logarithmic dollar')\n",
        "    logit_data = np.log10(all_data_df[temp].tolist())\n",
        "    logit_mean = np.mean(logit_data)\n",
        "    logit_std = np.std(logit_data)\n",
        "    n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "    y = norm.pdf(bins, logit_mean, logit_std)\n",
        "    plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "    plt.xlim(left = 0, right = 10)\n",
        "    plt.grid(True)\n",
        "    plt.legend()\n",
        "    # plt.show()\n",
        "    plt.savefig(os.path.join(root_dir, task_folder, \"all_data_\" + temp))\n",
        "    plt.clf()\n",
        "\n",
        "\n",
        "    temp = 'Total Cash'\n",
        "    plt.title(temp + \" for all integrated data\")\n",
        "    plt.ylabel(\"Percentage per range\")\n",
        "    plt.xlabel('Range in logarithmic dollar')\n",
        "    logit_data = np.log10(cash_df[temp].tolist())\n",
        "    logit_mean = np.mean(logit_data)\n",
        "    logit_std = np.std(logit_data)\n",
        "    n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "    y = norm.pdf(bins, logit_mean, logit_std)\n",
        "    plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "    plt.xlim(left = 0, right = 10)\n",
        "    plt.grid(True)\n",
        "    plt.legend()\n",
        "    # plt.show()\n",
        "    plt.savefig(os.path.join(root_dir, task_folder, \"all_data_\" + temp))\n",
        "    plt.clf()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Computing Correlation matrix\n",
            "[[1.         0.9184407  0.91587359 0.69832747 0.73210701]\n",
            " [0.9184407  1.         0.99241205 0.83236358 0.86259628]\n",
            " [0.91587359 0.99241205 1.         0.83807289 0.85956991]\n",
            " [0.69832747 0.83236358 0.83807289 1.         0.99624224]\n",
            " [0.73210701 0.86259628 0.85956991 0.99624224 1.        ]]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RyzBSxX9VZnc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "b6069840-f38e-4b64-e3ff-be4982ed9439"
      },
      "source": [
        "if not year_analyzed:\n",
        "    # year level analysis\n",
        "    all_year = pd.unique(all_data_df['CalendarYear']).tolist()\n",
        "    all_year = range(min(all_year), max(all_year) + 1)\n",
        "    year_mean = defaultdict(list)\n",
        "\n",
        "    for curr_year in all_year:\n",
        "        temp = 'Total Annual Remuneration'\n",
        "        year_data_df = all_annual_df[all_annual_df['CalendarYear'] == curr_year]\n",
        "        print(str(year_data_df.shape[0]) + ' data pieces chosen for year ' + str(curr_year) + ' on ' + temp)\n",
        "        if year_data_df.shape[0] > 0:\n",
        "            plt.title(temp + \" for yearly integrated data\")\n",
        "            plt.ylabel(\"Percentage per range\")\n",
        "            plt.xlabel('Range in logarithmic dollar')\n",
        "            logit_data = np.log10(year_data_df[temp].tolist())\n",
        "            logit_mean = np.mean(logit_data)\n",
        "            year_mean[temp].append(np.mean(year_data_df[temp].tolist()))\n",
        "            logit_std = np.std(logit_data)\n",
        "            n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "            y = norm.pdf(bins, logit_mean, logit_std)\n",
        "            plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "            plt.xlim(left = 0, right = 10)\n",
        "            plt.grid(True)\n",
        "            plt.legend()\n",
        "            # plt.show()\n",
        "            if not os.path.isdir(os.path.join(root_dir, task_folder, str(curr_year))):\n",
        "                os.mkdir(os.path.join(root_dir, task_folder, str(curr_year)))\n",
        "            plt.savefig(os.path.join(root_dir, task_folder, str(curr_year), \"year_data_\" + temp))\n",
        "            plt.clf()\n",
        "        else:\n",
        "            year_mean[temp].append(0)\n",
        "\n",
        "\n",
        "        temp = 'Total Remuneration Plus'\n",
        "        year_data_df = all_annual_plus_df[all_annual_plus_df['CalendarYear'] == curr_year]\n",
        "        print(str(year_data_df.shape[0]) + ' data pieces chosen for year ' + str(curr_year) + ' on ' + temp)\n",
        "        if year_data_df.shape[0] > 0:\n",
        "            plt.title(temp + \" for yearly integrated data\")\n",
        "            plt.ylabel(\"Percentage per range\")\n",
        "            plt.xlabel('Range in logarithmic dollar')\n",
        "            logit_data = np.log10(year_data_df[temp].tolist())\n",
        "            logit_mean = np.mean(logit_data)\n",
        "            year_mean[temp].append(np.mean(year_data_df[temp].tolist()))\n",
        "            logit_std = np.std(logit_data)\n",
        "            n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "            y = norm.pdf(bins, logit_mean, logit_std)\n",
        "            plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "            plt.xlim(left = 0, right = 10)\n",
        "            plt.grid(True)\n",
        "            plt.legend()\n",
        "            # plt.show()\n",
        "            if not os.path.isdir(os.path.join(root_dir, task_folder, str(curr_year))):\n",
        "                os.mkdir(os.path.join(root_dir, task_folder, str(curr_year)))\n",
        "            plt.savefig(os.path.join(root_dir, task_folder, str(curr_year), \"year_data_\" + temp))\n",
        "            plt.clf()\n",
        "        else:\n",
        "            year_mean[temp].append(0)\n",
        "\n",
        "\n",
        "        temp = 'Total Cash'\n",
        "        year_data_df = cash_df[cash_df['CalendarYear'] == curr_year]\n",
        "        print(str(year_data_df.shape[0]) + ' data pieces chosen for year ' + str(curr_year) + ' on ' + temp)\n",
        "        plt.title(temp + \" for yearly integrated data\")\n",
        "        plt.ylabel(\"Percentage per range\")\n",
        "        plt.xlabel('Range in logarithmic dollar')\n",
        "        logit_data = np.log10(year_data_df[temp].tolist())\n",
        "        logit_mean = np.mean(logit_data)\n",
        "        year_mean[temp].append(np.mean(year_data_df[temp].tolist()))\n",
        "        logit_std = np.std(logit_data)\n",
        "        n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "        y = norm.pdf(bins, logit_mean, logit_std)\n",
        "        plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "        plt.xlim(left = 0, right = 10)\n",
        "        plt.grid(True)\n",
        "        plt.legend()\n",
        "        # plt.show()\n",
        "        if not os.path.isdir(os.path.join(root_dir, task_folder, str(curr_year))):\n",
        "            os.mkdir(os.path.join(root_dir, task_folder, str(curr_year)))\n",
        "        plt.savefig(os.path.join(root_dir, task_folder, str(curr_year), \"year_data_\" + temp))\n",
        "        plt.clf()\n",
        "        print(str(curr_year) + ' year analysis finished\\n')\n",
        "\n",
        "\n",
        "    # linear regression of time with three variables\n",
        "    plt.title('Total year trend')\n",
        "    plt.xlabel('Year span')\n",
        "    plt.ylabel('Range in linear dollar')\n",
        "    plt.xlim(min(all_year) - 1, max(all_year) + 1)\n",
        "    plt.plot(all_year, year_mean['Total Annual Remuneration'], label = \"Total Annual Remuneration\")\n",
        "    plt.plot(all_year, year_mean['Total Remuneration Plus'], label = 'Total Remuneration Plus')\n",
        "    plt.plot(all_year, year_mean['Total Cash'], label = 'Total Cash')\n",
        "    plt.legend()\n",
        "    plt.savefig(os.path.join(root_dir, task_folder, 'Year Trend'))\n",
        "    plt.clf()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "436733 data pieces chosen for year 2008 on Total Annual Remuneration\n",
            "436733 data pieces chosen for year 2008 on Total Remuneration Plus\n",
            "800165 data pieces chosen for year 2008 on Total Cash\n",
            "2008 year analysis finished\n",
            "\n",
            "338409 data pieces chosen for year 2009 on Total Annual Remuneration\n",
            "338409 data pieces chosen for year 2009 on Total Remuneration Plus\n",
            "891611 data pieces chosen for year 2009 on Total Cash\n",
            "2009 year analysis finished\n",
            "\n",
            "371154 data pieces chosen for year 2010 on Total Annual Remuneration\n",
            "371154 data pieces chosen for year 2010 on Total Remuneration Plus\n",
            "991692 data pieces chosen for year 2010 on Total Cash\n",
            "2010 year analysis finished\n",
            "\n",
            "354197 data pieces chosen for year 2011 on Total Annual Remuneration\n",
            "354197 data pieces chosen for year 2011 on Total Remuneration Plus\n",
            "993702 data pieces chosen for year 2011 on Total Cash\n",
            "2011 year analysis finished\n",
            "\n",
            "0 data pieces chosen for year 2012 on Total Annual Remuneration\n",
            "0 data pieces chosen for year 2012 on Total Remuneration Plus\n",
            "1492471 data pieces chosen for year 2012 on Total Cash\n",
            "2012 year analysis finished\n",
            "\n",
            "0 data pieces chosen for year 2013 on Total Annual Remuneration\n",
            "0 data pieces chosen for year 2013 on Total Remuneration Plus\n",
            "1331305 data pieces chosen for year 2013 on Total Cash\n",
            "2013 year analysis finished\n",
            "\n",
            "0 data pieces chosen for year 2014 on Total Annual Remuneration\n",
            "0 data pieces chosen for year 2014 on Total Remuneration Plus\n",
            "1576874 data pieces chosen for year 2014 on Total Cash\n",
            "2014 year analysis finished\n",
            "\n",
            "0 data pieces chosen for year 2015 on Total Annual Remuneration\n",
            "0 data pieces chosen for year 2015 on Total Remuneration Plus\n",
            "3757097 data pieces chosen for year 2015 on Total Cash\n",
            "2015 year analysis finished\n",
            "\n",
            "0 data pieces chosen for year 2016 on Total Annual Remuneration\n",
            "0 data pieces chosen for year 2016 on Total Remuneration Plus\n",
            "4787347 data pieces chosen for year 2016 on Total Cash\n",
            "2016 year analysis finished\n",
            "\n",
            "2349608 data pieces chosen for year 2017 on Total Annual Remuneration\n",
            "2344525 data pieces chosen for year 2017 on Total Remuneration Plus\n",
            "4449594 data pieces chosen for year 2017 on Total Cash\n",
            "2017 year analysis finished\n",
            "\n",
            "1883230 data pieces chosen for year 2018 on Total Annual Remuneration\n",
            "1838842 data pieces chosen for year 2018 on Total Remuneration Plus\n",
            "4220185 data pieces chosen for year 2018 on Total Cash\n",
            "2018 year analysis finished\n",
            "\n",
            "1055112 data pieces chosen for year 2019 on Total Annual Remuneration\n",
            "1046226 data pieces chosen for year 2019 on Total Remuneration Plus\n",
            "3093614 data pieces chosen for year 2019 on Total Cash\n",
            "2019 year analysis finished\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "atovKBvkVgIp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "c6876774-5436-4f3e-adde-58c0129307ff"
      },
      "source": [
        "if not sector_analyzed:\n",
        "    all_industry = pd.unique(cash_df['IndustryName']).tolist()\n",
        "    all_year = pd.unique(all_data_df['CalendarYear']).tolist()\n",
        "    all_year = range(min(all_year), max(all_year) + 1)\n",
        "    for curr_industry in all_industry:\n",
        "        industry = re.sub(r'\\s+', ' ', curr_industry)\n",
        "        if not os.path.isdir(os.path.join(root_dir, task_folder, industry)):\n",
        "            os.mkdir(os.path.join(root_dir, task_folder, industry))\n",
        "            \n",
        "        industry_data_df = all_data_df[all_data_df['IndustryName'] == curr_industry]\n",
        "        industry_remuneration_df = all_annual_df[all_annual_df['IndustryName'] == curr_industry]\n",
        "        industry_remuneration_plus_df = all_annual_plus_df[all_annual_plus_df['IndustryName'] == curr_industry]\n",
        "        industry_cash_df = cash_df[cash_df['IndustryName'] == curr_industry]\n",
        "\n",
        "        all_columns = ['Base Salary', 'Total Annual Remuneration', 'Total Cash', 'Total Direct Compensation', 'Total Remuneration Plus']\n",
        "        if industry_data_df.shape[0] > 0:\n",
        "            corelation_matrix = compute_corelation(all_columns, industry_data_df)\n",
        "            print(corelation_matrix)\n",
        "            np.save(os.path.join(root_dir, task_folder, industry, 'cor_mat.npy'), corelation_matrix)\n",
        "\n",
        "\n",
        "        if industry_remuneration_df.shape[0] > 0:\n",
        "            temp = 'Total Annual Remuneration'\n",
        "            plt.title(temp + \" for all data in \" + industry)\n",
        "            plt.ylabel(\"Percentage per range\")\n",
        "            plt.xlabel('Range in logarithmic dollar')\n",
        "            print(str(industry_remuneration_df[temp].shape[0]) + ' data pieces chosen for ' + industry + ' on ' + temp)\n",
        "            logit_data = np.log10(industry_remuneration_df[temp].tolist())\n",
        "            logit_mean = np.mean(logit_data)\n",
        "            logit_std = np.std(logit_data)\n",
        "            n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "            y = norm.pdf(bins, logit_mean, logit_std)\n",
        "            plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "            plt.xlim(left = 0, right = 10)\n",
        "            plt.grid(True)\n",
        "            plt.legend()\n",
        "            # plt.show()\n",
        "            plt.savefig(os.path.join(root_dir, task_folder, industry, \"all_data_\" + temp))\n",
        "            plt.clf()\n",
        "\n",
        "\n",
        "        if industry_remuneration_plus_df.shape[0] > 0:\n",
        "            temp = 'Total Remuneration Plus'\n",
        "            plt.title(temp + \" for all data in \" + industry)\n",
        "            plt.ylabel(\"Percentage per range\")\n",
        "            plt.xlabel('Range in logarithmic dollar')\n",
        "            print(str(industry_remuneration_plus_df[temp].shape[0]) + ' data pieces chosen for ' + industry + ' on ' + temp)\n",
        "            logit_data = np.log10(industry_remuneration_plus_df[temp].tolist())\n",
        "            logit_mean = np.mean(logit_data)\n",
        "            logit_std = np.std(logit_data)\n",
        "            n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "            y = norm.pdf(bins, logit_mean, logit_std)\n",
        "            plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "            plt.xlim(left = 0, right = 10)\n",
        "            plt.grid(True)\n",
        "            plt.legend()\n",
        "            # plt.show()\n",
        "            plt.savefig(os.path.join(root_dir, task_folder, industry, \"all_data_\" + temp))\n",
        "            plt.clf()\n",
        "\n",
        "\n",
        "        if industry_cash_df.shape[0] > 0:\n",
        "            temp = 'Total Cash'\n",
        "            plt.title(temp + \" for all data in \" + industry)\n",
        "            plt.ylabel(\"Percentage per range\")\n",
        "            plt.xlabel('Range in logarithmic dollar')\n",
        "            print(str(industry_cash_df[temp].shape[0]) + ' data pieces chosen for ' + industry + ' on ' + temp)\n",
        "            logit_data = np.log10(industry_cash_df[temp].tolist())\n",
        "            logit_mean = np.mean(logit_data)\n",
        "            logit_std = np.std(logit_data)\n",
        "            n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "            y = norm.pdf(bins, logit_mean, logit_std)\n",
        "            plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "            plt.xlim(left = 0, right = 10)\n",
        "            plt.grid(True)\n",
        "            plt.legend()\n",
        "            # plt.show()\n",
        "            plt.savefig(os.path.join(root_dir, task_folder, industry, \"all_data_\" + temp))\n",
        "            plt.clf()\n",
        "            print(curr_industry + ' sector analysis finished\\n')\n",
        "\n",
        "    "
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Computing Correlation matrix\n",
            "[[1.         0.90395786 0.90291244 0.66753586 0.69395849]\n",
            " [0.90395786 1.         0.99283417 0.81362939 0.83725524]\n",
            " [0.90291244 0.99283417 1.         0.81297715 0.82938614]\n",
            " [0.66753586 0.81362939 0.81297715 1.         0.9973469 ]\n",
            " [0.69395849 0.83725524 0.82938614 0.9973469  1.        ]]\n",
            "5051392 data pieces chosen for specialty retail on Total Annual Remuneration\n",
            "5004378 data pieces chosen for specialty retail on Total Remuneration Plus\n",
            "18032842 data pieces chosen for specialty retail on Total Cash\n",
            "specialty retail sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.8890719  0.89707538 0.77917896 0.79936239]\n",
            " [0.8890719  1.         0.99236901 0.89335251 0.91981227]\n",
            " [0.89707538 0.99236901 1.         0.89911944 0.91633806]\n",
            " [0.77917896 0.89335251 0.89911944 1.         0.99584021]\n",
            " [0.79936239 0.91981227 0.91633806 0.99584021 1.        ]]\n",
            "69511 data pieces chosen for food product on Total Annual Remuneration\n",
            "68866 data pieces chosen for food product on Total Remuneration Plus\n",
            "314462 data pieces chosen for food product on Total Cash\n",
            "food product sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.96639839 0.9626551  0.93339133 0.9478513 ]\n",
            " [0.96639839 1.         0.99461465 0.96700932 0.98048697]\n",
            " [0.9626551  0.99461465 1.         0.97315458 0.97697453]\n",
            " [0.93339133 0.96700932 0.97315458 1.         0.99492173]\n",
            " [0.9478513  0.98048697 0.97697453 0.99492173 1.        ]]\n",
            "34247 data pieces chosen for insurance on Total Annual Remuneration\n",
            "34225 data pieces chosen for insurance on Total Remuneration Plus\n",
            "165528 data pieces chosen for insurance on Total Cash\n",
            "insurance sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.94569791 0.95683916 0.82372398 0.84546684]\n",
            " [0.94569791 1.         0.98543313 0.9004097  0.93294606]\n",
            " [0.95683916 0.98543313 1.         0.90763309 0.91944564]\n",
            " [0.82372398 0.9004097  0.90763309 1.         0.99057214]\n",
            " [0.84546684 0.93294606 0.91944564 0.99057214 1.        ]]\n",
            "864746 data pieces chosen for healthcare provider and service on Total Annual Remuneration\n",
            "861206 data pieces chosen for healthcare provider and service on Total Remuneration Plus\n",
            "4855639 data pieces chosen for healthcare provider and service on Total Cash\n",
            "healthcare provider and service sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.95050555 0.94715906 0.74152103 0.76534074]\n",
            " [0.95050555 1.         0.99419154 0.84783633 0.8689235 ]\n",
            " [0.94715906 0.99419154 1.         0.85024197 0.8655207 ]\n",
            " [0.74152103 0.84783633 0.85024197 1.         0.99786996]\n",
            " [0.76534074 0.8689235  0.8655207  0.99786996 1.        ]]\n",
            "153858 data pieces chosen for machinery on Total Annual Remuneration\n",
            "152485 data pieces chosen for machinery on Total Remuneration Plus\n",
            "692471 data pieces chosen for machinery on Total Cash\n",
            "machinery sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.91324185 0.91572034 0.76918615 0.79115194]\n",
            " [0.91324185 1.         0.99049556 0.89081562 0.91487908]\n",
            " [0.91572034 0.99049556 1.         0.89794611 0.91063807]\n",
            " [0.76918615 0.89081562 0.89794611 1.         0.99567356]\n",
            " [0.79115194 0.91487908 0.91063807 0.99567356 1.        ]]\n",
            "28935 data pieces chosen for metal and mining on Total Annual Remuneration\n",
            "28923 data pieces chosen for metal and mining on Total Remuneration Plus\n",
            "125892 data pieces chosen for metal and mining on Total Cash\n",
            "metal and mining sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.85938813 0.84324361 0.70759282 0.73179921]\n",
            " [0.85938813 1.         0.99428519 0.84652684 0.86537491]\n",
            " [0.84324361 0.99428519 1.         0.8504058  0.86360339]\n",
            " [0.70759282 0.84652684 0.8504058  1.         0.99812109]\n",
            " [0.73179921 0.86537491 0.86360339 0.99812109 1.        ]]\n",
            "158617 data pieces chosen for chemical on Total Annual Remuneration\n",
            "156506 data pieces chosen for chemical on Total Remuneration Plus\n",
            "797449 data pieces chosen for chemical on Total Cash\n",
            "chemical sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.94098884 0.93689525 0.74671416 0.77216482]\n",
            " [0.94098884 1.         0.99556722 0.84196389 0.86673036]\n",
            " [0.93689525 0.99556722 1.         0.8618074  0.88152462]\n",
            " [0.74671416 0.84196389 0.8618074  1.         0.99808387]\n",
            " [0.77216482 0.86673036 0.88152462 0.99808387 1.        ]]\n",
            "24634 data pieces chosen for pharmaceutical on Total Annual Remuneration\n",
            "24618 data pieces chosen for pharmaceutical on Total Remuneration Plus\n",
            "185081 data pieces chosen for pharmaceutical on Total Cash\n",
            "pharmaceutical sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.89280582 0.88240167 0.6779201  0.70203279]\n",
            " [0.89280582 1.         0.99500231 0.83153199 0.85020078]\n",
            " [0.88240167 0.99500231 1.         0.83734073 0.85083014]\n",
            " [0.6779201  0.83153199 0.83734073 1.         0.99825292]\n",
            " [0.70203279 0.85020078 0.85083014 0.99825292 1.        ]]\n",
            "110250 data pieces chosen for hotel restaurant and leisure on Total Annual Remuneration\n",
            "109815 data pieces chosen for hotel restaurant and leisure on Total Remuneration Plus\n",
            "848026 data pieces chosen for hotel restaurant and leisure on Total Cash\n",
            "hotel restaurant and leisure sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.93037277 0.94213364 0.80788138 0.82302445]\n",
            " [0.93037277 1.         0.98911518 0.91063902 0.93350107]\n",
            " [0.94213364 0.98911518 1.         0.92029083 0.93039435]\n",
            " [0.80788138 0.91063902 0.92029083 1.         0.99545318]\n",
            " [0.82302445 0.93350107 0.93039435 0.99545318 1.        ]]\n",
            "21693 data pieces chosen for container and packaging on Total Annual Remuneration\n",
            "21693 data pieces chosen for container and packaging on Total Remuneration Plus\n",
            "140300 data pieces chosen for container and packaging on Total Cash\n",
            "container and packaging sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.95252238 0.93679386 0.7803741  0.80471851]\n",
            " [0.95252238 1.         0.99165961 0.90789957 0.92155656]\n",
            " [0.93679386 0.99165961 1.         0.89974947 0.90596742]\n",
            " [0.7803741  0.90789957 0.89974947 1.         0.99761451]\n",
            " [0.80471851 0.92155656 0.90596742 0.99761451 1.        ]]\n",
            "26012 data pieces chosen for capital market on Total Annual Remuneration\n",
            "23223 data pieces chosen for capital market on Total Remuneration Plus\n",
            "304391 data pieces chosen for capital market on Total Cash\n",
            "capital market sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.92972598 0.9289182  0.77772918 0.80180762]\n",
            " [0.92972598 1.         0.99492432 0.93655297 0.95353266]\n",
            " [0.9289182  0.99492432 1.         0.94435542 0.95526833]\n",
            " [0.77772918 0.93655297 0.94435542 1.         0.99751073]\n",
            " [0.80180762 0.95353266 0.95526833 0.99751073 1.        ]]\n",
            "8902 data pieces chosen for personal product on Total Annual Remuneration\n",
            "8902 data pieces chosen for personal product on Total Remuneration Plus\n",
            "50252 data pieces chosen for personal product on Total Cash\n",
            "personal product sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.93224907 0.92802524 0.77354255 0.80403442]\n",
            " [0.93224907 1.         0.99222834 0.8835715  0.90962161]\n",
            " [0.92802524 0.99222834 1.         0.88744601 0.90424016]\n",
            " [0.77354255 0.8835715  0.88744601 1.         0.99591214]\n",
            " [0.80403442 0.90962161 0.90424016 0.99591214 1.        ]]\n",
            "99601 data pieces chosen for electric utility on Total Annual Remuneration\n",
            "99540 data pieces chosen for electric utility on Total Remuneration Plus\n",
            "350539 data pieces chosen for electric utility on Total Cash\n",
            "electric utility sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.99721562 0.9963997  0.9963997  0.99735937]\n",
            " [0.99721562 1.         0.99994757 0.99994757 0.99996835]\n",
            " [0.9963997  0.99994757 1.         1.         0.99989185]\n",
            " [0.9963997  0.99994757 1.         1.         0.99989185]\n",
            " [0.99735937 0.99996835 0.99989185 0.99989185 1.        ]]\n",
            "554 data pieces chosen for bank on Total Annual Remuneration\n",
            "554 data pieces chosen for bank on Total Remuneration Plus\n",
            "38711 data pieces chosen for bank on Total Cash\n",
            "bank sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.95810543 0.96452971 0.9517099  0.94955489]\n",
            " [0.95810543 1.         0.99927999 0.99042007 0.99375153]\n",
            " [0.96452971 0.99927999 1.         0.99202889 0.9936352 ]\n",
            " [0.9517099  0.99042007 0.99202889 1.         0.99871751]\n",
            " [0.94955489 0.99375153 0.9936352  0.99871751 1.        ]]\n",
            "3267 data pieces chosen for healthcare equipment and supply on Total Annual Remuneration\n",
            "3267 data pieces chosen for healthcare equipment and supply on Total Remuneration Plus\n",
            "16619 data pieces chosen for healthcare equipment and supply on Total Cash\n",
            "healthcare equipment and supply sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.95471537 0.95300371 0.69107792 0.71728865]\n",
            " [0.95471537 1.         0.99677238 0.82946132 0.85079164]\n",
            " [0.95300371 0.99677238 1.         0.8350955  0.85331506]\n",
            " [0.69107792 0.82946132 0.8350955  1.         0.99864403]\n",
            " [0.71728865 0.85079164 0.85331506 0.99864403 1.        ]]\n",
            "16516 data pieces chosen for road and rail on Total Annual Remuneration\n",
            "16516 data pieces chosen for road and rail on Total Remuneration Plus\n",
            "75048 data pieces chosen for road and rail on Total Cash\n",
            "road and rail sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.94420429 0.93776277 0.85743265 0.87398228]\n",
            " [0.94420429 1.         0.99849579 0.93736525 0.94647843]\n",
            " [0.93776277 0.99849579 1.         0.94126184 0.94775313]\n",
            " [0.85743265 0.93736525 0.94126184 1.         0.99863352]\n",
            " [0.87398228 0.94647843 0.94775313 0.99863352 1.        ]]\n",
            "9069 data pieces chosen for diversified consumer service on Total Annual Remuneration\n",
            "9069 data pieces chosen for diversified consumer service on Total Remuneration Plus\n",
            "52976 data pieces chosen for diversified consumer service on Total Cash\n",
            "diversified consumer service sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.97688776 0.97605174 0.76274728 0.78939923]\n",
            " [0.97688776 1.         0.99919    0.84531473 0.86791226]\n",
            " [0.97605174 0.99919    1.         0.84857426 0.87013616]\n",
            " [0.76274728 0.84531473 0.84857426 1.         0.99882089]\n",
            " [0.78939923 0.86791226 0.87013616 0.99882089 1.        ]]\n",
            "4079 data pieces chosen for construction and engineering on Total Annual Remuneration\n",
            "4077 data pieces chosen for construction and engineering on Total Remuneration Plus\n",
            "21236 data pieces chosen for construction and engineering on Total Cash\n",
            "construction and engineering sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.96046833 0.96272099 0.88311901 0.90047213]\n",
            " [0.96046833 1.         0.99597175 0.88988441 0.91272763]\n",
            " [0.96272099 0.99597175 1.         0.88935114 0.9065025 ]\n",
            " [0.88311901 0.88988441 0.88935114 1.         0.99655593]\n",
            " [0.90047213 0.91272763 0.9065025  0.99655593 1.        ]]\n",
            "48315 data pieces chosen for beverage on Total Annual Remuneration\n",
            "48068 data pieces chosen for beverage on Total Remuneration Plus\n",
            "734081 data pieces chosen for beverage on Total Cash\n",
            "beverage sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.97895473 0.97585523 0.85933207 0.86414747]\n",
            " [0.97895473 1.         0.99878968 0.93127587 0.93483827]\n",
            " [0.97585523 0.99878968 1.         0.93599381 0.93870252]\n",
            " [0.85933207 0.93127587 0.93599381 1.         0.99983315]\n",
            " [0.86414747 0.93483827 0.93870252 0.99983315 1.        ]]\n",
            "1084 data pieces chosen for energy equipment and service on Total Annual Remuneration\n",
            "1071 data pieces chosen for energy equipment and service on Total Remuneration Plus\n",
            "60370 data pieces chosen for energy equipment and service on Total Cash\n",
            "energy equipment and service sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.92918433 0.91872744 0.90957644 0.92487952]\n",
            " [0.92918433 1.         0.99871664 0.99216378 0.99458559]\n",
            " [0.91872744 0.99871664 1.         0.99308314 0.99241269]\n",
            " [0.90957644 0.99216378 0.99308314 1.         0.99784843]\n",
            " [0.92487952 0.99458559 0.99241269 0.99784843 1.        ]]\n",
            "14478 data pieces chosen for aerospace and defense on Total Annual Remuneration\n",
            "14401 data pieces chosen for aerospace and defense on Total Remuneration Plus\n",
            "109775 data pieces chosen for aerospace and defense on Total Cash\n",
            "aerospace and defense sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.90929099 0.90066032 0.72199505 0.7347145 ]\n",
            " [0.90929099 1.         0.99926481 0.8710622  0.8791801 ]\n",
            " [0.90066032 0.99926481 1.         0.87617615 0.88356826]\n",
            " [0.72199505 0.8710622  0.87617615 1.         0.9997368 ]\n",
            " [0.7347145  0.8791801  0.88356826 0.9997368  1.        ]]\n",
            "1200 data pieces chosen for oil gas and consumable fuel on Total Annual Remuneration\n",
            "1200 data pieces chosen for oil gas and consumable fuel on Total Remuneration Plus\n",
            "9655 data pieces chosen for oil gas and consumable fuel on Total Cash\n",
            "oil gas and consumable fuel sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.99272327 0.99335456 0.99335456 0.99254717]\n",
            " [0.99272327 1.         0.99864287 0.99864287 0.9997555 ]\n",
            " [0.99335456 0.99864287 1.         1.         0.99768763]\n",
            " [0.99335456 0.99864287 1.         1.         0.99768763]\n",
            " [0.99254717 0.9997555  0.99768763 0.99768763 1.        ]]\n",
            "4541 data pieces chosen for semiconductor and semiconductor equipment on Total Annual Remuneration\n",
            "4541 data pieces chosen for semiconductor and semiconductor equipment on Total Remuneration Plus\n",
            "158164 data pieces chosen for semiconductor and semiconductor equipment on Total Cash\n",
            "semiconductor and semiconductor equipment sector analysis finished\n",
            "\n",
            "565 data pieces chosen for household durables on Total Annual Remuneration\n",
            "565 data pieces chosen for household durables on Total Remuneration Plus\n",
            "41349 data pieces chosen for household durables on Total Cash\n",
            "household durables sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.99975564 1.         1.         0.99965909]\n",
            " [0.99975564 1.         0.99975564 0.99975564 0.99989488]\n",
            " [1.         0.99975564 1.         1.         0.99965909]\n",
            " [1.         0.99975564 1.         1.         0.99965909]\n",
            " [0.99965909 0.99989488 0.99965909 0.99965909 1.        ]]\n",
            "176 data pieces chosen for equity real estate investment trust reit on Total Annual Remuneration\n",
            "176 data pieces chosen for equity real estate investment trust reit on Total Remuneration Plus\n",
            "9009 data pieces chosen for equity real estate investment trust reit on Total Cash\n",
            "equity real estate investment trust reit sector analysis finished\n",
            "\n",
            "41017 data pieces chosen for diversified telecommunication service on Total Cash\n",
            "diversified telecommunication service sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.95809183 0.97055261 0.81155073 0.81488888]\n",
            " [0.95809183 1.         0.99394555 0.92576098 0.93050611]\n",
            " [0.97055261 0.99394555 1.         0.90470513 0.90477971]\n",
            " [0.81155073 0.92576098 0.90470513 1.         0.99848079]\n",
            " [0.81488888 0.93050611 0.90477971 0.99848079 1.        ]]\n",
            "11635 data pieces chosen for building product on Total Annual Remuneration\n",
            "11635 data pieces chosen for building product on Total Remuneration Plus\n",
            "25400 data pieces chosen for building product on Total Cash\n",
            "building product sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.99817908 0.99780759 0.99780759 0.99862989]\n",
            " [0.99817908 1.         0.99936595 0.99936595 0.99981213]\n",
            " [0.99780759 0.99936595 1.         1.         0.99895852]\n",
            " [0.99780759 0.99936595 1.         1.         0.99895852]\n",
            " [0.99862989 0.99981213 0.99895852 0.99895852 1.        ]]\n",
            "1959 data pieces chosen for professional service on Total Annual Remuneration\n",
            "1959 data pieces chosen for professional service on Total Remuneration Plus\n",
            "38046 data pieces chosen for professional service on Total Cash\n",
            "professional service sector analysis finished\n",
            "\n",
            "Computing Correlation matrix\n",
            "[[1.         0.9704832  0.96600417 0.95548704 0.96236127]\n",
            " [0.9704832  1.         0.99983915 0.98983082 0.99312985]\n",
            " [0.96600417 0.99983915 1.         0.99005451 0.99307941]\n",
            " [0.95548704 0.98983082 0.99005451 1.         0.99953367]\n",
            " [0.96236127 0.99312985 0.99307941 0.99953367 1.        ]]\n",
            "18607 data pieces chosen for electrical equipment on Total Annual Remuneration\n",
            "18607 data pieces chosen for electrical equipment on Total Remuneration Plus\n",
            "87594 data pieces chosen for electrical equipment on Total Cash\n",
            "electrical equipment sector analysis finished\n",
            "\n",
            "3735 data pieces chosen for internet and direct marketing retail on Total Cash\n",
            "internet and direct marketing retail sector analysis finished\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jYG6wJmjWMSz",
        "colab_type": "code",
        "outputId": "6693f870-80e4-4a64-f9bc-65978c716f68",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "if not sector_annual_analyzed:\n",
        "    all_industry = pd.unique(cash_df['IndustryName']).tolist()\n",
        "    all_year = pd.unique(all_data_df['CalendarYear']).tolist()\n",
        "    all_year = range(min(all_year), max(all_year) + 1)\n",
        "    for curr_industry in all_industry:    \n",
        "        year_mean = defaultdict(list) \n",
        "        industry_remuneration_df = all_annual_df[all_annual_df['IndustryName'] == curr_industry]\n",
        "        industry_remuneration_plus_df = all_annual_plus_df[all_annual_plus_df['IndustryName'] == curr_industry]\n",
        "        industry_cash_df = cash_df[cash_df['IndustryName'] == curr_industry]\n",
        "        industry = re.sub(r'\\s+', ' ', curr_industry)\n",
        "        for curr_year in all_year:\n",
        "            if not os.path.isdir(os.path.join(root_dir, task_folder, str(curr_year), industry)):\n",
        "                os.mkdir(os.path.join(root_dir, task_folder, str(curr_year), industry))\n",
        "            industry_year_remuneration_df = industry_remuneration_df[industry_remuneration_df['CalendarYear'] == curr_year]\n",
        "            industry_year_remuneration_plus_df = industry_remuneration_plus_df[industry_remuneration_plus_df['CalendarYear'] == curr_year]\n",
        "            industry_year_cash_df = industry_cash_df[industry_cash_df['CalendarYear'] == curr_year]\n",
        "                \n",
        "            temp = 'Total Annual Remuneration'\n",
        "            plt.title(temp + \" for all data in \" + industry + \" of year \" + str(curr_year))\n",
        "            plt.ylabel(\"Percentage per range\")\n",
        "            plt.xlabel('Range in logarithmic dollar')\n",
        "            if industry_year_remuneration_df[temp].size > 0:\n",
        "                print(str(industry_year_remuneration_df[temp].shape[0]) + ' data pieces chosen for ' + curr_industry + ' of year ' + str(curr_year) + ' on ' + temp)\n",
        "                logit_data = np.log10(industry_year_remuneration_df[temp].tolist())\n",
        "                logit_mean = np.mean(logit_data)\n",
        "                year_mean[temp].append(np.mean(industry_year_remuneration_df[temp].tolist()))\n",
        "                logit_std = np.std(logit_data)\n",
        "                n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "                y = norm.pdf(bins, logit_mean, logit_std)\n",
        "                plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "                plt.xlim(left = 0, right = 10)\n",
        "                plt.grid(True)\n",
        "                plt.legend()\n",
        "                # plt.show()\n",
        "                plt.savefig(os.path.join(root_dir, task_folder, str(curr_year), industry, \"all_data_\" + temp))     \n",
        "            else:\n",
        "                year_mean[temp].append(0)\n",
        "            plt.clf()\n",
        "\n",
        "\n",
        "            temp = 'Total Remuneration Plus'\n",
        "            plt.title(temp + \" for all data in \" + industry + \" of year \" + str(curr_year))\n",
        "            plt.ylabel(\"Percentage per range\")\n",
        "            plt.xlabel('Range in logarithmic dollar')\n",
        "            if industry_year_remuneration_plus_df[temp].size > 0:\n",
        "                print(str(industry_year_remuneration_plus_df[temp].shape[0]) + ' data pieces chosen for ' + curr_industry + ' of year ' + str(curr_year) + ' on ' + temp)\n",
        "                logit_data = np.log10(industry_year_remuneration_plus_df[temp].tolist())\n",
        "                logit_mean = np.mean(logit_data)\n",
        "                year_mean[temp].append(np.mean(industry_year_remuneration_plus_df[temp].tolist()))\n",
        "                logit_std = np.std(logit_data)\n",
        "                n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "                y = norm.pdf(bins, logit_mean, logit_std)\n",
        "                plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "                plt.xlim(left = 0, right = 10)\n",
        "                plt.grid(True)\n",
        "                plt.legend()\n",
        "                # plt.show()\n",
        "                plt.savefig(os.path.join(root_dir, task_folder, str(curr_year), industry, \"all_data_\" + temp))\n",
        "            else:\n",
        "                year_mean[temp].append(0)\n",
        "            plt.clf()\n",
        "\n",
        "\n",
        "            temp = 'Total Cash'\n",
        "            plt.title(temp + \" for all data in \" + industry + \" of year \" + str(curr_year))\n",
        "            plt.ylabel(\"Percentage per range\")\n",
        "            plt.xlabel('Range in logarithmic dollar')\n",
        "            if industry_year_cash_df[temp].size > 0:\n",
        "                print(str(industry_year_cash_df[temp].shape[0]) + ' data pieces chosen for ' + curr_industry + ' of year ' + str(curr_year) + ' on ' + temp)\n",
        "                logit_data = np.log10(industry_year_cash_df[temp].tolist())\n",
        "                logit_mean = np.mean(logit_data)\n",
        "                year_mean[temp].append(np.mean(industry_year_cash_df[temp].tolist()))\n",
        "                logit_std = np.std(logit_data)\n",
        "                n, bins, _ = plt.hist(logit_data, bins=1000, density=True)\n",
        "                y = norm.pdf(bins, logit_mean, logit_std)\n",
        "                plt.plot(bins, y, label = \"Mean: \" + str(logit_mean) + \" , Std: \" + str(logit_std))\n",
        "                plt.xlim(left = 0, right = 10)\n",
        "                plt.grid(True)\n",
        "                plt.legend()\n",
        "                # plt.show()\n",
        "                plt.savefig(os.path.join(root_dir, task_folder, str(curr_year), industry, \"all_data_\" + temp))\n",
        "            else:\n",
        "                year_mean[temp].append(0)\n",
        "            plt.clf()\n",
        "                \n",
        "\n",
        "        plt.title(curr_industry + ' year trend')\n",
        "        plt.xlabel('Year span')\n",
        "        plt.ylabel('Range in linear dollar')\n",
        "        plt.xlim(min(all_year) - 1, max(all_year) + 1)\n",
        "        plt.plot(all_year, year_mean['Total Annual Remuneration'], label = 'Total Annual Remuneration')\n",
        "        plt.plot(all_year, year_mean['Total Remuneration Plus'], label = 'Total Remuneration Plus')\n",
        "        plt.plot(all_year, year_mean['Total Cash'], label = 'Total Cash')\n",
        "        plt.legend()\n",
        "        plt.savefig(os.path.join(root_dir, task_folder, industry, 'Year Trend'))\n",
        "        plt.clf()\n",
        "        print(curr_industry + ' sector trend analysis finished\\n')\n",
        "\n",
        "\n",
        "print('\\n\\n-----------Analysis Ends-----------')"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "248967 data pieces chosen for specialty retail of year 2008 on Total Annual Remuneration\n",
            "248967 data pieces chosen for specialty retail of year 2008 on Total Remuneration Plus\n",
            "305307 data pieces chosen for specialty retail of year 2008 on Total Cash\n",
            "131779 data pieces chosen for specialty retail of year 2009 on Total Annual Remuneration\n",
            "131779 data pieces chosen for specialty retail of year 2009 on Total Remuneration Plus\n",
            "317705 data pieces chosen for specialty retail of year 2009 on Total Cash\n",
            "136813 data pieces chosen for specialty retail of year 2010 on Total Annual Remuneration\n",
            "136813 data pieces chosen for specialty retail of year 2010 on Total Remuneration Plus\n",
            "325976 data pieces chosen for specialty retail of year 2010 on Total Cash\n",
            "93682 data pieces chosen for specialty retail of year 2011 on Total Annual Remuneration\n",
            "93682 data pieces chosen for specialty retail of year 2011 on Total Remuneration Plus\n",
            "326677 data pieces chosen for specialty retail of year 2011 on Total Cash\n",
            "350573 data pieces chosen for specialty retail of year 2012 on Total Cash\n",
            "318466 data pieces chosen for specialty retail of year 2013 on Total Cash\n",
            "406695 data pieces chosen for specialty retail of year 2014 on Total Cash\n",
            "2539186 data pieces chosen for specialty retail of year 2015 on Total Cash\n",
            "3636558 data pieces chosen for specialty retail of year 2016 on Total Cash\n",
            "2062026 data pieces chosen for specialty retail of year 2017 on Total Annual Remuneration\n",
            "2061523 data pieces chosen for specialty retail of year 2017 on Total Remuneration Plus\n",
            "3644191 data pieces chosen for specialty retail of year 2017 on Total Cash\n",
            "1549778 data pieces chosen for specialty retail of year 2018 on Total Annual Remuneration\n",
            "1508000 data pieces chosen for specialty retail of year 2018 on Total Remuneration Plus\n",
            "3499215 data pieces chosen for specialty retail of year 2018 on Total Cash\n",
            "828347 data pieces chosen for specialty retail of year 2019 on Total Annual Remuneration\n",
            "823614 data pieces chosen for specialty retail of year 2019 on Total Remuneration Plus\n",
            "2362293 data pieces chosen for specialty retail of year 2019 on Total Cash\n",
            "specialty retail sector trend analysis finished\n",
            "\n",
            "6683 data pieces chosen for food product of year 2008 on Total Annual Remuneration\n",
            "6683 data pieces chosen for food product of year 2008 on Total Remuneration Plus\n",
            "18389 data pieces chosen for food product of year 2008 on Total Cash\n",
            "11610 data pieces chosen for food product of year 2009 on Total Annual Remuneration\n",
            "11610 data pieces chosen for food product of year 2009 on Total Remuneration Plus\n",
            "20987 data pieces chosen for food product of year 2009 on Total Cash\n",
            "13222 data pieces chosen for food product of year 2010 on Total Annual Remuneration\n",
            "13222 data pieces chosen for food product of year 2010 on Total Remuneration Plus\n",
            "19000 data pieces chosen for food product of year 2010 on Total Cash\n",
            "9351 data pieces chosen for food product of year 2011 on Total Annual Remuneration\n",
            "9351 data pieces chosen for food product of year 2011 on Total Remuneration Plus\n",
            "20163 data pieces chosen for food product of year 2011 on Total Cash\n",
            "24415 data pieces chosen for food product of year 2012 on Total Cash\n",
            "15736 data pieces chosen for food product of year 2013 on Total Cash\n",
            "26924 data pieces chosen for food product of year 2014 on Total Cash\n",
            "26748 data pieces chosen for food product of year 2015 on Total Cash\n",
            "28944 data pieces chosen for food product of year 2016 on Total Cash\n",
            "5765 data pieces chosen for food product of year 2017 on Total Annual Remuneration\n",
            "5429 data pieces chosen for food product of year 2017 on Total Remuneration Plus\n",
            "33934 data pieces chosen for food product of year 2017 on Total Cash\n",
            "14062 data pieces chosen for food product of year 2018 on Total Annual Remuneration\n",
            "13753 data pieces chosen for food product of year 2018 on Total Remuneration Plus\n",
            "37771 data pieces chosen for food product of year 2018 on Total Cash\n",
            "8818 data pieces chosen for food product of year 2019 on Total Annual Remuneration\n",
            "8818 data pieces chosen for food product of year 2019 on Total Remuneration Plus\n",
            "41451 data pieces chosen for food product of year 2019 on Total Cash\n",
            "food product sector trend analysis finished\n",
            "\n",
            "8729 data pieces chosen for insurance of year 2008 on Total Annual Remuneration\n",
            "8729 data pieces chosen for insurance of year 2008 on Total Remuneration Plus\n",
            "26392 data pieces chosen for insurance of year 2008 on Total Cash\n",
            "6814 data pieces chosen for insurance of year 2009 on Total Annual Remuneration\n",
            "6814 data pieces chosen for insurance of year 2009 on Total Remuneration Plus\n",
            "22556 data pieces chosen for insurance of year 2009 on Total Cash\n",
            "6045 data pieces chosen for insurance of year 2010 on Total Annual Remuneration\n",
            "6045 data pieces chosen for insurance of year 2010 on Total Remuneration Plus\n",
            "20498 data pieces chosen for insurance of year 2010 on Total Cash\n",
            "5024 data pieces chosen for insurance of year 2011 on Total Annual Remuneration\n",
            "5024 data pieces chosen for insurance of year 2011 on Total Remuneration Plus\n",
            "20012 data pieces chosen for insurance of year 2011 on Total Cash\n",
            "19257 data pieces chosen for insurance of year 2012 on Total Cash\n",
            "19494 data pieces chosen for insurance of year 2013 on Total Cash\n",
            "18422 data pieces chosen for insurance of year 2014 on Total Cash\n",
            "5682 data pieces chosen for insurance of year 2015 on Total Cash\n",
            "3061 data pieces chosen for insurance of year 2016 on Total Cash\n",
            "1499 data pieces chosen for insurance of year 2017 on Total Annual Remuneration\n",
            "1496 data pieces chosen for insurance of year 2017 on Total Remuneration Plus\n",
            "3804 data pieces chosen for insurance of year 2017 on Total Cash\n",
            "2876 data pieces chosen for insurance of year 2018 on Total Annual Remuneration\n",
            "2867 data pieces chosen for insurance of year 2018 on Total Remuneration Plus\n",
            "3090 data pieces chosen for insurance of year 2018 on Total Cash\n",
            "3260 data pieces chosen for insurance of year 2019 on Total Annual Remuneration\n",
            "3250 data pieces chosen for insurance of year 2019 on Total Remuneration Plus\n",
            "3260 data pieces chosen for insurance of year 2019 on Total Cash\n",
            "insurance sector trend analysis finished\n",
            "\n",
            "69878 data pieces chosen for healthcare provider and service of year 2008 on Total Annual Remuneration\n",
            "69878 data pieces chosen for healthcare provider and service of year 2008 on Total Remuneration Plus\n",
            "177711 data pieces chosen for healthcare provider and service of year 2008 on Total Cash\n",
            "123148 data pieces chosen for healthcare provider and service of year 2009 on Total Annual Remuneration\n",
            "123148 data pieces chosen for healthcare provider and service of year 2009 on Total Remuneration Plus\n",
            "263941 data pieces chosen for healthcare provider and service of year 2009 on Total Cash\n",
            "106190 data pieces chosen for healthcare provider and service of year 2010 on Total Annual Remuneration\n",
            "106190 data pieces chosen for healthcare provider and service of year 2010 on Total Remuneration Plus\n",
            "298796 data pieces chosen for healthcare provider and service of year 2010 on Total Cash\n",
            "147608 data pieces chosen for healthcare provider and service of year 2011 on Total Annual Remuneration\n",
            "147608 data pieces chosen for healthcare provider and service of year 2011 on Total Remuneration Plus\n",
            "300097 data pieces chosen for healthcare provider and service of year 2011 on Total Cash\n",
            "588938 data pieces chosen for healthcare provider and service of year 2012 on Total Cash\n",
            "604633 data pieces chosen for healthcare provider and service of year 2013 on Total Cash\n",
            "615208 data pieces chosen for healthcare provider and service of year 2014 on Total Cash\n",
            "655551 data pieces chosen for healthcare provider and service of year 2015 on Total Cash\n",
            "553998 data pieces chosen for healthcare provider and service of year 2016 on Total Cash\n",
            "152789 data pieces chosen for healthcare provider and service of year 2017 on Total Annual Remuneration\n",
            "151816 data pieces chosen for healthcare provider and service of year 2017 on Total Remuneration Plus\n",
            "308057 data pieces chosen for healthcare provider and service of year 2017 on Total Cash\n",
            "174946 data pieces chosen for healthcare provider and service of year 2018 on Total Annual Remuneration\n",
            "174925 data pieces chosen for healthcare provider and service of year 2018 on Total Remuneration Plus\n",
            "258147 data pieces chosen for healthcare provider and service of year 2018 on Total Cash\n",
            "90187 data pieces chosen for healthcare provider and service of year 2019 on Total Annual Remuneration\n",
            "87641 data pieces chosen for healthcare provider and service of year 2019 on Total Remuneration Plus\n",
            "230562 data pieces chosen for healthcare provider and service of year 2019 on Total Cash\n",
            "healthcare provider and service sector trend analysis finished\n",
            "\n",
            "19236 data pieces chosen for machinery of year 2008 on Total Annual Remuneration\n",
            "19236 data pieces chosen for machinery of year 2008 on Total Remuneration Plus\n",
            "56827 data pieces chosen for machinery of year 2008 on Total Cash\n",
            "12740 data pieces chosen for machinery of year 2009 on Total Annual Remuneration\n",
            "12740 data pieces chosen for machinery of year 2009 on Total Remuneration Plus\n",
            "50719 data pieces chosen for machinery of year 2009 on Total Cash\n",
            "8092 data pieces chosen for machinery of year 2010 on Total Annual Remuneration\n",
            "8092 data pieces chosen for machinery of year 2010 on Total Remuneration Plus\n",
            "56128 data pieces chosen for machinery of year 2010 on Total Cash\n",
            "16156 data pieces chosen for machinery of year 2011 on Total Annual Remuneration\n",
            "16156 data pieces chosen for machinery of year 2011 on Total Remuneration Plus\n",
            "38807 data pieces chosen for machinery of year 2011 on Total Cash\n",
            "58276 data pieces chosen for machinery of year 2012 on Total Cash\n",
            "51900 data pieces chosen for machinery of year 2013 on Total Cash\n",
            "47307 data pieces chosen for machinery of year 2014 on Total Cash\n",
            "68154 data pieces chosen for machinery of year 2015 on Total Cash\n",
            "92994 data pieces chosen for machinery of year 2016 on Total Cash\n",
            "30399 data pieces chosen for machinery of year 2017 on Total Annual Remuneration\n",
            "30337 data pieces chosen for machinery of year 2017 on Total Remuneration Plus\n",
            "65680 data pieces chosen for machinery of year 2017 on Total Cash\n",
            "38333 data pieces chosen for machinery of year 2018 on Total Annual Remuneration\n",
            "38327 data pieces chosen for machinery of year 2018 on Total Remuneration Plus\n",
            "48692 data pieces chosen for machinery of year 2018 on Total Cash\n",
            "28902 data pieces chosen for machinery of year 2019 on Total Annual Remuneration\n",
            "27597 data pieces chosen for machinery of year 2019 on Total Remuneration Plus\n",
            "56987 data pieces chosen for machinery of year 2019 on Total Cash\n",
            "machinery sector trend analysis finished\n",
            "\n",
            "6350 data pieces chosen for metal and mining of year 2008 on Total Annual Remuneration\n",
            "6350 data pieces chosen for metal and mining of year 2008 on Total Remuneration Plus\n",
            "8502 data pieces chosen for metal and mining of year 2008 on Total Cash\n",
            "5216 data pieces chosen for metal and mining of year 2009 on Total Annual Remuneration\n",
            "5216 data pieces chosen for metal and mining of year 2009 on Total Remuneration Plus\n",
            "10001 data pieces chosen for metal and mining of year 2009 on Total Cash\n",
            "5521 data pieces chosen for metal and mining of year 2010 on Total Annual Remuneration\n",
            "5521 data pieces chosen for metal and mining of year 2010 on Total Remuneration Plus\n",
            "9786 data pieces chosen for metal and mining of year 2010 on Total Cash\n",
            "3950 data pieces chosen for metal and mining of year 2011 on Total Annual Remuneration\n",
            "3950 data pieces chosen for metal and mining of year 2011 on Total Remuneration Plus\n",
            "11988 data pieces chosen for metal and mining of year 2011 on Total Cash\n",
            "17794 data pieces chosen for metal and mining of year 2012 on Total Cash\n",
            "15266 data pieces chosen for metal and mining of year 2013 on Total Cash\n",
            "10803 data pieces chosen for metal and mining of year 2014 on Total Cash\n",
            "8991 data pieces chosen for metal and mining of year 2015 on Total Cash\n",
            "9562 data pieces chosen for metal and mining of year 2016 on Total Cash\n",
            "1223 data pieces chosen for metal and mining of year 2017 on Total Annual Remuneration\n",
            "1211 data pieces chosen for metal and mining of year 2017 on Total Remuneration Plus\n",
            "8260 data pieces chosen for metal and mining of year 2017 on Total Cash\n",
            "1958 data pieces chosen for metal and mining of year 2018 on Total Annual Remuneration\n",
            "1958 data pieces chosen for metal and mining of year 2018 on Total Remuneration Plus\n",
            "7504 data pieces chosen for metal and mining of year 2018 on Total Cash\n",
            "4717 data pieces chosen for metal and mining of year 2019 on Total Annual Remuneration\n",
            "4717 data pieces chosen for metal and mining of year 2019 on Total Remuneration Plus\n",
            "7435 data pieces chosen for metal and mining of year 2019 on Total Cash\n",
            "metal and mining sector trend analysis finished\n",
            "\n",
            "25011 data pieces chosen for chemical of year 2008 on Total Annual Remuneration\n",
            "25011 data pieces chosen for chemical of year 2008 on Total Remuneration Plus\n",
            "53502 data pieces chosen for chemical of year 2008 on Total Cash\n",
            "16866 data pieces chosen for chemical of year 2009 on Total Annual Remuneration\n",
            "16866 data pieces chosen for chemical of year 2009 on Total Remuneration Plus\n",
            "50424 data pieces chosen for chemical of year 2009 on Total Cash\n",
            "22168 data pieces chosen for chemical of year 2010 on Total Annual Remuneration\n",
            "22168 data pieces chosen for chemical of year 2010 on Total Remuneration Plus\n",
            "56260 data pieces chosen for chemical of year 2010 on Total Cash\n",
            "31003 data pieces chosen for chemical of year 2011 on Total Annual Remuneration\n",
            "31003 data pieces chosen for chemical of year 2011 on Total Remuneration Plus\n",
            "57029 data pieces chosen for chemical of year 2011 on Total Cash\n",
            "76739 data pieces chosen for chemical of year 2012 on Total Cash\n",
            "72887 data pieces chosen for chemical of year 2013 on Total Cash\n",
            "87571 data pieces chosen for chemical of year 2014 on Total Cash\n",
            "71943 data pieces chosen for chemical of year 2015 on Total Cash\n",
            "70647 data pieces chosen for chemical of year 2016 on Total Cash\n",
            "18593 data pieces chosen for chemical of year 2017 on Total Annual Remuneration\n",
            "18205 data pieces chosen for chemical of year 2017 on Total Remuneration Plus\n",
            "74843 data pieces chosen for chemical of year 2017 on Total Cash\n",
            "25719 data pieces chosen for chemical of year 2018 on Total Annual Remuneration\n",
            "24138 data pieces chosen for chemical of year 2018 on Total Remuneration Plus\n",
            "62977 data pieces chosen for chemical of year 2018 on Total Cash\n",
            "19257 data pieces chosen for chemical of year 2019 on Total Annual Remuneration\n",
            "19115 data pieces chosen for chemical of year 2019 on Total Remuneration Plus\n",
            "62627 data pieces chosen for chemical of year 2019 on Total Cash\n",
            "chemical sector trend analysis finished\n",
            "\n",
            "490 data pieces chosen for pharmaceutical of year 2008 on Total Annual Remuneration\n",
            "490 data pieces chosen for pharmaceutical of year 2008 on Total Remuneration Plus\n",
            "3204 data pieces chosen for pharmaceutical of year 2008 on Total Cash\n",
            "5486 data pieces chosen for pharmaceutical of year 2009 on Total Annual Remuneration\n",
            "5486 data pieces chosen for pharmaceutical of year 2009 on Total Remuneration Plus\n",
            "12569 data pieces chosen for pharmaceutical of year 2009 on Total Cash\n",
            "7480 data pieces chosen for pharmaceutical of year 2010 on Total Annual Remuneration\n",
            "7480 data pieces chosen for pharmaceutical of year 2010 on Total Remuneration Plus\n",
            "23493 data pieces chosen for pharmaceutical of year 2010 on Total Cash\n",
            "673 data pieces chosen for pharmaceutical of year 2011 on Total Annual Remuneration\n",
            "673 data pieces chosen for pharmaceutical of year 2011 on Total Remuneration Plus\n",
            "19067 data pieces chosen for pharmaceutical of year 2011 on Total Cash\n",
            "8261 data pieces chosen for pharmaceutical of year 2012 on Total Cash\n",
            "21814 data pieces chosen for pharmaceutical of year 2013 on Total Cash\n",
            "24165 data pieces chosen for pharmaceutical of year 2014 on Total Cash\n",
            "24295 data pieces chosen for pharmaceutical of year 2015 on Total Cash\n",
            "20185 data pieces chosen for pharmaceutical of year 2016 on Total Cash\n",
            "8104 data pieces chosen for pharmaceutical of year 2017 on Total Annual Remuneration\n",
            "8100 data pieces chosen for pharmaceutical of year 2017 on Total Remuneration Plus\n",
            "23083 data pieces chosen for pharmaceutical of year 2017 on Total Cash\n",
            "702 data pieces chosen for pharmaceutical of year 2018 on Total Annual Remuneration\n",
            "702 data pieces chosen for pharmaceutical of year 2018 on Total Remuneration Plus\n",
            "2282 data pieces chosen for pharmaceutical of year 2018 on Total Cash\n",
            "1699 data pieces chosen for pharmaceutical of year 2019 on Total Annual Remuneration\n",
            "1687 data pieces chosen for pharmaceutical of year 2019 on Total Remuneration Plus\n",
            "2663 data pieces chosen for pharmaceutical of year 2019 on Total Cash\n",
            "pharmaceutical sector trend analysis finished\n",
            "\n",
            "34575 data pieces chosen for hotel restaurant and leisure of year 2008 on Total Annual Remuneration\n",
            "34575 data pieces chosen for hotel restaurant and leisure of year 2008 on Total Remuneration Plus\n",
            "86643 data pieces chosen for hotel restaurant and leisure of year 2008 on Total Cash\n",
            "1873 data pieces chosen for hotel restaurant and leisure of year 2009 on Total Annual Remuneration\n",
            "1873 data pieces chosen for hotel restaurant and leisure of year 2009 on Total Remuneration Plus\n",
            "62279 data pieces chosen for hotel restaurant and leisure of year 2009 on Total Cash\n",
            "26644 data pieces chosen for hotel restaurant and leisure of year 2010 on Total Annual Remuneration\n",
            "26644 data pieces chosen for hotel restaurant and leisure of year 2010 on Total Remuneration Plus\n",
            "64575 data pieces chosen for hotel restaurant and leisure of year 2010 on Total Cash\n",
            "19788 data pieces chosen for hotel restaurant and leisure of year 2011 on Total Annual Remuneration\n",
            "19788 data pieces chosen for hotel restaurant and leisure of year 2011 on Total Remuneration Plus\n",
            "68472 data pieces chosen for hotel restaurant and leisure of year 2011 on Total Cash\n",
            "133211 data pieces chosen for hotel restaurant and leisure of year 2012 on Total Cash\n",
            "43885 data pieces chosen for hotel restaurant and leisure of year 2013 on Total Cash\n",
            "27429 data pieces chosen for hotel restaurant and leisure of year 2014 on Total Cash\n",
            "51277 data pieces chosen for hotel restaurant and leisure of year 2015 on Total Cash\n",
            "71210 data pieces chosen for hotel restaurant and leisure of year 2016 on Total Cash\n",
            "12597 data pieces chosen for hotel restaurant and leisure of year 2017 on Total Annual Remuneration\n",
            "12597 data pieces chosen for hotel restaurant and leisure of year 2017 on Total Remuneration Plus\n",
            "60156 data pieces chosen for hotel restaurant and leisure of year 2017 on Total Cash\n",
            "14191 data pieces chosen for hotel restaurant and leisure of year 2018 on Total Annual Remuneration\n",
            "13756 data pieces chosen for hotel restaurant and leisure of year 2018 on Total Remuneration Plus\n",
            "96110 data pieces chosen for hotel restaurant and leisure of year 2018 on Total Cash\n",
            "582 data pieces chosen for hotel restaurant and leisure of year 2019 on Total Annual Remuneration\n",
            "582 data pieces chosen for hotel restaurant and leisure of year 2019 on Total Remuneration Plus\n",
            "82779 data pieces chosen for hotel restaurant and leisure of year 2019 on Total Cash\n",
            "hotel restaurant and leisure sector trend analysis finished\n",
            "\n",
            "3646 data pieces chosen for container and packaging of year 2008 on Total Annual Remuneration\n",
            "3646 data pieces chosen for container and packaging of year 2008 on Total Remuneration Plus\n",
            "16807 data pieces chosen for container and packaging of year 2008 on Total Cash\n",
            "2722 data pieces chosen for container and packaging of year 2009 on Total Annual Remuneration\n",
            "2722 data pieces chosen for container and packaging of year 2009 on Total Remuneration Plus\n",
            "17108 data pieces chosen for container and packaging of year 2009 on Total Cash\n",
            "350 data pieces chosen for container and packaging of year 2010 on Total Annual Remuneration\n",
            "350 data pieces chosen for container and packaging of year 2010 on Total Remuneration Plus\n",
            "11124 data pieces chosen for container and packaging of year 2010 on Total Cash\n",
            "757 data pieces chosen for container and packaging of year 2011 on Total Annual Remuneration\n",
            "757 data pieces chosen for container and packaging of year 2011 on Total Remuneration Plus\n",
            "9992 data pieces chosen for container and packaging of year 2011 on Total Cash\n",
            "13685 data pieces chosen for container and packaging of year 2012 on Total Cash\n",
            "6960 data pieces chosen for container and packaging of year 2013 on Total Cash\n",
            "12540 data pieces chosen for container and packaging of year 2014 on Total Cash\n",
            "14623 data pieces chosen for container and packaging of year 2015 on Total Cash\n",
            "12888 data pieces chosen for container and packaging of year 2016 on Total Cash\n",
            "8670 data pieces chosen for container and packaging of year 2017 on Total Annual Remuneration\n",
            "8670 data pieces chosen for container and packaging of year 2017 on Total Remuneration Plus\n",
            "14796 data pieces chosen for container and packaging of year 2017 on Total Cash\n",
            "2547 data pieces chosen for container and packaging of year 2018 on Total Annual Remuneration\n",
            "2547 data pieces chosen for container and packaging of year 2018 on Total Remuneration Plus\n",
            "3407 data pieces chosen for container and packaging of year 2018 on Total Cash\n",
            "3001 data pieces chosen for container and packaging of year 2019 on Total Annual Remuneration\n",
            "3001 data pieces chosen for container and packaging of year 2019 on Total Remuneration Plus\n",
            "6370 data pieces chosen for container and packaging of year 2019 on Total Cash\n",
            "container and packaging sector trend analysis finished\n",
            "\n",
            "4577 data pieces chosen for capital market of year 2008 on Total Annual Remuneration\n",
            "4577 data pieces chosen for capital market of year 2008 on Total Remuneration Plus\n",
            "14401 data pieces chosen for capital market of year 2008 on Total Cash\n",
            "5352 data pieces chosen for capital market of year 2009 on Total Annual Remuneration\n",
            "5352 data pieces chosen for capital market of year 2009 on Total Remuneration Plus\n",
            "20261 data pieces chosen for capital market of year 2009 on Total Cash\n",
            "5000 data pieces chosen for capital market of year 2010 on Total Annual Remuneration\n",
            "5000 data pieces chosen for capital market of year 2010 on Total Remuneration Plus\n",
            "19218 data pieces chosen for capital market of year 2010 on Total Cash\n",
            "3945 data pieces chosen for capital market of year 2011 on Total Annual Remuneration\n",
            "3945 data pieces chosen for capital market of year 2011 on Total Remuneration Plus\n",
            "20353 data pieces chosen for capital market of year 2011 on Total Cash\n",
            "24579 data pieces chosen for capital market of year 2012 on Total Cash\n",
            "26971 data pieces chosen for capital market of year 2013 on Total Cash\n",
            "31681 data pieces chosen for capital market of year 2014 on Total Cash\n",
            "32464 data pieces chosen for capital market of year 2015 on Total Cash\n",
            "31916 data pieces chosen for capital market of year 2016 on Total Cash\n",
            "2789 data pieces chosen for capital market of year 2017 on Total Annual Remuneration\n",
            "32476 data pieces chosen for capital market of year 2017 on Total Cash\n",
            "24754 data pieces chosen for capital market of year 2018 on Total Cash\n",
            "4349 data pieces chosen for capital market of year 2019 on Total Annual Remuneration\n",
            "4349 data pieces chosen for capital market of year 2019 on Total Remuneration Plus\n",
            "25317 data pieces chosen for capital market of year 2019 on Total Cash\n",
            "capital market sector trend analysis finished\n",
            "\n",
            "1598 data pieces chosen for personal product of year 2008 on Total Annual Remuneration\n",
            "1598 data pieces chosen for personal product of year 2008 on Total Remuneration Plus\n",
            "3338 data pieces chosen for personal product of year 2008 on Total Cash\n",
            "2656 data pieces chosen for personal product of year 2009 on Total Annual Remuneration\n",
            "2656 data pieces chosen for personal product of year 2009 on Total Remuneration Plus\n",
            "7004 data pieces chosen for personal product of year 2009 on Total Cash\n",
            "1849 data pieces chosen for personal product of year 2010 on Total Annual Remuneration\n",
            "1849 data pieces chosen for personal product of year 2010 on Total Remuneration Plus\n",
            "6448 data pieces chosen for personal product of year 2010 on Total Cash\n",
            "2093 data pieces chosen for personal product of year 2011 on Total Annual Remuneration\n",
            "2093 data pieces chosen for personal product of year 2011 on Total Remuneration Plus\n",
            "4327 data pieces chosen for personal product of year 2011 on Total Cash\n",
            "3801 data pieces chosen for personal product of year 2012 on Total Cash\n",
            "3891 data pieces chosen for personal product of year 2013 on Total Cash\n",
            "4110 data pieces chosen for personal product of year 2014 on Total Cash\n",
            "3546 data pieces chosen for personal product of year 2015 on Total Cash\n",
            "5144 data pieces chosen for personal product of year 2016 on Total Cash\n",
            "2947 data pieces chosen for personal product of year 2017 on Total Cash\n",
            "134 data pieces chosen for personal product of year 2018 on Total Annual Remuneration\n",
            "134 data pieces chosen for personal product of year 2018 on Total Remuneration Plus\n",
            "1400 data pieces chosen for personal product of year 2018 on Total Cash\n",
            "572 data pieces chosen for personal product of year 2019 on Total Annual Remuneration\n",
            "572 data pieces chosen for personal product of year 2019 on Total Remuneration Plus\n",
            "4296 data pieces chosen for personal product of year 2019 on Total Cash\n",
            "personal product sector trend analysis finished\n",
            "\n",
            "6858 data pieces chosen for electric utility of year 2008 on Total Annual Remuneration\n",
            "6858 data pieces chosen for electric utility of year 2008 on Total Remuneration Plus\n",
            "21177 data pieces chosen for electric utility of year 2008 on Total Cash\n",
            "11447 data pieces chosen for electric utility of year 2009 on Total Annual Remuneration\n",
            "11447 data pieces chosen for electric utility of year 2009 on Total Remuneration Plus\n",
            "29269 data pieces chosen for electric utility of year 2009 on Total Cash\n",
            "11860 data pieces chosen for electric utility of year 2010 on Total Annual Remuneration\n",
            "11860 data pieces chosen for electric utility of year 2010 on Total Remuneration Plus\n",
            "28558 data pieces chosen for electric utility of year 2010 on Total Cash\n",
            "11404 data pieces chosen for electric utility of year 2011 on Total Annual Remuneration\n",
            "11404 data pieces chosen for electric utility of year 2011 on Total Remuneration Plus\n",
            "28280 data pieces chosen for electric utility of year 2011 on Total Cash\n",
            "30580 data pieces chosen for electric utility of year 2012 on Total Cash\n",
            "28635 data pieces chosen for electric utility of year 2013 on Total Cash\n",
            "27831 data pieces chosen for electric utility of year 2014 on Total Cash\n",
            "43788 data pieces chosen for electric utility of year 2015 on Total Cash\n",
            "37538 data pieces chosen for electric utility of year 2016 on Total Cash\n",
            "22603 data pieces chosen for electric utility of year 2017 on Total Annual Remuneration\n",
            "22603 data pieces chosen for electric utility of year 2017 on Total Remuneration Plus\n",
            "27705 data pieces chosen for electric utility of year 2017 on Total Cash\n",
            "19274 data pieces chosen for electric utility of year 2018 on Total Annual Remuneration\n",
            "19274 data pieces chosen for electric utility of year 2018 on Total Remuneration Plus\n",
            "26499 data pieces chosen for electric utility of year 2018 on Total Cash\n",
            "16155 data pieces chosen for electric utility of year 2019 on Total Annual Remuneration\n",
            "16094 data pieces chosen for electric utility of year 2019 on Total Remuneration Plus\n",
            "20679 data pieces chosen for electric utility of year 2019 on Total Cash\n",
            "electric utility sector trend analysis finished\n",
            "\n",
            "1993 data pieces chosen for bank of year 2008 on Total Cash\n",
            "2262 data pieces chosen for bank of year 2009 on Total Cash\n",
            "1339 data pieces chosen for bank of year 2010 on Total Cash\n",
            "10104 data pieces chosen for bank of year 2011 on Total Cash\n",
            "1453 data pieces chosen for bank of year 2012 on Total Cash\n",
            "1415 data pieces chosen for bank of year 2013 on Total Cash\n",
            "958 data pieces chosen for bank of year 2014 on Total Cash\n",
            "2085 data pieces chosen for bank of year 2015 on Total Cash\n",
            "2326 data pieces chosen for bank of year 2016 on Total Cash\n",
            "4020 data pieces chosen for bank of year 2017 on Total Cash\n",
            "554 data pieces chosen for bank of year 2018 on Total Annual Remuneration\n",
            "554 data pieces chosen for bank of year 2018 on Total Remuneration Plus\n",
            "554 data pieces chosen for bank of year 2018 on Total Cash\n",
            "10202 data pieces chosen for bank of year 2019 on Total Cash\n",
            "bank sector trend analysis finished\n",
            "\n",
            "185 data pieces chosen for healthcare equipment and supply of year 2008 on Total Cash\n",
            "1582 data pieces chosen for healthcare equipment and supply of year 2009 on Total Cash\n",
            "531 data pieces chosen for healthcare equipment and supply of year 2010 on Total Annual Remuneration\n",
            "531 data pieces chosen for healthcare equipment and supply of year 2010 on Total Remuneration Plus\n",
            "1422 data pieces chosen for healthcare equipment and supply of year 2010 on Total Cash\n",
            "498 data pieces chosen for healthcare equipment and supply of year 2011 on Total Annual Remuneration\n",
            "498 data pieces chosen for healthcare equipment and supply of year 2011 on Total Remuneration Plus\n",
            "1594 data pieces chosen for healthcare equipment and supply of year 2011 on Total Cash\n",
            "1807 data pieces chosen for healthcare equipment and supply of year 2012 on Total Cash\n",
            "1628 data pieces chosen for healthcare equipment and supply of year 2013 on Total Cash\n",
            "1767 data pieces chosen for healthcare equipment and supply of year 2014 on Total Cash\n",
            "1859 data pieces chosen for healthcare equipment and supply of year 2015 on Total Cash\n",
            "2264 data pieces chosen for healthcare equipment and supply of year 2016 on Total Cash\n",
            "2238 data pieces chosen for healthcare equipment and supply of year 2017 on Total Annual Remuneration\n",
            "2238 data pieces chosen for healthcare equipment and supply of year 2017 on Total Remuneration Plus\n",
            "2238 data pieces chosen for healthcare equipment and supply of year 2017 on Total Cash\n",
            "273 data pieces chosen for healthcare equipment and supply of year 2018 on Total Cash\n",
            "healthcare equipment and supply sector trend analysis finished\n",
            "\n",
            "135 data pieces chosen for road and rail of year 2008 on Total Annual Remuneration\n",
            "135 data pieces chosen for road and rail of year 2008 on Total Remuneration Plus\n",
            "3796 data pieces chosen for road and rail of year 2008 on Total Cash\n",
            "132 data pieces chosen for road and rail of year 2009 on Total Annual Remuneration\n",
            "132 data pieces chosen for road and rail of year 2009 on Total Remuneration Plus\n",
            "703 data pieces chosen for road and rail of year 2009 on Total Cash\n",
            "434 data pieces chosen for road and rail of year 2010 on Total Annual Remuneration\n",
            "434 data pieces chosen for road and rail of year 2010 on Total Remuneration Plus\n",
            "1419 data pieces chosen for road and rail of year 2010 on Total Cash\n",
            "380 data pieces chosen for road and rail of year 2011 on Total Annual Remuneration\n",
            "380 data pieces chosen for road and rail of year 2011 on Total Remuneration Plus\n",
            "831 data pieces chosen for road and rail of year 2011 on Total Cash\n",
            "2945 data pieces chosen for road and rail of year 2012 on Total Cash\n",
            "4092 data pieces chosen for road and rail of year 2013 on Total Cash\n",
            "6481 data pieces chosen for road and rail of year 2014 on Total Cash\n",
            "6406 data pieces chosen for road and rail of year 2015 on Total Cash\n",
            "18164 data pieces chosen for road and rail of year 2016 on Total Cash\n",
            "1132 data pieces chosen for road and rail of year 2017 on Total Annual Remuneration\n",
            "1132 data pieces chosen for road and rail of year 2017 on Total Remuneration Plus\n",
            "7092 data pieces chosen for road and rail of year 2017 on Total Cash\n",
            "4176 data pieces chosen for road and rail of year 2018 on Total Annual Remuneration\n",
            "4176 data pieces chosen for road and rail of year 2018 on Total Remuneration Plus\n",
            "7653 data pieces chosen for road and rail of year 2018 on Total Cash\n",
            "10127 data pieces chosen for road and rail of year 2019 on Total Annual Remuneration\n",
            "10127 data pieces chosen for road and rail of year 2019 on Total Remuneration Plus\n",
            "15466 data pieces chosen for road and rail of year 2019 on Total Cash\n",
            "road and rail sector trend analysis finished\n",
            "\n",
            "1991 data pieces chosen for diversified consumer service of year 2008 on Total Cash\n",
            "568 data pieces chosen for diversified consumer service of year 2009 on Total Annual Remuneration\n",
            "568 data pieces chosen for diversified consumer service of year 2009 on Total Remuneration Plus\n",
            "2241 data pieces chosen for diversified consumer service of year 2009 on Total Cash\n",
            "1352 data pieces chosen for diversified consumer service of year 2010 on Total Annual Remuneration\n",
            "1352 data pieces chosen for diversified consumer service of year 2010 on Total Remuneration Plus\n",
            "2416 data pieces chosen for diversified consumer service of year 2010 on Total Cash\n",
            "433 data pieces chosen for diversified consumer service of year 2011 on Total Annual Remuneration\n",
            "433 data pieces chosen for diversified consumer service of year 2011 on Total Remuneration Plus\n",
            "2228 data pieces chosen for diversified consumer service of year 2011 on Total Cash\n",
            "4190 data pieces chosen for diversified consumer service of year 2012 on Total Cash\n",
            "2722 data pieces chosen for diversified consumer service of year 2013 on Total Cash\n",
            "4464 data pieces chosen for diversified consumer service of year 2014 on Total Cash\n",
            "7766 data pieces chosen for diversified consumer service of year 2015 on Total Cash\n",
            "9396 data pieces chosen for diversified consumer service of year 2016 on Total Cash\n",
            "567 data pieces chosen for diversified consumer service of year 2017 on Total Annual Remuneration\n",
            "567 data pieces chosen for diversified consumer service of year 2017 on Total Remuneration Plus\n",
            "5412 data pieces chosen for diversified consumer service of year 2017 on Total Cash\n",
            "3371 data pieces chosen for diversified consumer service of year 2018 on Total Annual Remuneration\n",
            "3371 data pieces chosen for diversified consumer service of year 2018 on Total Remuneration Plus\n",
            "5718 data pieces chosen for diversified consumer service of year 2018 on Total Cash\n",
            "2778 data pieces chosen for diversified consumer service of year 2019 on Total Annual Remuneration\n",
            "2778 data pieces chosen for diversified consumer service of year 2019 on Total Remuneration Plus\n",
            "4432 data pieces chosen for diversified consumer service of year 2019 on Total Cash\n",
            "diversified consumer service sector trend analysis finished\n",
            "\n",
            "2214 data pieces chosen for construction and engineering of year 2012 on Total Cash\n",
            "1873 data pieces chosen for construction and engineering of year 2013 on Total Cash\n",
            "1981 data pieces chosen for construction and engineering of year 2014 on Total Cash\n",
            "4047 data pieces chosen for construction and engineering of year 2015 on Total Cash\n",
            "2321 data pieces chosen for construction and engineering of year 2016 on Total Cash\n",
            "311 data pieces chosen for construction and engineering of year 2017 on Total Annual Remuneration\n",
            "311 data pieces chosen for construction and engineering of year 2017 on Total Remuneration Plus\n",
            "5032 data pieces chosen for construction and engineering of year 2017 on Total Cash\n",
            "3768 data pieces chosen for construction and engineering of year 2018 on Total Annual Remuneration\n",
            "3766 data pieces chosen for construction and engineering of year 2018 on Total Remuneration Plus\n",
            "3768 data pieces chosen for construction and engineering of year 2018 on Total Cash\n",
            "construction and engineering sector trend analysis finished\n",
            "\n",
            "386 data pieces chosen for beverage of year 2010 on Total Annual Remuneration\n",
            "386 data pieces chosen for beverage of year 2010 on Total Remuneration Plus\n",
            "16232 data pieces chosen for beverage of year 2010 on Total Cash\n",
            "393 data pieces chosen for beverage of year 2011 on Total Annual Remuneration\n",
            "393 data pieces chosen for beverage of year 2011 on Total Remuneration Plus\n",
            "27529 data pieces chosen for beverage of year 2011 on Total Cash\n",
            "25416 data pieces chosen for beverage of year 2012 on Total Cash\n",
            "16166 data pieces chosen for beverage of year 2013 on Total Cash\n",
            "123300 data pieces chosen for beverage of year 2014 on Total Cash\n",
            "111293 data pieces chosen for beverage of year 2015 on Total Cash\n",
            "131542 data pieces chosen for beverage of year 2016 on Total Cash\n",
            "16378 data pieces chosen for beverage of year 2017 on Total Annual Remuneration\n",
            "16378 data pieces chosen for beverage of year 2017 on Total Remuneration Plus\n",
            "102409 data pieces chosen for beverage of year 2017 on Total Cash\n",
            "15718 data pieces chosen for beverage of year 2018 on Total Annual Remuneration\n",
            "15471 data pieces chosen for beverage of year 2018 on Total Remuneration Plus\n",
            "81701 data pieces chosen for beverage of year 2018 on Total Cash\n",
            "15440 data pieces chosen for beverage of year 2019 on Total Annual Remuneration\n",
            "15440 data pieces chosen for beverage of year 2019 on Total Remuneration Plus\n",
            "98493 data pieces chosen for beverage of year 2019 on Total Cash\n",
            "beverage sector trend analysis finished\n",
            "\n",
            "14665 data pieces chosen for energy equipment and service of year 2012 on Total Cash\n",
            "17554 data pieces chosen for energy equipment and service of year 2013 on Total Cash\n",
            "23035 data pieces chosen for energy equipment and service of year 2014 on Total Cash\n",
            "2646 data pieces chosen for energy equipment and service of year 2015 on Total Cash\n",
            "544 data pieces chosen for energy equipment and service of year 2016 on Total Cash\n",
            "375 data pieces chosen for energy equipment and service of year 2017 on Total Annual Remuneration\n",
            "375 data pieces chosen for energy equipment and service of year 2017 on Total Remuneration Plus\n",
            "420 data pieces chosen for energy equipment and service of year 2017 on Total Cash\n",
            "234 data pieces chosen for energy equipment and service of year 2018 on Total Annual Remuneration\n",
            "234 data pieces chosen for energy equipment and service of year 2018 on Total Remuneration Plus\n",
            "1031 data pieces chosen for energy equipment and service of year 2018 on Total Cash\n",
            "475 data pieces chosen for energy equipment and service of year 2019 on Total Annual Remuneration\n",
            "462 data pieces chosen for energy equipment and service of year 2019 on Total Remuneration Plus\n",
            "475 data pieces chosen for energy equipment and service of year 2019 on Total Cash\n",
            "energy equipment and service sector trend analysis finished\n",
            "\n",
            "8451 data pieces chosen for aerospace and defense of year 2010 on Total Annual Remuneration\n",
            "8451 data pieces chosen for aerospace and defense of year 2010 on Total Remuneration Plus\n",
            "10587 data pieces chosen for aerospace and defense of year 2010 on Total Cash\n",
            "6563 data pieces chosen for aerospace and defense of year 2011 on Total Cash\n",
            "45378 data pieces chosen for aerospace and defense of year 2012 on Total Cash\n",
            "8603 data pieces chosen for aerospace and defense of year 2013 on Total Cash\n",
            "8392 data pieces chosen for aerospace and defense of year 2014 on Total Cash\n",
            "9289 data pieces chosen for aerospace and defense of year 2015 on Total Cash\n",
            "4227 data pieces chosen for aerospace and defense of year 2016 on Total Cash\n",
            "452 data pieces chosen for aerospace and defense of year 2017 on Total Annual Remuneration\n",
            "439 data pieces chosen for aerospace and defense of year 2017 on Total Remuneration Plus\n",
            "4866 data pieces chosen for aerospace and defense of year 2017 on Total Cash\n",
            "2118 data pieces chosen for aerospace and defense of year 2018 on Total Annual Remuneration\n",
            "2118 data pieces chosen for aerospace and defense of year 2018 on Total Remuneration Plus\n",
            "4131 data pieces chosen for aerospace and defense of year 2018 on Total Cash\n",
            "3457 data pieces chosen for aerospace and defense of year 2019 on Total Annual Remuneration\n",
            "3393 data pieces chosen for aerospace and defense of year 2019 on Total Remuneration Plus\n",
            "7739 data pieces chosen for aerospace and defense of year 2019 on Total Cash\n",
            "aerospace and defense sector trend analysis finished\n",
            "\n",
            "965 data pieces chosen for oil gas and consumable fuel of year 2010 on Total Cash\n",
            "1200 data pieces chosen for oil gas and consumable fuel of year 2011 on Total Annual Remuneration\n",
            "1200 data pieces chosen for oil gas and consumable fuel of year 2011 on Total Remuneration Plus\n",
            "2775 data pieces chosen for oil gas and consumable fuel of year 2011 on Total Cash\n",
            "2581 data pieces chosen for oil gas and consumable fuel of year 2012 on Total Cash\n",
            "2474 data pieces chosen for oil gas and consumable fuel of year 2013 on Total Cash\n",
            "847 data pieces chosen for oil gas and consumable fuel of year 2014 on Total Cash\n",
            "4 data pieces chosen for oil gas and consumable fuel of year 2015 on Total Cash\n",
            "2 data pieces chosen for oil gas and consumable fuel of year 2016 on Total Cash\n",
            "4 data pieces chosen for oil gas and consumable fuel of year 2017 on Total Cash\n",
            "3 data pieces chosen for oil gas and consumable fuel of year 2018 on Total Cash\n",
            "oil gas and consumable fuel sector trend analysis finished\n",
            "\n",
            "141 data pieces chosen for semiconductor and semiconductor equipment of year 2010 on Total Cash\n",
            "4541 data pieces chosen for semiconductor and semiconductor equipment of year 2011 on Total Annual Remuneration\n",
            "4541 data pieces chosen for semiconductor and semiconductor equipment of year 2011 on Total Remuneration Plus\n",
            "5375 data pieces chosen for semiconductor and semiconductor equipment of year 2011 on Total Cash\n",
            "2713 data pieces chosen for semiconductor and semiconductor equipment of year 2012 on Total Cash\n",
            "39974 data pieces chosen for semiconductor and semiconductor equipment of year 2014 on Total Cash\n",
            "37515 data pieces chosen for semiconductor and semiconductor equipment of year 2015 on Total Cash\n",
            "8292 data pieces chosen for semiconductor and semiconductor equipment of year 2016 on Total Cash\n",
            "302 data pieces chosen for semiconductor and semiconductor equipment of year 2017 on Total Cash\n",
            "30835 data pieces chosen for semiconductor and semiconductor equipment of year 2018 on Total Cash\n",
            "33017 data pieces chosen for semiconductor and semiconductor equipment of year 2019 on Total Cash\n",
            "semiconductor and semiconductor equipment sector trend analysis finished\n",
            "\n",
            "62 data pieces chosen for household durables of year 2011 on Total Cash\n",
            "1424 data pieces chosen for household durables of year 2012 on Total Cash\n",
            "1391 data pieces chosen for household durables of year 2013 on Total Cash\n",
            "6252 data pieces chosen for household durables of year 2014 on Total Cash\n",
            "11136 data pieces chosen for household durables of year 2015 on Total Cash\n",
            "15172 data pieces chosen for household durables of year 2016 on Total Cash\n",
            "4867 data pieces chosen for household durables of year 2017 on Total Cash\n",
            "565 data pieces chosen for household durables of year 2018 on Total Annual Remuneration\n",
            "565 data pieces chosen for household durables of year 2018 on Total Remuneration Plus\n",
            "981 data pieces chosen for household durables of year 2018 on Total Cash\n",
            "64 data pieces chosen for household durables of year 2019 on Total Cash\n",
            "household durables sector trend analysis finished\n",
            "\n",
            "2924 data pieces chosen for equity real estate investment trust reit of year 2012 on Total Cash\n",
            "83 data pieces chosen for equity real estate investment trust reit of year 2013 on Total Cash\n",
            "809 data pieces chosen for equity real estate investment trust reit of year 2014 on Total Cash\n",
            "2626 data pieces chosen for equity real estate investment trust reit of year 2015 on Total Cash\n",
            "2344 data pieces chosen for equity real estate investment trust reit of year 2016 on Total Cash\n",
            "49 data pieces chosen for equity real estate investment trust reit of year 2017 on Total Annual Remuneration\n",
            "49 data pieces chosen for equity real estate investment trust reit of year 2017 on Total Remuneration Plus\n",
            "58 data pieces chosen for equity real estate investment trust reit of year 2017 on Total Cash\n",
            "100 data pieces chosen for equity real estate investment trust reit of year 2018 on Total Annual Remuneration\n",
            "100 data pieces chosen for equity real estate investment trust reit of year 2018 on Total Remuneration Plus\n",
            "100 data pieces chosen for equity real estate investment trust reit of year 2018 on Total Cash\n",
            "27 data pieces chosen for equity real estate investment trust reit of year 2019 on Total Annual Remuneration\n",
            "27 data pieces chosen for equity real estate investment trust reit of year 2019 on Total Remuneration Plus\n",
            "65 data pieces chosen for equity real estate investment trust reit of year 2019 on Total Cash\n",
            "equity real estate investment trust reit sector trend analysis finished\n",
            "\n",
            "16103 data pieces chosen for diversified telecommunication service of year 2012 on Total Cash\n",
            "24829 data pieces chosen for diversified telecommunication service of year 2013 on Total Cash\n",
            "85 data pieces chosen for diversified telecommunication service of year 2015 on Total Cash\n",
            "diversified telecommunication service sector trend analysis finished\n",
            "\n",
            "8766 data pieces chosen for building product of year 2010 on Total Annual Remuneration\n",
            "8766 data pieces chosen for building product of year 2010 on Total Remuneration Plus\n",
            "9801 data pieces chosen for building product of year 2010 on Total Cash\n",
            "1318 data pieces chosen for building product of year 2011 on Total Annual Remuneration\n",
            "1318 data pieces chosen for building product of year 2011 on Total Remuneration Plus\n",
            "2622 data pieces chosen for building product of year 2011 on Total Cash\n",
            "3358 data pieces chosen for building product of year 2012 on Total Cash\n",
            "3041 data pieces chosen for building product of year 2013 on Total Cash\n",
            "2925 data pieces chosen for building product of year 2014 on Total Cash\n",
            "2070 data pieces chosen for building product of year 2015 on Total Cash\n",
            "3 data pieces chosen for building product of year 2016 on Total Cash\n",
            "136 data pieces chosen for building product of year 2017 on Total Annual Remuneration\n",
            "136 data pieces chosen for building product of year 2017 on Total Remuneration Plus\n",
            "165 data pieces chosen for building product of year 2017 on Total Cash\n",
            "130 data pieces chosen for building product of year 2018 on Total Annual Remuneration\n",
            "130 data pieces chosen for building product of year 2018 on Total Remuneration Plus\n",
            "130 data pieces chosen for building product of year 2018 on Total Cash\n",
            "1285 data pieces chosen for building product of year 2019 on Total Annual Remuneration\n",
            "1285 data pieces chosen for building product of year 2019 on Total Remuneration Plus\n",
            "1285 data pieces chosen for building product of year 2019 on Total Cash\n",
            "building product sector trend analysis finished\n",
            "\n",
            "4724 data pieces chosen for professional service of year 2012 on Total Cash\n",
            "3043 data pieces chosen for professional service of year 2013 on Total Cash\n",
            "6694 data pieces chosen for professional service of year 2014 on Total Cash\n",
            "2699 data pieces chosen for professional service of year 2015 on Total Cash\n",
            "7420 data pieces chosen for professional service of year 2016 on Total Cash\n",
            "913 data pieces chosen for professional service of year 2017 on Total Annual Remuneration\n",
            "913 data pieces chosen for professional service of year 2017 on Total Remuneration Plus\n",
            "4957 data pieces chosen for professional service of year 2017 on Total Cash\n",
            "166 data pieces chosen for professional service of year 2018 on Total Annual Remuneration\n",
            "166 data pieces chosen for professional service of year 2018 on Total Remuneration Plus\n",
            "3649 data pieces chosen for professional service of year 2018 on Total Cash\n",
            "880 data pieces chosen for professional service of year 2019 on Total Annual Remuneration\n",
            "880 data pieces chosen for professional service of year 2019 on Total Remuneration Plus\n",
            "4860 data pieces chosen for professional service of year 2019 on Total Cash\n",
            "professional service sector trend analysis finished\n",
            "\n",
            "7510 data pieces chosen for electrical equipment of year 2010 on Total Cash\n",
            "8755 data pieces chosen for electrical equipment of year 2011 on Total Cash\n",
            "8663 data pieces chosen for electrical equipment of year 2012 on Total Cash\n",
            "9970 data pieces chosen for electrical equipment of year 2013 on Total Cash\n",
            "8264 data pieces chosen for electrical equipment of year 2014 on Total Cash\n",
            "9323 data pieces chosen for electrical equipment of year 2015 on Total Cash\n",
            "8682 data pieces chosen for electrical equipment of year 2016 on Total Cash\n",
            "7820 data pieces chosen for electrical equipment of year 2017 on Total Cash\n",
            "7810 data pieces chosen for electrical equipment of year 2018 on Total Annual Remuneration\n",
            "7810 data pieces chosen for electrical equipment of year 2018 on Total Remuneration Plus\n",
            "7810 data pieces chosen for electrical equipment of year 2018 on Total Cash\n",
            "10797 data pieces chosen for electrical equipment of year 2019 on Total Annual Remuneration\n",
            "10797 data pieces chosen for electrical equipment of year 2019 on Total Remuneration Plus\n",
            "10797 data pieces chosen for electrical equipment of year 2019 on Total Cash\n",
            "electrical equipment sector trend analysis finished\n",
            "\n",
            "1804 data pieces chosen for internet and direct marketing retail of year 2012 on Total Cash\n",
            "1883 data pieces chosen for internet and direct marketing retail of year 2013 on Total Cash\n",
            "45 data pieces chosen for internet and direct marketing retail of year 2014 on Total Cash\n",
            "3 data pieces chosen for internet and direct marketing retail of year 2016 on Total Cash\n",
            "internet and direct marketing retail sector trend analysis finished\n",
            "\n",
            "\n",
            "\n",
            "-----------Analysis Ends-----------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PFFwCIw785JN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}