data-generator-multicore-normal-zero.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "from datetime import timedelta\n",
    "import random\n",
    "import math\n",
    "import concurrent.futures\n",
    "from multiprocessing import Manager"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Almost all configurable variables are kept here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "days_to_go_back = 1\n",
    "frequency = '1S'   #every second\n",
    "\n",
    "mac_list=[\n",
    "    \"F6:B8:15:1E:A8:02\",\n",
    "    \"6C:D0:7E:63:10:F1\",\n",
    "    \"97:46:62:3B:15:2C\",\n",
    "    \"05:A6:2E:30:98:E2\"\n",
    "]\n",
    "#one flag for each mac\n",
    "\n",
    "#should be between 9 (am) and 16 (pm) to be safe\n",
    "list_attack_down=[\n",
    "    {\n",
    "        \"start\":datetime(2023,12,6,12,0,0),\n",
    "        \"stop\":datetime(2023,12,6,12,15,0),\n",
    "        \"plc\":[1,3]\n",
    "    }\n",
    "]\n",
    "#should be between 21 (pm) and (6am) to be safe\n",
    "list_attack_up=[\n",
    "    {\n",
    "        \"start\":datetime(2023,12,6,3,0,0),\n",
    "        \"stop\":datetime(2023,12,6,3,15,0),\n",
    "        \"plc\":[2]\n",
    "    }\n",
    "]\n",
    "\n",
    "plc_state=[\"ok\",\"ok\",\"ok\",\"ok\"]\n",
    "plc_counter=[0,0,0,0]\n",
    "max_cycle_time=[41,48,34,74]\n",
    "#mu is the mean\n",
    "mu_cycle_time=[24,27,21,38]\n",
    "mu_ram=[7464,8197,6465,9883]\n",
    "mu_packets=20\n",
    "mu_packets_size=50\n",
    "#sigma is the standard deviation\n",
    "sigma_cycle_time=4\n",
    "sigma_ram=256\n",
    "sigma_packets=3\n",
    "sigma_packets_size=15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#day of weeks\n",
    "#Monday =0 .. Sunday=6\n",
    "work_days=range(5)\n",
    "\n",
    "#window_check must be in seconds and \n",
    "new_day_check_time = datetime(1,1,1,0,0,0,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "## those variables are for future features\n",
    "#down period is in seconds\n",
    "down_for_all_day=False\n",
    "#up period is in seconds\n",
    "up_period=3600\n",
    "\n",
    "start_working_time=8\n",
    "end_working_time=17\n",
    "\n",
    "packets_nmap_probability=0.01\n",
    "packets_big_probability=0.01\n",
    "#probabilità che i sistemi siano spenti durante un giorno normale\n",
    "random_down_for_all_day_probability=0.01\n",
    "#probabilità che i sistemi siano accesi un ora durante un giorno di shutdown\n",
    "random_up_probability=0.01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## variables regarding the temporal space to generate the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = (datetime.today() - timedelta(days = days_to_go_back)).replace(hour=0,minute=0,second=0, microsecond=0)\n",
    "stop_date = datetime.today().replace(hour=0,minute=0,second=0, microsecond=0)\n",
    "\n",
    "datelist = pd.date_range(start_date, stop_date, freq='24H').tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "2023-12-06 00:00:00\n"
     ]
    }
   ],
   "source": [
    "# how many rows are created\n",
    "print(len(datelist))\n",
    "#to check that the first line is actually the day you chose\n",
    "print(datelist[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions to simplify the code down and making easier to mantain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_cpu_cycle_time(date, mac_index):\n",
    "    return math.floor(random.normalvariate(mu=mu_cycle_time[mac_index],sigma=sigma_cycle_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_ram_usage(date, mac_index):\n",
    "    return math.floor(random.normalvariate(mu=mu_ram[mac_index],sigma=sigma_ram))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ping packets are 56 bytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_rx_size(packet_n, date, mac_index):\n",
    "    result = 0\n",
    "    for i in range(packet_n):\n",
    "        result += random.normalvariate(mu=mu_packets_size,sigma=sigma_packets_size)\n",
    "    return math.floor(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_tx_size(packet_n, date, mac_index):\n",
    "    result = 0\n",
    "    for i in range(packet_n):\n",
    "        result += random.normalvariate(mu=mu_packets_size,sigma=sigma_packets_size)\n",
    "    return math.floor(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## randomize the starting time with a margin of 15 minnutes, to make shure the data are not too predictable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO: refactoring with variables and not static integers\n",
    "\n",
    "def randomize_start_and_stop():\n",
    "    day_start_time = datetime(1,1,1,0,0,0)\n",
    "    day_stop_time = datetime(1,1,1,0,0,0)\n",
    "\n",
    "    day_start_time = day_start_time.replace(hour=random.randint(7,8))\n",
    "    if (day_start_time.hour == 8):\n",
    "        day_start_time = day_start_time.replace(minute=random.randint(0,15))\n",
    "    else:\n",
    "        day_start_time = day_start_time.replace(minute=59-random.randint(0,15))\n",
    "\n",
    "    day_stop_time = day_start_time.replace(hour=random.randint(16,17))\n",
    "    if (day_stop_time.hour == 17):\n",
    "        day_stop_time = day_stop_time.replace(minute=random.randint(0,15))\n",
    "    else:\n",
    "        day_stop_time = day_stop_time.replace(minute=59-random.randint(0,15))\n",
    "\n",
    "    return day_start_time, day_stop_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_plc_state(day_start_time, day_stop_time, date, plc_index):\n",
    "    # Check if the current date is within the working hours\n",
    "    if date.dayofweek in work_days and day_start_time.time() <= date.time() <= day_stop_time.time():\n",
    "        # Check for attack_down\n",
    "        #attack_down = any(att['start'] <= date <= att['stop'] and att['plc'] == plc_index for att in list_attack_down)\n",
    "        attack_down = any(att['start'] <= date <= att['stop'] and plc_index in att['plc'] for att in list_attack_down)\n",
    "\n",
    "        if not attack_down:\n",
    "            plc_state[plc_index] = \"ok\"\n",
    "        else:\n",
    "            plc_state[plc_index] = \"atk_down\"\n",
    "    else:\n",
    "        # Check for attack_up\n",
    "        #attack_up = any(att['start'] <= date <= att['stop'] and att['plc'] == plc_index for att in list_attack_up)\n",
    "        attack_up = any(att['start'] <= date <= att['stop'] and plc_index in att['plc'] for att in list_attack_up)\n",
    "\n",
    "        if not attack_up:\n",
    "            plc_state[plc_index] = \"off\"\n",
    "        else:\n",
    "            plc_state[plc_index] = \"atk_up\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO: make the function right and add in the upper cell\n",
    "def check_plc_status_old(date, plc_index):\n",
    "\n",
    "    if(date.dayofweek in work_days):\n",
    "        if (plc_state[plc_index]==False):\n",
    "            if (plc_counter[plc_index]>=up_period):\n",
    "                plc_state[plc_index]=True\n",
    "            plc_counter += 1\n",
    "\n",
    "        elif (random.random()<=random_up_probability):\n",
    "            plc_state[plc_index]=False\n",
    "\n",
    "    elif (plc_state[plc_index]==True):\n",
    "        if (plc_counter[plc_index]>=down_for_all_day):\n",
    "            plc_state[plc_index]=False\n",
    "        plc_counter += 1\n",
    "    \n",
    "    elif (random.random()<=random_down_for_all_day_probability):\n",
    "            plc_state[plc_index]=True\n",
    "            plc_counter += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## REAL process function that generate the data\n",
    "This function get a date, devide it with a frequency that can be 10 or 1 second (manual mod), generate the rows of each subdivision, append all of them in a list, and then return the result."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#columns names\n",
    "columns =[\"timestamp\",\"mac_address\",\"cpu_max_cycle\",\"cpu_current_cycle\",\"ram_usage\",\"rx_packets\",\"rx_bytes\",\"tx_packets\",\"tx_bytes\",\"flag\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_data(date):\n",
    "\n",
    "    sublist = pd.date_range(date, date + timedelta(hours=23,minutes=59,seconds=59), freq=frequency).tolist()\n",
    "\n",
    "    result = []\n",
    "\n",
    "    day_start_time, day_stop_time = randomize_start_and_stop()\n",
    "    \n",
    "    for subdate in sublist:\n",
    "            \n",
    "        for plc in range(0,len(mac_list)):\n",
    "            check_plc_state(day_start_time, day_stop_time, subdate, plc)\n",
    "\n",
    "            if plc_state[plc] in [\"ok\", \"atk_up\"]:\n",
    "                cpu_time = generate_cpu_cycle_time(subdate, plc)\n",
    "                ram_usage = generate_ram_usage(subdate, plc)\n",
    "                tx_packets = math.floor(random.normalvariate(mu_packets, sigma_packets))\n",
    "                tx_bytes = generate_tx_size(tx_packets, subdate, plc)\n",
    "                rx_packets = math.floor(random.normalvariate(mu_packets, sigma_packets))\n",
    "                rx_bytes = generate_rx_size(rx_packets, subdate, plc)\n",
    "                new_row = {'timestamp': subdate,\n",
    "                        'mac_address': mac_list[plc],\n",
    "                        'cpu_max_cycle': max_cycle_time[plc],\n",
    "                        'cpu_current_cycle': cpu_time,\n",
    "                        'ram_usage': ram_usage,\n",
    "                        'rx_packets': rx_packets,\n",
    "                        'rx_bytes': rx_bytes,\n",
    "                        'tx_packets': tx_packets,\n",
    "                        'tx_bytes': tx_bytes,\n",
    "                        'flag': plc_state[plc]}\n",
    "                result.append(new_row)\n",
    "            else:\n",
    "                new_row = {'timestamp': subdate,\n",
    "                        'mac_address': mac_list[plc],\n",
    "                        'cpu_max_cycle': 0,\n",
    "                        'cpu_current_cycle': 0,\n",
    "                        'ram_usage': 0,\n",
    "                        'rx_packets': 0,\n",
    "                        'rx_bytes': 0,\n",
    "                        'tx_packets': 0,\n",
    "                        'tx_bytes': 0,\n",
    "                        'flag': plc_state[plc]}\n",
    "                result.append(new_row)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parallelising part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use a ThreadPoolExecutor for parallel processing\n",
    "max_process_workers = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "with concurrent.futures.ProcessPoolExecutor(max_workers=max_process_workers) as executor:\n",
    "    futures2 = [executor.submit(process_data, date) for date in datelist]\n",
    "    \n",
    "# Wait for all tasks to complete\n",
    "results2 = [future.result() for future in concurrent.futures.as_completed(futures2)]\n",
    "# Flatten the list of lists into a single list of dictionaries\n",
    "flat_results = [item for sublist in results2 for item in sublist]\n",
    "# Create a DataFrame from the results\n",
    "df = pd.DataFrame(flat_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(691200, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mac_address</th>\n",
       "      <th>cpu_max_cycle</th>\n",
       "      <th>cpu_current_cycle</th>\n",
       "      <th>ram_usage</th>\n",
       "      <th>rx_packets</th>\n",
       "      <th>rx_bytes</th>\n",
       "      <th>tx_packets</th>\n",
       "      <th>tx_bytes</th>\n",
       "      <th>flag</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timestamp</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2023-12-07 00:00:00</th>\n",
       "      <td>F6:B8:15:1E:A8:02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>off</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-07 00:00:00</th>\n",
       "      <td>6C:D0:7E:63:10:F1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>off</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-07 00:00:00</th>\n",
       "      <td>97:46:62:3B:15:2C</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>off</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-07 00:00:00</th>\n",
       "      <td>05:A6:2E:30:98:E2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>off</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-07 00:00:01</th>\n",
       "      <td>F6:B8:15:1E:A8:02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>off</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           mac_address  cpu_max_cycle  cpu_current_cycle  \\\n",
       "timestamp                                                                  \n",
       "2023-12-07 00:00:00  F6:B8:15:1E:A8:02              0                  0   \n",
       "2023-12-07 00:00:00  6C:D0:7E:63:10:F1              0                  0   \n",
       "2023-12-07 00:00:00  97:46:62:3B:15:2C              0                  0   \n",
       "2023-12-07 00:00:00  05:A6:2E:30:98:E2              0                  0   \n",
       "2023-12-07 00:00:01  F6:B8:15:1E:A8:02              0                  0   \n",
       "\n",
       "                     ram_usage  rx_packets  rx_bytes  tx_packets  tx_bytes  \\\n",
       "timestamp                                                                    \n",
       "2023-12-07 00:00:00          0           0         0           0         0   \n",
       "2023-12-07 00:00:00          0           0         0           0         0   \n",
       "2023-12-07 00:00:00          0           0         0           0         0   \n",
       "2023-12-07 00:00:00          0           0         0           0         0   \n",
       "2023-12-07 00:00:01          0           0         0           0         0   \n",
       "\n",
       "                    flag  \n",
       "timestamp                 \n",
       "2023-12-07 00:00:00  off  \n",
       "2023-12-07 00:00:00  off  \n",
       "2023-12-07 00:00:00  off  \n",
       "2023-12-07 00:00:00  off  \n",
       "2023-12-07 00:00:01  off  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.set_index('timestamp', inplace=True)\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"norm_zero_{}_{}d_with_attacks.csv\".format(frequency,days_to_go_back))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}