descriptive_stats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:52:36.420574Z",
     "iopub.status.busy": "2022-05-30T09:52:36.413505Z",
     "iopub.status.idle": "2022-05-30T09:52:38.440881Z",
     "shell.execute_reply": "2022-05-30T09:52:38.439595Z"
    }
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:52:38.452831Z",
     "iopub.status.busy": "2022-05-30T09:52:38.451450Z",
     "iopub.status.idle": "2022-05-30T09:52:39.608510Z",
     "shell.execute_reply": "2022-05-30T09:52:39.609687Z"
    }
   },
   "outputs": [],
   "source": [
    "f = open(\"cwe_mitigation_ids_temp.json\")\n",
    "w_mitigation = json.load(f)\n",
    "\n",
    "f = open(\"capec_mitigation_temp.json\")\n",
    "ap_mitigation = json.load(f)\n",
    "\n",
    "f = open(\"cwe_temp.json\")\n",
    "cwe = json.load(f)\n",
    "\n",
    "f = open(\"capec_temp.json\")\n",
    "capec = json.load(f)\n",
    "\n",
    "f = open(\"ap_names.json\")\n",
    "ap_names = json.load(f)\n",
    "\n",
    "f = open(\"cwe_names.json\")\n",
    "cwe_names = json.load(f)\n",
    "\n",
    "f = open(\"technique_names.json\")\n",
    "technique_names = json.load(f)\n",
    "\n",
    "f = open(\"tactic_names.json\")\n",
    "tactic_names = json.load(f)\n",
    "\n",
    "f = open(\"cve.json\")\n",
    "cve = json.load(f)\n",
    "\n",
    "with open(\"w_dict.json\", \"r\") as f:\n",
    "    w_dict = json.load(f)\n",
    "with open(\"ap_dict.json\", \"r\") as f:\n",
    "    ap_dict = json.load(f)\n",
    "\n",
    "with open(\"ap_mitigation_descriptions.json\", \"r\") as f:\n",
    "    ap_mitigation_descriptions = list(set(json.load(f)))\n",
    "with open(\"cwe_mitigation_descriptions.json\", \"r\") as f:\n",
    "    cwe_mitigation_descriptions = list(set(json.load(f)))\n",
    "with open(\"tech_mitigation_names.json\", \"r\") as f:\n",
    "    tech_mitigation_descriptions = list(set(json.load(f)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:52:39.619722Z",
     "iopub.status.busy": "2022-05-30T09:52:39.618161Z",
     "iopub.status.idle": "2022-05-30T09:52:41.897894Z",
     "shell.execute_reply": "2022-05-30T09:52:41.898920Z"
    }
   },
   "outputs": [],
   "source": [
    "encode = spacy.load(\"en_core_web_lg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:52:41.907730Z",
     "iopub.status.busy": "2022-05-30T09:52:41.906306Z",
     "iopub.status.idle": "2022-05-30T09:52:41.909533Z",
     "shell.execute_reply": "2022-05-30T09:52:41.910474Z"
    }
   },
   "outputs": [],
   "source": [
    "def process_text(text, stop_words=False, punct=False, lemma=False):\n",
    "    doc = encode(text)\n",
    "    result = []\n",
    "    for token in doc:\n",
    "        if punct:\n",
    "            if token.is_punct:\n",
    "                continue\n",
    "            if token.is_space:\n",
    "                continue\n",
    "        if stop_words:\n",
    "            if token.is_stop:\n",
    "                continue\n",
    "        if lemma:\n",
    "            result.append(token.lemma_)\n",
    "        else:\n",
    "            result.append(token.text)\n",
    "\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:52:41.917116Z",
     "iopub.status.busy": "2022-05-30T09:52:41.915739Z",
     "iopub.status.idle": "2022-05-30T09:52:41.920049Z",
     "shell.execute_reply": "2022-05-30T09:52:41.918842Z"
    }
   },
   "outputs": [],
   "source": [
    "aggregated_data = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:52:41.930279Z",
     "iopub.status.busy": "2022-05-30T09:52:41.928940Z",
     "iopub.status.idle": "2022-05-30T09:52:41.932618Z",
     "shell.execute_reply": "2022-05-30T09:52:41.933645Z"
    }
   },
   "outputs": [],
   "source": [
    "def compute_and_aggregate_data(key, input):\n",
    "    tokens_original = []\n",
    "    tokens_stop_words_removed = []\n",
    "    tokens_punct_removed = []\n",
    "    chars_original = []\n",
    "    chars_lemma = []\n",
    "\n",
    "    for text in input:\n",
    "        tokens_original.append(len(process_text(text)))\n",
    "        tokens_stop_words_removed.append(len(process_text(text, stop_words=True)))\n",
    "        tokens_punct_removed.append(len(process_text(text, punct=True)))\n",
    "        chars_original.append(len(text))\n",
    "        chars_lemma.append(len(\" \".join((process_text(text, lemma=True)))))\n",
    "\n",
    "    aggregated_data[key] = {}\n",
    "    aggregated_data[key][\"# of tokens\"] = np.mean(tokens_original)\n",
    "    aggregated_data[key][\"# of tokens after removing stop words\"] = np.mean(\n",
    "        tokens_stop_words_removed\n",
    "    )\n",
    "    aggregated_data[key][\"# of tokens after removing punctuation\"] = np.mean(\n",
    "        tokens_punct_removed\n",
    "    )\n",
    "    aggregated_data[key][\"# of chars\"] = np.mean(chars_original)\n",
    "    aggregated_data[key][\"# of chars after lemmatization\"] = np.mean(chars_lemma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:52:41.941280Z",
     "iopub.status.busy": "2022-05-30T09:52:41.939820Z",
     "iopub.status.idle": "2022-05-30T09:53:09.736443Z",
     "shell.execute_reply": "2022-05-30T09:53:09.735418Z"
    }
   },
   "outputs": [],
   "source": [
    "aggregated_data[\"CWE Names\"] = {}\n",
    "compute_and_aggregate_data(\"CWE Names\", cwe_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:53:09.743056Z",
     "iopub.status.busy": "2022-05-30T09:53:09.741926Z",
     "iopub.status.idle": "2022-05-30T09:53:25.221136Z",
     "shell.execute_reply": "2022-05-30T09:53:25.219485Z"
    }
   },
   "outputs": [],
   "source": [
    "aggregated_data[\"CAPEC Names\"] = {}\n",
    "compute_and_aggregate_data(\"CAPEC Names\", ap_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:53:25.231914Z",
     "iopub.status.busy": "2022-05-30T09:53:25.230073Z",
     "iopub.status.idle": "2022-05-30T09:54:24.337831Z",
     "shell.execute_reply": "2022-05-30T09:54:24.339049Z"
    }
   },
   "outputs": [],
   "source": [
    "aggregated_data[\"CWE Mitigations\"] = {}\n",
    "compute_and_aggregate_data(\"CWE Mitigations\", cwe_mitigation_descriptions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:54:24.346653Z",
     "iopub.status.busy": "2022-05-30T09:54:24.345219Z",
     "iopub.status.idle": "2022-05-30T09:54:57.897441Z",
     "shell.execute_reply": "2022-05-30T09:54:57.896428Z"
    }
   },
   "outputs": [],
   "source": [
    "aggregated_data[\"CAPEC Mitigations\"] = {}\n",
    "compute_and_aggregate_data(\"CAPEC Mitigations\", ap_mitigation_descriptions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:54:57.904002Z",
     "iopub.status.busy": "2022-05-30T09:54:57.903028Z",
     "iopub.status.idle": "2022-05-30T09:55:13.643668Z",
     "shell.execute_reply": "2022-05-30T09:55:13.642145Z"
    }
   },
   "outputs": [],
   "source": [
    "aggregated_data[\"Technique Names\"] = {}\n",
    "compute_and_aggregate_data(\"Technique Names\", technique_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:55:13.651867Z",
     "iopub.status.busy": "2022-05-30T09:55:13.650482Z",
     "iopub.status.idle": "2022-05-30T09:55:14.836558Z",
     "shell.execute_reply": "2022-05-30T09:55:14.835334Z"
    }
   },
   "outputs": [],
   "source": [
    "aggregated_data[\"Technique Mitigations\"] = {}\n",
    "compute_and_aggregate_data(\"Technique Mitigations\", tech_mitigation_descriptions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:55:14.844918Z",
     "iopub.status.busy": "2022-05-30T09:55:14.843641Z",
     "iopub.status.idle": "2022-05-30T09:55:15.275327Z",
     "shell.execute_reply": "2022-05-30T09:55:15.273973Z"
    }
   },
   "outputs": [],
   "source": [
    "aggregated_data[\"Tactic Names\"] = {}\n",
    "compute_and_aggregate_data(\"Tactic Names\", tactic_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T09:55:15.290651Z",
     "iopub.status.busy": "2022-05-30T09:55:15.285465Z",
     "iopub.status.idle": "2022-05-30T10:11:43.241991Z",
     "shell.execute_reply": "2022-05-30T10:11:43.240751Z"
    }
   },
   "outputs": [],
   "source": [
    "cve_names = []\n",
    "\n",
    "for v in cve:\n",
    "    if v[\"original_id\"].split(\"-\")[1] == \"2021\":\n",
    "        cve_names.append(v[\"metadata\"][\"description\"])\n",
    "\n",
    "aggregated_data[\"CVE Names\"] = {}\n",
    "compute_and_aggregate_data(\"CVE Names\", cve_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T10:11:43.277434Z",
     "iopub.status.busy": "2022-05-30T10:11:43.268329Z",
     "iopub.status.idle": "2022-05-30T10:11:43.317310Z",
     "shell.execute_reply": "2022-05-30T10:11:43.318202Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th># of tokens</th>\n",
       "      <th># of tokens after removing stop words</th>\n",
       "      <th># of tokens after removing punctuation</th>\n",
       "      <th># of chars</th>\n",
       "      <th># of chars after lemmatization</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CWE Names</th>\n",
       "      <td>6.806277</td>\n",
       "      <td>5.436147</td>\n",
       "      <td>6.129870</td>\n",
       "      <td>45.454545</td>\n",
       "      <td>45.696970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CAPEC Names</th>\n",
       "      <td>4.064103</td>\n",
       "      <td>3.591575</td>\n",
       "      <td>3.846154</td>\n",
       "      <td>28.653846</td>\n",
       "      <td>28.128205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CWE Mitigations</th>\n",
       "      <td>42.518929</td>\n",
       "      <td>25.731302</td>\n",
       "      <td>36.602955</td>\n",
       "      <td>245.997230</td>\n",
       "      <td>244.262235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CAPEC Mitigations</th>\n",
       "      <td>25.239437</td>\n",
       "      <td>16.034571</td>\n",
       "      <td>22.188220</td>\n",
       "      <td>146.505762</td>\n",
       "      <td>145.057618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Technique Names</th>\n",
       "      <td>2.830389</td>\n",
       "      <td>2.657244</td>\n",
       "      <td>2.729682</td>\n",
       "      <td>20.406360</td>\n",
       "      <td>20.367491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Technique Mitigations</th>\n",
       "      <td>3.023256</td>\n",
       "      <td>2.813953</td>\n",
       "      <td>2.906977</td>\n",
       "      <td>23.511628</td>\n",
       "      <td>23.604651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tactic Names</th>\n",
       "      <td>2.142857</td>\n",
       "      <td>2.071429</td>\n",
       "      <td>1.571429</td>\n",
       "      <td>13.714286</td>\n",
       "      <td>14.857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CVE Names</th>\n",
       "      <td>59.166632</td>\n",
       "      <td>41.176008</td>\n",
       "      <td>50.513994</td>\n",
       "      <td>330.769247</td>\n",
       "      <td>333.206460</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       # of tokens  # of tokens after removing stop words  \\\n",
       "CWE Names                 6.806277                               5.436147   \n",
       "CAPEC Names               4.064103                               3.591575   \n",
       "CWE Mitigations          42.518929                              25.731302   \n",
       "CAPEC Mitigations        25.239437                              16.034571   \n",
       "Technique Names           2.830389                               2.657244   \n",
       "Technique Mitigations     3.023256                               2.813953   \n",
       "Tactic Names              2.142857                               2.071429   \n",
       "CVE Names                59.166632                              41.176008   \n",
       "\n",
       "                       # of tokens after removing punctuation  # of chars  \\\n",
       "CWE Names                                            6.129870   45.454545   \n",
       "CAPEC Names                                          3.846154   28.653846   \n",
       "CWE Mitigations                                     36.602955  245.997230   \n",
       "CAPEC Mitigations                                   22.188220  146.505762   \n",
       "Technique Names                                      2.729682   20.406360   \n",
       "Technique Mitigations                                2.906977   23.511628   \n",
       "Tactic Names                                         1.571429   13.714286   \n",
       "CVE Names                                           50.513994  330.769247   \n",
       "\n",
       "                       # of chars after lemmatization  \n",
       "CWE Names                                   45.696970  \n",
       "CAPEC Names                                 28.128205  \n",
       "CWE Mitigations                            244.262235  \n",
       "CAPEC Mitigations                          145.057618  \n",
       "Technique Names                             20.367491  \n",
       "Technique Mitigations                       23.604651  \n",
       "Tactic Names                                14.857143  \n",
       "CVE Names                                  333.206460  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aggregated_df = pd.DataFrame(aggregated_data).T\n",
    "aggregated_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T10:11:43.324702Z",
     "iopub.status.busy": "2022-05-30T10:11:43.323799Z",
     "iopub.status.idle": "2022-05-30T10:11:43.329917Z",
     "shell.execute_reply": "2022-05-30T10:11:43.328548Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(\"descriptive_stats.json\", \"w\") as f:\n",
    "    json.dump(aggregated_data, f)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "98b0a9b7b4eaaa670588a142fd0a9b87eaafe866f1db4228be72b4211d12040f"
  },
  "kernelspec": {
   "display_name": "Python 3.7.10 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}