CreateDataSets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0bbec040-eb82-42b7-9e40-559d67abc2b5",
   "metadata": {},
   "source": [
    "# Sample Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ce6874ba-21c7-4762-ba4c-d8ee92ce76d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, requests, zipfile, json, time, xmltodict, datetime\n",
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch.helpers import bulk, parallel_bulk, streaming_bulk\n",
    "import sys, logging, ndjson\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import yaml\n",
    "from yaml import Loader\n",
    "from copy import deepcopy\n",
    "import warnings\n",
    "from IPython.display import JSON\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8d44e4db-dcb4-42dc-861e-2bb98860a85f",
   "metadata": {},
   "outputs": [],
   "source": [
    "container_host = 'localhost'\n",
    "elastic_user = 'elastic'\n",
    "elastic_password = 'elastic_password'\n",
    "es1_url = f'https://{container_host}:9200'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "448ac53a-15bc-485a-aa5e-13b57541f783",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_es_instance():\n",
    "    es = Elasticsearch(es1_url, \n",
    "                       basic_auth=(elastic_user, elastic_password), \n",
    "                       verify_certs=False)\n",
    "    return es"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d9f09939-6afb-4c53-bcb0-406a71c0eb93",
   "metadata": {},
   "outputs": [],
   "source": [
    "es = get_es_instance()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f5fe81c-f7b3-42fe-8454-3ad0eba30733",
   "metadata": {},
   "source": [
    "# Data Files Load"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "671a82c7-7b27-47fc-be90-bcec3e146183",
   "metadata": {},
   "source": [
    "## JSON Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "58d8629d-df3e-4eb8-9885-9ae16c54a2ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = './elasticsearch/data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dfaf5c58-164f-4f42-b455-05b388cad59a",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_files = ['recipes.json', 'movies.json', 'chicago-taxi-data.csv', 'web.log', 'firewall.log', \n",
    "              'webapp.csv\", webapp-tagged.csv', 'trips.csv', 'webapp-password-spraying-attemps.csv',\n",
    "              'webapp-new-data.csv'\n",
    "             ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1c225140-17a6-48c0-ab4e-b92b6b6e50a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading recipes.json\n",
      "Ingest errors: 0\n",
      "loading movies.json\n",
      "Ingest errors: 34307\n"
     ]
    }
   ],
   "source": [
    "for file in data_files:\n",
    "    if file.endswith('json'):\n",
    "        print(f'loading {file}')\n",
    "        with open(os.path.join(data_path, file), 'r') as f:\n",
    "            file_errors = 0\n",
    "            data = ndjson.loads(f.read())\n",
    "            index_name = file.split('.')[0]\n",
    "            for item in data:\n",
    "                if '_id' in item:\n",
    "                    item.pop('_id')\n",
    "                try:\n",
    "                    es.index(index=index_name, pipeline=None, document=item)\n",
    "                except:\n",
    "                    file_errors += 1\n",
    "            print(f'Ingest errors: {file_errors}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "078266df-6843-4761-b2dc-7a1ca85b78ca",
   "metadata": {},
   "source": [
    "# CSV Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "606b8dc1-1171-426e-b231-934a960f3142",
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_files = [x for x in data_files if x.split('.')[-1] == 'csv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "edebb9b3-4137-4426-8803-c0528f0e0095",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['chicago-taxi-data.csv',\n",
       " 'webapp.csv\", webapp-tagged.csv',\n",
       " 'trips.csv',\n",
       " 'webapp-password-spraying-attemps.csvwebapp-new-data.csv']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csv_files"
   ]
  },
  {
   "cell_type": "raw",
   "id": "15b06777-0cdc-4a4b-85fe-99e9c8d3deb1",
   "metadata": {},
   "source": [
    "Some type of critical bug is occurring above (filenames have blended together).  It will be a result of illegal wire tapping or some other failed introspection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8f5a6dfa-6765-4801-bdaf-820b177b5ea7",
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_files = [\n",
    "    'chicago-taxi-data.csv',\n",
    "    'webapp.csv', \n",
    "    'webapp-tagged.csv',\n",
    "    'webapp-password-spraying-attemps.csv',\n",
    "    'webapp-new-data.csv',\n",
    "    'trips.csv'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "8950fe52-0d5f-4bc4-a965-d8e0f53f646e",
   "metadata": {},
   "outputs": [],
   "source": [
    "taxis = pd.read_csv(os.path.join(data_path, 'chicago-taxi-data.csv')).T.to_dict()\n",
    "for record in taxis:\n",
    "    es.index(index='chicago_taxis', document=taxis[record])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "1382f0a5-0feb-473f-84b5-6563ab2ca7fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "webaapp_files = [x for x in csv_files if x.startswith('webapp')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "68675679-a42e-417c-a332-931134f1e0f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['webapp.csv',\n",
       " 'webapp-tagged.csv',\n",
       " 'webapp-password-spraying-attemps.csv',\n",
       " 'webapp-new-data.csv']"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "webaapp_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "274d3f03-1cd0-4ae2-bcf1-562f16e4b435",
   "metadata": {},
   "outputs": [],
   "source": [
    "webapp = pd.read_csv(os.path.join(data_path, webaapp_files[0])).T.to_dict()\n",
    "for record in webapp:\n",
    "    es.index(index='webapp', document=webapp[record])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "a8e0f2b9-6e23-4232-9e26-f858f07b9194",
   "metadata": {},
   "outputs": [],
   "source": [
    "webapp = pd.read_csv(os.path.join(data_path, webaapp_files[1])).T.to_dict()\n",
    "for record in webapp:\n",
    "    es.index(index='webapp-tagged', document=webapp[record])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "ce4095bb-f91f-4b20-a902-22ba0df3fe5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "webapp = pd.read_csv(os.path.join(data_path, webaapp_files[2])).T.to_dict()\n",
    "for record in webapp:\n",
    "    es.index(index='webapp-password-spraying-attempts', document=webapp[record])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "99ae0660-dfdd-47f2-ad11-fdbb3b86ba88",
   "metadata": {},
   "outputs": [],
   "source": [
    "webapp = pd.read_csv(os.path.join(data_path, webaapp_files[3])).T.to_dict()\n",
    "for record in webapp:\n",
    "    es.index(index='webapp-new-data', document=webapp[record])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "5bf78c99-fafe-4e6f-b9ed-389ff58f6a12",
   "metadata": {},
   "outputs": [],
   "source": [
    "trips = pd.read_csv(os.path.join(data_path, 'trips.csv')).T.to_dict()\n",
    "for record in trips:\n",
    "    es.index(index='trips', document=trips[record])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b045dfda-6453-4ec2-8c28-5fda6b5a451a",
   "metadata": {},
   "source": [
    "# Log Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "1f9444a0-ffb4-4de7-9008-b61dcdb273d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(data_path, 'web.log'), 'r') as f:\n",
    "    data = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "a219b33e-4303-4299-8d78-928b127e516a",
   "metadata": {},
   "outputs": [],
   "source": [
    "for line in data:\n",
    "    es.index(index='raw_web_log', document={'message':line})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "de4484b8-3957-418e-9e03-8192a6e8bf39",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(data_path, 'firewall.log'), 'r') as f:\n",
    "    data = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "4255595d-3583-491f-8826-7651d6ac3563",
   "metadata": {},
   "outputs": [],
   "source": [
    "for line in data:\n",
    "    es.index(index='firewall_log', document=json.loads(line))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fce2151-18ba-4b23-88d4-34c50e681378",
   "metadata": {},
   "source": [
    "# NVD Load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3378715a-6d1a-49f8-a7bb-866bb752b405",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.mkdir('demo')\n",
    "os.mkdir('demo/data')\n",
    "os.mkdir('demo/data/db')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f4acc80d-a332-437f-a73b-a92701bf60d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from NVD_Loader import NVDLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "339f7a23-c90a-4374-b0c4-f11b20d072b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "nvd_loader = NVDLoader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "abbcd9cb-6ab9-4a0c-941c-97024ec16e79",
   "metadata": {},
   "outputs": [],
   "source": [
    "nvd_loader.download_files(nvd_loader.historic_database, output_path=os.path.join(os.curdir, 'demo', 'data'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0b4ab39d-28e8-4085-ba8e-c20f13a47c32",
   "metadata": {},
   "outputs": [],
   "source": [
    "nvd_loader.extract_archives(data_path='./demo/data/', verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7da1a865-172e-457b-a8ab-dd94c48c18b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'total_read_documents': 187216, 'unique_cve_ids': 187216}\n"
     ]
    }
   ],
   "source": [
    "# Actual Document Count\n",
    "file_list = sorted(os.listdir(os.path.join(os.curdir, 'demo', 'data','db')))\n",
    "total_documents_to_be_ingested = nvd_loader.document_total_for_directory(file_list, data_path=os.path.join(os.curdir, 'demo', 'data', 'db'), verbose=False)\n",
    "print(total_documents_to_be_ingested)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "722ea524-67b8-4df0-993d-a77a1af0ab1f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "round: 1: Now ingesting nvdcve-1.1-2002.json from ./demo/data/db to nvd\n",
      "round: 2: Now ingesting nvdcve-1.1-2003.json from ./demo/data/db to nvd\n",
      "The ingest process is %9.52 complete\n",
      "round: 3: Now ingesting nvdcve-1.1-2004.json from ./demo/data/db to nvd\n",
      "round: 4: Now ingesting nvdcve-1.1-2005.json from ./demo/data/db to nvd\n",
      "The ingest process is %19.05 complete\n",
      "round: 5: Now ingesting nvdcve-1.1-2006.json from ./demo/data/db to nvd\n",
      "round: 6: Now ingesting nvdcve-1.1-2007.json from ./demo/data/db to nvd\n",
      "The ingest process is %28.57 complete\n",
      "round: 7: Now ingesting nvdcve-1.1-2008.json from ./demo/data/db to nvd\n",
      "round: 8: Now ingesting nvdcve-1.1-2009.json from ./demo/data/db to nvd\n",
      "The ingest process is %38.1 complete\n",
      "round: 9: Now ingesting nvdcve-1.1-2010.json from ./demo/data/db to nvd\n",
      "round: 10: Now ingesting nvdcve-1.1-2011.json from ./demo/data/db to nvd\n",
      "The ingest process is %47.62 complete\n",
      "round: 11: Now ingesting nvdcve-1.1-2012.json from ./demo/data/db to nvd\n",
      "round: 12: Now ingesting nvdcve-1.1-2013.json from ./demo/data/db to nvd\n",
      "The ingest process is %57.14 complete\n",
      "round: 13: Now ingesting nvdcve-1.1-2014.json from ./demo/data/db to nvd\n",
      "round: 14: Now ingesting nvdcve-1.1-2015.json from ./demo/data/db to nvd\n",
      "The ingest process is %66.67 complete\n",
      "round: 15: Now ingesting nvdcve-1.1-2016.json from ./demo/data/db to nvd\n",
      "round: 16: Now ingesting nvdcve-1.1-2017.json from ./demo/data/db to nvd\n",
      "The ingest process is %76.19 complete\n",
      "round: 17: Now ingesting nvdcve-1.1-2018.json from ./demo/data/db to nvd\n",
      "round: 18: Now ingesting nvdcve-1.1-2019.json from ./demo/data/db to nvd\n",
      "The ingest process is %85.71 complete\n",
      "round: 19: Now ingesting nvdcve-1.1-2020.json from ./demo/data/db to nvd\n",
      "round: 20: Now ingesting nvdcve-1.1-2021.json from ./demo/data/db to nvd\n",
      "The ingest process is %95.24 complete\n",
      "round: 21: Now ingesting nvdcve-1.1-2022.json from ./demo/data/db to nvd\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'187216 documents sent to elasticsearch'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create dataset with singleton (slow)\n",
    "nvd_loader.ingest_bulk_json_dataset(file_list, target_index='nvd', data_path=os.path.join(os.curdir, 'demo', 'data', 'db'), verbose=True, ingest_method='singleton')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e90d7a47-653a-4425-8ff1-89d63d250bc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "round: 1: Now ingesting nvdcve-1.1-recent.json from demo/data/db to nvd_recent\n",
      "Now removing nvdcve-1.1-recent.json.zip from demo/data\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create and Populate the nvd_recent index\n",
    "nvd_loader.create_nvd_recent_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9be05a4e-a788-4970-ab0b-d0f9cf796b95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Now removing nvdcpematch-1.0.json.zip from ./demo/data\n"
     ]
    }
   ],
   "source": [
    "# Load the CPE Match Data Set\n",
    "nvd_loader.create_cpe_match_index(target_index='cpe_match')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "50ef95b9-b970-4681-a03a-4739ed105383",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'health': 'yellow',\n",
       " 'status': 'open',\n",
       " 'index': 'cpe_dictionary',\n",
       " 'uuid': 'Cr2VALr2S-qSNLcYF3f4Kw',\n",
       " 'pri': '1',\n",
       " 'rep': '1',\n",
       " 'docs.count': '620856',\n",
       " 'docs.deleted': '0',\n",
       " 'store.size': '118.3mb',\n",
       " 'pri.store.size': '118.3mb'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Create the CPE Dictionary Dataset\n",
    "nvd_loader.download_files(nvd_loader.cpe_dictionary_feed, output_path=os.path.join(os.curdir, 'demo', 'data','dictionary'))\n",
    "nvd_loader.extract_archives(data_path=os.path.join(os.curdir, 'demo', 'data','dictionary'), verbose=False)\n",
    "target_file = [x for x in os.listdir(os.path.join(os.curdir, 'demo', 'data','db')) if x.endswith('.xml')][0]\n",
    "nvd_loader.load_cpe_dictionary(target_file=os.path.join(os.curdir, 'demo', 'data','db', target_file), target_index='cpe_dictionary', ingest_method='bulk')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "38428979-d465-47ea-8539-b3e77217698a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the CCE Data Set\n",
    "#nvd_loader.download_files(nvd_loader.cce_database_dictionary, output_path=os.path.join(os.curdir, 'demo', 'data', 'cce'), verbose=False)\n",
    "files = os.listdir(os.path.join(os.curdir, 'demo', 'data', 'cce'))\n",
    "nvd_loader.load_cce_data(file_list=files, data_path=os.path.join(os.curdir, 'demo', 'data', 'cce'), verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "89731325-b18f-4e89-9f22-3cb2962c85d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "round: 1: Now ingesting nvdcve-1.1-modified.json from demo/data/db to nvd\n",
      "Now removing nvdcve-1.1-modified.json.zip from demo/data\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'2147 documents sent to elasticsearch'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# update dataset\n",
    "nvd_loader.update_cve_data(nvd_loader.two_hour_stream_feeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f9edceb3-58d1-413c-a964-96f9c24739f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test Aggregations\n",
    "# Perform some Aggregations\n",
    "# https://qualysguard.qg2.apps.qualys.com/qwebhelp/fo_portal/setup/cvss_vector_strings.htm\n",
    "# Pipeline 1 (High Priority Network Attacks)\n",
    "remote_root_cves_requiring_network_signatures = {\n",
    "    \"bool\": {\n",
    "      \"must\":[\n",
    "        {\"match_phrase\": {\"impact.baseMetricV3.cvssV3.vectorString\": \"/AV:N/\"}},\n",
    "        {\"match_phrase\": {\"impact.baseMetricV3.cvssV3.vectorString\": \"/PR:N/\"}},\n",
    "        {\"match_phrase\": {\"impact.baseMetricV3.cvssV3.vectorString\": \"/UI:N/\"}},\n",
    "        {\"match_phrase\": {\"impact.baseMetricV3.cvssV3.vectorString\": \"/I:H/\"}},\n",
    "        {\"match_phrase\": {\"impact.baseMetricV2.obtainAllPrivilege\": \"true\"}}\n",
    "      ],\n",
    "      \"filter\": [\n",
    "        {\"range\": {\"lastModifiedDate\": {\"gte\": \"2008-01-01\"}}}\n",
    "        ]\n",
    "    }\n",
    "}\n",
    "high_priority_signature_targets = nvd_loader.client.search(index='nvd', query=remote_root_cves_requiring_network_signatures)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6772fde4-ff47-4f4d-85fe-fd5388d984ee",
   "metadata": {},
   "source": [
    "# Elastic Common Schema Generator Templates Load"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26fe3a2d-927f-41a1-ad33-6cbf5d756381",
   "metadata": {},
   "source": [
    "## Host Records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c13a26d5-a895-4c1a-95dd-9c22ddc00866",
   "metadata": {},
   "outputs": [],
   "source": [
    "from elastic_common_schema_generator import ECSRecords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0da36d44-c461-4d7f-85c6-c30018ce3eca",
   "metadata": {},
   "outputs": [],
   "source": [
    "record_generator = ECSRecords()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "71f13c29-d783-44a0-a5d4-f0b255ce747e",
   "metadata": {},
   "outputs": [],
   "source": [
    "specification = record_generator.pull_schema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "33def587-d33f-475c-9c27-c4d3511a0380",
   "metadata": {},
   "outputs": [],
   "source": [
    "processor_list, code_start = record_generator.generate_processor_list(schema=specification, object_type='host')"
   ]
  },
  {
   "cell_type": "raw",
   "id": "13255661-66d3-45d4-9b1d-257c3666bce0",
   "metadata": {},
   "source": [
    "This step requires the NVD dataset to be loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "802c3814-c3e5-4c2a-aeac-54b8eab26a1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "demo_data_set = record_generator.create_records({'document_type':'demo_dataset'}, ['host', 'cve'], count=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e5302ab4-d6ba-4eef-8646-509f6de79014",
   "metadata": {},
   "outputs": [],
   "source": [
    "for document in demo_data_set:\n",
    "    document['@timestamp'] = datetime.datetime.now()\n",
    "    es.index(document=document, index='demo_machines')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6f33a2fb-6040-4d4e-84cf-6fccda4d4b92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'took': 1,\n",
       " 'timed_out': False,\n",
       " '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0},\n",
       " 'hits': {'total': {'value': 1000, 'relation': 'eq'},\n",
       "  'max_score': 1.0,\n",
       "  'hits': [{'_index': 'demo_machines',\n",
       "    '_id': 'Hof254ABnDNUNBHfihCf',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'document_type': 'demo_dataset',\n",
       "     'host.architecture': 'arm64',\n",
       "     'host.boot.id': '2d5da9c6-a6f5-4318-819f-69853dcbe11f',\n",
       "     'host.cpu.usage': '',\n",
       "     'host.disk.read.bytes': '',\n",
       "     'host.disk.write.bytes': '',\n",
       "     'host.domain': 'thefirm.remote',\n",
       "     'host.geo.city_name': 'New York',\n",
       "     'host.geo.continent_code': 'NA',\n",
       "     'host.geo.continent_name': 'North America',\n",
       "     'host.geo.country_iso_code': 'US',\n",
       "     'host.geo.country_name': 'United States',\n",
       "     'host.geo.location': {'lat': 40.75464371951375,\n",
       "      'lon': -73.98552474566057},\n",
       "     'host.geo.name': '',\n",
       "     'host.geo.postal_code': '',\n",
       "     'host.geo.region_iso_code': '',\n",
       "     'host.geo.region_name': '',\n",
       "     'host.geo.timezone': '',\n",
       "     'host.hostname': 'desktop-73.campbell.net.thefirm.remote',\n",
       "     'host.id': '9b2bcf69-c1fd-4672-bd1a-51051a3c1287',\n",
       "     'host.ip': '182.97.194.134',\n",
       "     'host.mac': ['5c:d5:0c:2e:28:de'],\n",
       "     'host.name': 'desktop-73.campbell.net.thefirm.remote',\n",
       "     'host.network.egress.bytes': 16583484.991296,\n",
       "     'host.network.egress.packets': 27166,\n",
       "     'host.network.ingress.bytes': 22158191.584,\n",
       "     'host.network.ingress.packets': 8424,\n",
       "     'host.os.family': 'redhat',\n",
       "     'host.os.full': 'redhat',\n",
       "     'host.os.kernel': '',\n",
       "     'host.os.name': 'Redhat',\n",
       "     'host.os.platform': '',\n",
       "     'host.os.type': '',\n",
       "     'host.os.version': '',\n",
       "     'host.pid_ns_ino': '',\n",
       "     'host.type': 'kubernetes_node',\n",
       "     'host.uptime': '',\n",
       "     'cve_list': ['CVE-2006-3638',\n",
       "      'CVE-1999-0236',\n",
       "      'CVE-2022-24890',\n",
       "      'CVE-2004-0719',\n",
       "      'CVE-2005-2225',\n",
       "      'CVE-2006-5152',\n",
       "      'CVE-2004-1862'],\n",
       "     '@timestamp': '2022-05-21T14:52:15.804585'}}]}}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.search(size=1, index='demo_machines').raw"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4181db4-cfcb-4808-ab2d-9bf519b8186b",
   "metadata": {},
   "source": [
    "# HR Data Generator Load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e93c0673-4dd3-4edc-9392-572e8e34ac2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting hr_generator.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile hr_generator.py\n",
    "import os, requests, zipfile, json, random, datetime\n",
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch.helpers import bulk, parallel_bulk, streaming_bulk\n",
    "from faker import Faker\n",
    "import yaml\n",
    "from yaml import Loader\n",
    "from copy import deepcopy\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "class HRGenerator:\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.container_host = 'localhost'\n",
    "        self.elastic_user = 'elastic'\n",
    "        self.elastic_password = 'elastic_playground'\n",
    "        self.es1_url = f'https://{self.container_host}:9200'\n",
    "        self.es = Elasticsearch(self.es1_url, \n",
    "                       basic_auth=(self.elastic_user, self.elastic_password), \n",
    "                       verify_certs=False)\n",
    "        self.faker = Faker()\n",
    "    \n",
    "    def hr_data_generator(self, document_count):\n",
    "        faker = self.faker\n",
    "        documents = []\n",
    "        for i in range(document_count):\n",
    "            document = {}\n",
    "            document['@timestamp'] = datetime.datetime.now().isoformat()\n",
    "            document['first_name'] = faker.first_name()\n",
    "            document['last_name'] = faker.last_name()\n",
    "            document['middle_name'] = faker.random_element([faker.first_name(), faker.last_name()])\n",
    "            document['telephone_number'] = faker.phone_number()\n",
    "            document['email'] = faker.email()\n",
    "            document['employee-id'] = faker.uuid4()\n",
    "            document['bio'] = faker.text(max_nb_chars=200)\n",
    "            document['ip_address'] = faker.ipv4_public()\n",
    "            document['hired_date'] = faker.date()\n",
    "            document['paid_amount'] = random.choice(range(10000,1000000))\n",
    "            document['days_in_service'] = (datetime.datetime.now() - datetime.datetime.strptime(document['hired_date'], '%Y-%m-%d')).days\n",
    "            document['skills'] = self.skills_list(faker)\n",
    "            documents.append(document)\n",
    "        return documents\n",
    "    \n",
    "    def skills_list(self, faker):\n",
    "        path_list = [\n",
    "            \"technical_drawing\",\n",
    "            \"programming\",\n",
    "            \"photography\",\n",
    "            \"linguistics\",\n",
    "            \"psychology\",\n",
    "            \"chemistry\",\n",
    "            \"medical_research\",\n",
    "            \"medic\",\n",
    "            \"sports\",\n",
    "            \"martial arts\",\n",
    "            \"photographic memory\",\n",
    "            \"empathic\",\n",
    "            \"polygraph\",\n",
    "            \"forensics\",\n",
    "            \"land survey\",\n",
    "            \"aeronautics\",\n",
    "            \"pilot\",\n",
    "            \"mechanic\",\n",
    "            \"trader\",\n",
    "            \"historian\",\n",
    "            \"veterenarian\",\n",
    "            \"agriculture\",\n",
    "        ]\n",
    "        personal_list = faker.random_elements(path_list, unique=True, length=random.choice(range(1,4)))\n",
    "        output = []\n",
    "        for item in personal_list:\n",
    "            doc = {'skill_name':item, 'skill_level':random.choice(range(100))}\n",
    "            output.append(doc)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5c50204a-5513-4e70-8e20-9813b059f6b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from hr_generator import HRGenerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cecac443-f19d-49b3-929e-227ab233a15d",
   "metadata": {},
   "outputs": [],
   "source": [
    "hr_generator = HRGenerator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "19fc9b0e-b298-48aa-ab38-c51ff48f5a75",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = hr_generator.hr_data_generator(document_count=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3e699d9e-e4d0-44be-9ef8-50db391c702d",
   "metadata": {},
   "outputs": [],
   "source": [
    "for record in data:\n",
    "    es.index(index='persons', document=record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bd9dbba0-9322-494a-8de8-691853afd0c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'took': 1,\n",
       " 'timed_out': False,\n",
       " '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0},\n",
       " 'hits': {'total': {'value': 1000, 'relation': 'eq'},\n",
       "  'max_score': 1.0,\n",
       "  'hits': [{'_index': 'persons',\n",
       "    '_id': '6Yf-54ABnDNUNBHfWn_c',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'@timestamp': '2022-05-21T14:59:55.805653',\n",
       "     'first_name': 'Lori',\n",
       "     'last_name': 'Walker',\n",
       "     'middle_name': 'Roberta',\n",
       "     'telephone_number': '610-609-0481x214',\n",
       "     'email': 'ellispatrick@example.net',\n",
       "     'employee-id': 'da414e4b-26b1-4e30-8476-3d2e27f3a78b',\n",
       "     'bio': 'Leave her himself reflect. Western street drug store return future reflect bad. Current determine color control.',\n",
       "     'ip_address': '166.213.219.36',\n",
       "     'hired_date': '1971-09-28',\n",
       "     'paid_amount': 732927,\n",
       "     'days_in_service': 18498,\n",
       "     'skills': [{'skill_name': 'linguistics', 'skill_level': 4}]}}]}}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.search(index='persons', size=1).raw"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}