Crime.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "e713ee62-1ec9-44c2-8275-11120306855a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Dataset:\n",
    "#https://data.cityofchicago.org/Public-Safety/Crimes-2001-to-Present/ijzp-q8t2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b366b2d8-536a-42b1-9eff-f7fd16a8fe23",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n"
     ]
    }
   ],
   "source": [
    "# import libraries\n",
    "\n",
    "import boto3, sagemaker\n",
    "from sagemaker.amazon.amazon_estimator import get_image_uri\n",
    "from sagemaker.session import s3_input, Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6ae39adb-3d7b-4903-a406-5b25eb411702",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "bucket = 'crime-bucket-2001/Raw/'\n",
    "data_key = 'Crimes_2001_to_Present.csv'\n",
    "output_path = 's3://{}/{}/output'.format(bucket, data_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ef3b4405-2a93-4125-bd14-2bd558ff7458",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pyspark==3.5.0\n",
      "  Downloading pyspark-3.5.0.tar.gz (316.9 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m316.9/316.9 MB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting py4j==0.10.9.7 (from pyspark==3.5.0)\n",
      "  Downloading py4j-0.10.9.7-py2.py3-none-any.whl (200 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m200.5/200.5 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
      "  Building wheel for pyspark (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pyspark: filename=pyspark-3.5.0-py2.py3-none-any.whl size=317425344 sha256=1620db9d701710b902461f71d8d043a34bd71dae9f57345208ac5b3efc9ce23d\n",
      "  Stored in directory: /root/.cache/pip/wheels/41/4e/10/c2cf2467f71c678cfc8a6b9ac9241e5e44a01940da8fbb17fc\n",
      "Successfully built pyspark\n",
      "Installing collected packages: py4j, pyspark\n",
      "Successfully installed py4j-0.10.9.7 pyspark-3.5.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Collecting gdown\n",
      "  Downloading gdown-4.7.1-py3-none-any.whl (15 kB)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from gdown) (3.6.0)\n",
      "Requirement already satisfied: requests[socks] in /opt/conda/lib/python3.10/site-packages (from gdown) (2.31.0)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from gdown) (1.16.0)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from gdown) (4.64.1)\n",
      "Requirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.10/site-packages (from gdown) (4.11.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /opt/conda/lib/python3.10/site-packages (from beautifulsoup4->gdown) (2.3.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests[socks]->gdown) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests[socks]->gdown) (3.3)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests[socks]->gdown) (2.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests[socks]->gdown) (2023.7.22)\n",
      "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /opt/conda/lib/python3.10/site-packages (from requests[socks]->gdown) (1.7.1)\n",
      "Installing collected packages: gdown\n",
      "Successfully installed gdown-4.7.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install pyspark==3.5.0\n",
    "!pip install gdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f38f624f-ef3e-4446-9422-4dcd570ff5e7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting sagemaker_pyspark\n",
      "  Downloading sagemaker_pyspark-1.4.5.tar.gz (181.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m181.5/181.5 MB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting pyspark==3.3.0 (from sagemaker_pyspark)\n",
      "  Downloading pyspark-3.3.0.tar.gz (281.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m281.3/281.3 MB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from sagemaker_pyspark) (1.25.2)\n",
      "Collecting py4j==0.10.9.5 (from pyspark==3.3.0->sagemaker_pyspark)\n",
      "  Downloading py4j-0.10.9.5-py2.py3-none-any.whl (199 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.7/199.7 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hBuilding wheels for collected packages: sagemaker_pyspark, pyspark\n",
      "  Building wheel for sagemaker_pyspark (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sagemaker_pyspark: filename=sagemaker_pyspark-1.4.5-py3-none-any.whl size=181610592 sha256=b5bec89bda186807d8e802a0262bdfc89e5421457538d260607068d884124951\n",
      "  Stored in directory: /root/.cache/pip/wheels/ea/cb/32/140bffbc4ad8465e99b41cd848723a287b07650044483c62fa\n",
      "  Building wheel for pyspark (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pyspark: filename=pyspark-3.3.0-py2.py3-none-any.whl size=281764004 sha256=756b22fae63021ab0566bc35cce8d9fb59534c8f42e798a42dcf1180f29f72e9\n",
      "  Stored in directory: /root/.cache/pip/wheels/81/9c/6c/d5200fcf351ffa39cbe09911e99703283624cd037df58070d9\n",
      "Successfully built sagemaker_pyspark pyspark\n",
      "Installing collected packages: py4j, pyspark, sagemaker_pyspark\n",
      "  Attempting uninstall: py4j\n",
      "    Found existing installation: py4j 0.10.9.7\n",
      "    Uninstalling py4j-0.10.9.7:\n",
      "      Successfully uninstalled py4j-0.10.9.7\n",
      "  Attempting uninstall: pyspark\n",
      "    Found existing installation: pyspark 3.5.0\n",
      "    Uninstalling pyspark-3.5.0:\n",
      "      Successfully uninstalled pyspark-3.5.0\n",
      "Successfully installed py4j-0.10.9.5 pyspark-3.3.0 sagemaker_pyspark-1.4.5\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install sagemaker_pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d1d597ec-b038-4dd4-967d-e974f243ef8b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql import DataFrame\n",
    "from pyspark.sql.types import *\n",
    "from pyspark.sql.functions import *\n",
    "import sagemaker_pyspark\n",
    "from sagemaker_pyspark import IAMRole"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "803546e6-7b10-4fad-acfb-596294b52f48",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (2672543745.py, line 5)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  Cell \u001b[0;32mIn[55], line 5\u001b[0;36m\u001b[0m\n\u001b[0;31m    sudo yum -y update\u001b[0m\n\u001b[0m         ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "#!/bin/bash\n",
    "\n",
    "set -eux\n",
    "\n",
    "sudo yum -y update\n",
    "sudo yum -y install java-1.8.0-openjdk-devel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "0882983b-c285-4aab-9e3a-b3915fc247e3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "JAVA_HOME is not set\n"
     ]
    },
    {
     "ename": "PySparkRuntimeError",
     "evalue": "[JAVA_GATEWAY_EXITED] Java gateway process exited before sending its port number.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mPySparkRuntimeError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[49], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m spark \u001b[38;5;241m=\u001b[39m \u001b[43mSparkSession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbuilder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mappName\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mCrimeInsights\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetOrCreate\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pyspark/sql/session.py:497\u001b[0m, in \u001b[0;36mSparkSession.Builder.getOrCreate\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    495\u001b[0m     sparkConf\u001b[38;5;241m.\u001b[39mset(key, value)\n\u001b[1;32m    496\u001b[0m \u001b[38;5;66;03m# This SparkContext may be an existing one.\u001b[39;00m\n\u001b[0;32m--> 497\u001b[0m sc \u001b[38;5;241m=\u001b[39m \u001b[43mSparkContext\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetOrCreate\u001b[49m\u001b[43m(\u001b[49m\u001b[43msparkConf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    498\u001b[0m \u001b[38;5;66;03m# Do not update `SparkConf` for existing `SparkContext`, as it's shared\u001b[39;00m\n\u001b[1;32m    499\u001b[0m \u001b[38;5;66;03m# by all sessions.\u001b[39;00m\n\u001b[1;32m    500\u001b[0m session \u001b[38;5;241m=\u001b[39m SparkSession(sc, options\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_options)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pyspark/context.py:515\u001b[0m, in \u001b[0;36mSparkContext.getOrCreate\u001b[0;34m(cls, conf)\u001b[0m\n\u001b[1;32m    513\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m SparkContext\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[1;32m    514\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m SparkContext\u001b[38;5;241m.\u001b[39m_active_spark_context \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 515\u001b[0m         \u001b[43mSparkContext\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconf\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mSparkConf\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    516\u001b[0m     \u001b[38;5;28;01massert\u001b[39;00m SparkContext\u001b[38;5;241m.\u001b[39m_active_spark_context \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    517\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m SparkContext\u001b[38;5;241m.\u001b[39m_active_spark_context\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pyspark/context.py:201\u001b[0m, in \u001b[0;36mSparkContext.__init__\u001b[0;34m(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls, udf_profiler_cls, memory_profiler_cls)\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m gateway \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m gateway\u001b[38;5;241m.\u001b[39mgateway_parameters\u001b[38;5;241m.\u001b[39mauth_token \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    196\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    197\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou are trying to pass an insecure Py4j gateway to Spark. This\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    198\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m is not allowed as it is a security risk.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    199\u001b[0m     )\n\u001b[0;32m--> 201\u001b[0m \u001b[43mSparkContext\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ensure_initialized\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgateway\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgateway\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    202\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    203\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_do_init(\n\u001b[1;32m    204\u001b[0m         master,\n\u001b[1;32m    205\u001b[0m         appName,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    215\u001b[0m         memory_profiler_cls,\n\u001b[1;32m    216\u001b[0m     )\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pyspark/context.py:436\u001b[0m, in \u001b[0;36mSparkContext._ensure_initialized\u001b[0;34m(cls, instance, gateway, conf)\u001b[0m\n\u001b[1;32m    434\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m SparkContext\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[1;32m    435\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m SparkContext\u001b[38;5;241m.\u001b[39m_gateway:\n\u001b[0;32m--> 436\u001b[0m         SparkContext\u001b[38;5;241m.\u001b[39m_gateway \u001b[38;5;241m=\u001b[39m gateway \u001b[38;5;129;01mor\u001b[39;00m \u001b[43mlaunch_gateway\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    437\u001b[0m         SparkContext\u001b[38;5;241m.\u001b[39m_jvm \u001b[38;5;241m=\u001b[39m SparkContext\u001b[38;5;241m.\u001b[39m_gateway\u001b[38;5;241m.\u001b[39mjvm\n\u001b[1;32m    439\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m instance:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/pyspark/java_gateway.py:107\u001b[0m, in \u001b[0;36mlaunch_gateway\u001b[0;34m(conf, popen_kwargs)\u001b[0m\n\u001b[1;32m    104\u001b[0m     time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m0.1\u001b[39m)\n\u001b[1;32m    106\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39misfile(conn_info_file):\n\u001b[0;32m--> 107\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m PySparkRuntimeError(\n\u001b[1;32m    108\u001b[0m         error_class\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mJAVA_GATEWAY_EXITED\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    109\u001b[0m         message_parameters\u001b[38;5;241m=\u001b[39m{},\n\u001b[1;32m    110\u001b[0m     )\n\u001b[1;32m    112\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(conn_info_file, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrb\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m info:\n\u001b[1;32m    113\u001b[0m     gateway_port \u001b[38;5;241m=\u001b[39m read_int(info)\n",
      "\u001b[0;31mPySparkRuntimeError\u001b[0m: [JAVA_GATEWAY_EXITED] Java gateway process exited before sending its port number."
     ]
    }
   ],
   "source": [
    "spark = SparkSession.builder.appName(\"CrimeInsights\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d66bc8a4-bb3a-4563-bc4c-97e84a6654b6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "schema = StructType([StructField(\"ID\", IntegerType(), True),\n",
    "                            StructField(\"Case Number\", StringType(), True),\n",
    "                            StructField(\"Date\", StringType(), True ),\n",
    "                            StructField(\"Block\", StringType(), True),\n",
    "                            StructField(\"IUCR\", StringType(), True),\n",
    "                            StructField(\"Primary Type\", StringType(), True  ),\n",
    "                            StructField(\"Description\", StringType(), True ),\n",
    "                            StructField(\"Location Description\", StringType(), True ),\n",
    "                            StructField(\"Arrest\", BooleanType(), True),\n",
    "                            StructField(\"Domestic\", BooleanType(), True),\n",
    "                            StructField(\"Beat\", StringType(), True),\n",
    "                            StructField(\"District\", StringType(), True),\n",
    "                            StructField(\"Ward\", IntegerType(), True),\n",
    "                            StructField(\"Community Area\", IntegerType(), True),\n",
    "                            StructField(\"FBI Code\", StringType(), True ),\n",
    "                            StructField(\"X Coordinate\", DoubleType(), True),\n",
    "                            StructField(\"Y Coordinate\", DoubleType(), True ),\n",
    "                            StructField(\"Year\", IntegerType(), True),\n",
    "                            StructField(\"Updated On\", DateType(), True ),\n",
    "                            StructField(\"Latitude\", DoubleType(), True),\n",
    "                            StructField(\"Longitude\", DoubleType(), True),\n",
    "                            StructField(\"Location\", StringType(), True )\n",
    "                            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8b84f8c9-bc37-418a-a22d-82ebf163183d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/opt/conda/lib/python3.10/site-packages/sagemaker_pyspark/jars/aws-java-sdk-bundle-1.11.901.jar', '/opt/conda/lib/python3.10/site-packages/sagemaker_pyspark/jars/aws-java-sdk-core-1.12.262.jar', '/opt/conda/lib/python3.10/site-packages/sagemaker_pyspark/jars/aws-java-sdk-kms-1.12.262.jar', '/opt/conda/lib/python3.10/site-packages/sagemaker_pyspark/jars/aws-java-sdk-s3-1.12.262.jar', '/opt/conda/lib/python3.10/site-packages/sagemaker_pyspark/jars/aws-java-sdk-sagemaker-1.12.262.jar', '/opt/conda/lib/python3.10/site-packages/sagemaker_pyspark/jars/aws-java-sdk-sagemakerruntime-1.12.262.jar', '/opt/conda/lib/python3.10/site-packages/sagemaker_pyspark/jars/aws-java-sdk-sts-1.12.262.jar', '/opt/conda/lib/python3.10/site-packages/sagemaker_pyspark/jars/hadoop-aws-3.3.1.jar', '/opt/conda/lib/python3.10/site-packages/sagemaker_pyspark/jars/sagemaker-spark_2.12-spark_3.3.0-1.4.5.jar']\n"
     ]
    }
   ],
   "source": [
    "jars = sagemaker_pyspark.classpath_jars()\n",
    "print(jars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "aa933573-9d40-4df9-857d-494dae1c0d53",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Builder' object has no attribute 'read'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[47], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mspark\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[38;5;241m.\u001b[39mcsv(output_path)\n\u001b[1;32m      2\u001b[0m df\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Builder' object has no attribute 'read'"
     ]
    }
   ],
   "source": [
    "df = spark.read.csv(output_path)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb34c1f4-6da8-43e5-bd9e-8fbe70b1b8f1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (Data Science 3.0)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-west-2:236514542706:image/sagemaker-data-science-310-v1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}