glue/glue_workbook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AWS Glue Data Catalog & Glue Jobs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ERROR: Unable to get file://glue_table_input_template_tsv.json', FileNotFoundError(2, 'No such file or directory')]\n"
     ]
    }
   ],
   "source": [
    "# Initialize Working Variables\n",
    "import boto3\n",
    "import os\n",
    "import sys\n",
    "import json\n",
    "\n",
    "os.environ['AWS_DEFAULT_REGION'] = 'us-east-2'\n",
    "\n",
    "if './python' not in sys.path: sys.path.append('./python')\n",
    "#import glue_functions as glu\n",
    "import glue_functions_2212 as glu\n",
    "\n",
    "os.environ['GlueTableInputTemplateUrl'] = 'file://data/glue_table_input_template_parquet.json'\n",
    "\n",
    "glue_database_name = 'daab-lab-glue-fiscaldata'\n",
    "\n",
    "s3_bucket = 'daab-dtl-lab-datawork'\n",
    "s3_folder = 'rmyers07/FISCALDATA/PARQUET'\n",
    "\n",
    "#zip_archive_name = f's3://{s3_bucket}/{s3_folder}'\n",
    "#s3://daab-dtl-lab-datawork/rmyers07/data/arrived/\n",
    "\n",
    "data_file_names = [     # members extracted from ZIP archive\n",
    "]\n",
    "\n",
    "glue_table_names = [    # formulated from data_file_names\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created Glue Database 'daab-lab-glue-fiscaldata'\n"
     ]
    }
   ],
   "source": [
    "# create a Glue Database Using Python SDK\n",
    "# (shoulda been created by the CloudFormation stack)\n",
    "glue_client = boto3.client('glue')\n",
    "\n",
    "try:\n",
    "    response = glue_client.create_database(\n",
    "        DatabaseInput={\n",
    "            'Name': f'{glue_database_name}',\n",
    "            'Description': 'Created from boto3 script in glue_workbook.ipynb',\n",
    "            'LocationUri': f's3://{s3_bucket}/{s3_folder}'\n",
    "        }\n",
    "    )\n",
    "    print(f\"Created Glue Database '{glue_database_name}'\")\n",
    "except glue_client.exceptions.AlreadyExistsException:\n",
    "    print(f\"Using Existing Glue Database '{glue_database_name}'\")\n",
    "\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Glue Table 'avg_interest_rates' Updated in Database 'daab-lab-glue-fiscaldata'\n",
      "Glue Table 'top_federal' Updated in Database 'daab-lab-glue-fiscaldata'\n",
      "Glue Table 'top_state' Updated in Database 'daab-lab-glue-fiscaldata'\n"
     ]
    }
   ],
   "source": [
    "# create Glue Tables Using Python SDK\n",
    "import boto3\n",
    "glue_client = boto3.client(\"glue\")\n",
    "\n",
    "import sys\n",
    "if './python' not in sys.path: sys.path.append('./python')\n",
    "import glue_functions_2212 as glu\n",
    "\n",
    "glue_table_names = [\n",
    "  'avg_interest_rates',\n",
    "  'top_federal',\n",
    "  'top_state'\n",
    "]\n",
    "\n",
    "glue_table_input = json.loads(glu.get_file('file://python/glue_table_input_template_parquet.json'))\n",
    "for glue_table_name in glue_table_names:\n",
    "  \n",
    "  with open( f\"metadata/{glue_table_name}.raml\" , \"r\" ) as f:\n",
    "    raml = f.read()\n",
    "  column_metadata = glu.import_raml_to_glue( \"\", \"\", raml )\n",
    "  s3_url_table_location = f's3://{s3_bucket}/{s3_folder}/{glue_table_name}'\n",
    "\n",
    "  glue_table_input['TableInput']['Name'] = glue_table_name\n",
    "  glue_table_input['TableInput']['StorageDescriptor']['Columns'] = column_metadata\n",
    "  glue_table_input['TableInput']['StorageDescriptor']['Location'] = s3_url_table_location\n",
    "\n",
    "  try:\n",
    "    response = glue_client.create_table(\n",
    "      DatabaseName = glue_database_name,\n",
    "      TableInput = glue_table_input['TableInput']  \n",
    "    )\n",
    "    status = 'Created'\n",
    "  except glue_client.exceptions.AlreadyExistsException:\n",
    "    response = glue_client.update_table (\n",
    "        DatabaseName = glue_database_name,\n",
    "        TableInput = glue_table_input['TableInput']\n",
    "    )\n",
    "    status = 'Updated'\n",
    "\n",
    "  print(f\"Glue Table '{glue_table_name}' {status} in Database '{glue_database_name}'\" )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list Glue Tables\n",
    "tables, table_names = glu.list_glue_tables(glue_database_name)\n",
    "print( json.dumps(tables[1], indent=2, default=str) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## export a FiscalData CSV Data Dictionary file to a RAML Data Type\n",
    "import sys\n",
    "if './python' not in sys.path: sys.path.append('./python')\n",
    "\n",
    "from dd_functions import export_fddatadict_to_raml\n",
    "#help(export_fddatadict_to_raml)\n",
    "\n",
    "dd_files = [\n",
    "    \"metadata/Average Interest Rates on US Treasury Securities Data Dictionary.csv\",\n",
    "    \"metadata/Treasury Offset Program Data Dictionary.csv\"  # both Federal and State are in this file\n",
    "]\n",
    "for csv_input_path in dd_files:\n",
    "    raml_output_path = csv_input_path.replace('.csv','.raml').replace(' ','_').lower()\n",
    "\n",
    "    export_fddatadict_to_raml( csv_input_path, raml_output_path )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"Job\": {\n",
      "    \"Name\": \"convert_csv_to_parquet\",\n",
      "    \"Description\": \"\",\n",
      "    \"Role\": \"arn:aws:iam::442483223120:role/service-role/AWSGlueServiceRole-daab-lab-glue-tutorial\",\n",
      "    \"CreatedOn\": \"2022-12-24 09:19:06.996000-05:00\",\n",
      "    \"LastModifiedOn\": \"2022-12-24 09:19:06.996000-05:00\",\n",
      "    \"ExecutionProperty\": {\n",
      "      \"MaxConcurrentRuns\": 1\n",
      "    },\n",
      "    \"Command\": {\n",
      "      \"Name\": \"glueetl\",\n",
      "      \"ScriptLocation\": \"s3://aws-glue-assets-442483223120-us-east-2/scripts/convert_csv_to_parquet.py\",\n",
      "      \"PythonVersion\": \"3\"\n",
      "    },\n",
      "    \"DefaultArguments\": {\n",
      "      \"--enable-metrics\": \"true\",\n",
      "      \"--enable-spark-ui\": \"true\",\n",
      "      \"--spark-event-logs-path\": \"s3://aws-glue-assets-442483223120-us-east-2/sparkHistoryLogs/\",\n",
      "      \"--enable-job-insights\": \"true\",\n",
      "      \"--enable-glue-datacatalog\": \"true\",\n",
      "      \"--enable-continuous-cloudwatch-log\": \"true\",\n",
      "      \"--job-bookmark-option\": \"job-bookmark-enable\",\n",
      "      \"--job-language\": \"python\",\n",
      "      \"--TempDir\": \"s3://aws-glue-assets-442483223120-us-east-2/temporary/\",\n",
      "      \"--enable-auto-scaling\": \"true\"\n",
      "    },\n",
      "    \"MaxRetries\": 1,\n",
      "    \"AllocatedCapacity\": 10,\n",
      "    \"Timeout\": 120,\n",
      "    \"MaxCapacity\": 10.0,\n",
      "    \"WorkerType\": \"G.1X\",\n",
      "    \"NumberOfWorkers\": 10,\n",
      "    \"GlueVersion\": \"3.0\"\n",
      "  },\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"43ba8b01-819e-4ada-95b8-340cefcc2e20\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"date\": \"Thu, 29 Dec 2022 11:33:32 GMT\",\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"content-length\": \"1008\",\n",
      "      \"connection\": \"keep-alive\",\n",
      "      \"x-amzn-requestid\": \"43ba8b01-819e-4ada-95b8-340cefcc2e20\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Get a Glue Job\n",
    "import boto3\n",
    "import json\n",
    "\n",
    "glue_client = boto3.client('glue')\n",
    "\n",
    "glue_jobname = \"convert_csv_to_parquet\"\n",
    "glue_job = glue_client.get_job(\n",
    "    JobName = glue_jobname\n",
    ")\n",
    "\n",
    "print(json.dumps(glue_job,indent=2,default=str))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"JobRun\": {\n",
      "    \"Id\": \"jr_c96836c96968cce949ebbe9d29860b9847aa02c1e37ed24d6af7149d64e49114\",\n",
      "    \"Attempt\": 0,\n",
      "    \"JobName\": \"convert_csv_to_parquet.py-copy\",\n",
      "    \"StartedOn\": \"2022-12-19 08:38:37.830000-05:00\",\n",
      "    \"LastModifiedOn\": \"2022-12-19 08:40:49.512000-05:00\",\n",
      "    \"CompletedOn\": \"2022-12-19 08:40:49.512000-05:00\",\n",
      "    \"JobRunState\": \"SUCCEEDED\",\n",
      "    \"PredecessorRuns\": [],\n",
      "    \"AllocatedCapacity\": 10,\n",
      "    \"ExecutionTime\": 125,\n",
      "    \"Timeout\": 120,\n",
      "    \"MaxCapacity\": 10.0,\n",
      "    \"WorkerType\": \"G.1X\",\n",
      "    \"NumberOfWorkers\": 10,\n",
      "    \"LogGroupName\": \"/aws-glue/jobs\",\n",
      "    \"GlueVersion\": \"3.0\"\n",
      "  },\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"32e7a1c2-a106-4d76-b4f3-ae17823b9056\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"date\": \"Mon, 19 Dec 2022 13:43:10 GMT\",\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"content-length\": \"911\",\n",
      "      \"connection\": \"keep-alive\",\n",
      "      \"x-amzn-requestid\": \"32e7a1c2-a106-4d76-b4f3-ae17823b9056\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Get a Glue ETL Job Run\n",
    "glue_jobrun_id = 'jr_c96836c96968cce949ebbe9d29860b9847aa02c1e37ed24d6af7149d64e49114'\n",
    "\n",
    "glue_jobruns = glue_client.get_job_run(\n",
    "    JobName= glue_jobname,\n",
    "    RunId= glue_jobrun_id,\n",
    "    PredecessorsIncluded=True|False\n",
    ")\n",
    "print(json.dumps(glue_jobruns,indent=2,default=str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Partitions Updated for Glue Table 'avg_interest_rates' in Database 'daab-lab-glue-fiscaldata: ['D221212.full']'\n",
      "Partitions Updated for Glue Table 'avg_interest_rates' in Database 'daab-lab-glue-fiscaldata: ['D221213.full']'\n",
      "Partitions Updated for Glue Table 'avg_interest_rates' in Database 'daab-lab-glue-fiscaldata: ['D221216.full']'\n",
      "Partitions Updated for Glue Table 'top_federal' in Database 'daab-lab-glue-fiscaldata: ['D221212.full']'\n",
      "Partitions Updated for Glue Table 'top_federal' in Database 'daab-lab-glue-fiscaldata: ['D221213.full']'\n",
      "Partitions Updated for Glue Table 'top_federal' in Database 'daab-lab-glue-fiscaldata: ['D221216.full']'\n",
      "Partitions Updated for Glue Table 'top_state' in Database 'daab-lab-glue-fiscaldata: ['D221212.full']'\n",
      "Partitions Updated for Glue Table 'top_state' in Database 'daab-lab-glue-fiscaldata: ['D221213.full']'\n",
      "Partitions Updated for Glue Table 'top_state' in Database 'daab-lab-glue-fiscaldata: ['D221216.full']'\n"
     ]
    }
   ],
   "source": [
    "# CReate or UPdate a Glue Partition\n",
    "import os\n",
    "os.environ['AWS_DEFAULT_REGION'] = 'us-east-2'\n",
    "\n",
    "import sys\n",
    "if './python' not in sys.path: sys.path.append('./python')\n",
    "from glue_functions_2212 import crup_glue_partition\n",
    "\n",
    "### Driver\n",
    "glue_database_name = 'daab-lab-glue-fiscaldata'\n",
    "\n",
    "glue_table_names = [\n",
    "    'avg_interest_rates',\n",
    "    'top_federal',\n",
    "    'top_state'\n",
    "]\n",
    "partition_folders = [ \n",
    "    'D221212.full',\n",
    "    'D221213.full',\n",
    "    'D221216.full'\n",
    "]\n",
    "\n",
    "for glue_table_name in glue_table_names:\n",
    "\n",
    "    for partition_folder in partition_folders:\n",
    "\n",
    "        crup_glue_partition( glue_database_name, glue_table_name, [ partition_folder ] )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run a Glue ETL job\n",
    "lambda_resp = json.loads(\"{\\\"S3BucketName\\\": \\\"daab-dtl-lab-datawork\\\", \\\"S3Key\\\": \\\"rmyers07/FISCALDATA/ARRIVED/FDMD.FISCALDATA.D221212.FULL.ZIP\\\", \\\"S3Url\\\": \\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/FDMD.FISCALDATA.D221212.FULL.ZIP\\\", \\\"S3ExtractedUrls\\\": [\\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_federal/D221212.full/top_federal.D221212.full.csv.gz\\\", \\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_state/D221212.full/top_state.D221212.full.csv.gz\\\", \\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/avg_interest_rates/D221212.full/avg_interest_rates.D221212.full.csv.gz\\\"]}\")\n",
    "\n",
    "glue_jobname = 'convert_csv_to_parquet.py-copy'\n",
    "\n",
    "import datetime\n",
    "execName = datetime.datetime.now().strftime(\"%y%m%d-%H%M%S-\") + lambda_resp['S3Url'].split('/')[-1]\n",
    "\n",
    "import boto3\n",
    "glue_client = boto3.client('glue')\n",
    "\n",
    "table_names = []\n",
    "partition_folders = []\n",
    "for ext in lambda_resp['S3ExtractedUrls']:\n",
    "    filename = ext.split('/')[-1].split('.')\n",
    "    table_names.append( filename[0] )\n",
    "    folder = f\"{filename[1]}.{filename[2]}\"\n",
    "    if folder not in partition_folders:\n",
    "        partition_folders.append( f\"{filename[1]}.{filename[2]}\")\n",
    "\n",
    "response = glue_client.start_job_run(\n",
    "    JobName = glue_jobname ,\n",
    "    #JobRunId = execName ,\n",
    "    Arguments = {\n",
    "        '--GlueTableNames' : ','.join(table_names),\n",
    "        '--PartitionFolders' : ','.join(partition_folders)\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "input = {\n",
    "  \"statusCode\": 200,\n",
    "  \"body\": {\n",
    "    \"source\": \"aws.s3\",\n",
    "    \"detail\": {\n",
    "      \"bucket\": {\n",
    "        \"name\": \"daab-dtl-lab-datawork\"\n",
    "      },\n",
    "      \"object\": {\n",
    "        \"key\": \"rmyers07/FISCALDATA/ARRIVED/FDMD.FISCALDATA.D221212.FULL.ZIP\"\n",
    "      }\n",
    "    },\n",
    "    \"batch_parms\": {\n",
    "      \"GlueDatabaseName\": \"daab-lab-glue-fiscaldata\",\n",
    "      \"S3OutputBucket\": \"daab-dtl-lab-datawork\",\n",
    "      \"S3OutputFolder\": \"rmyers07/FISCALDATA/PARQUET\",\n",
    "      \"StepFnArn\": \"arn:aws:states:us-east-2:442483223120:stateMachine:daab-lab-Extract_ZIP_to_Parquet\",\n",
    "      \"BatchId\": \"FDMD.FISCALDATA.D221212.FULL.ZIP\",\n",
    "      \"ExecName\": \"221222-205127-FDMD.FISCALDATA.D221212.FULL.ZIP\",\n",
    "      \"ZipExtracted\": {\n",
    "        \"GlueTableNames\": [\n",
    "          \"top_federal\",\n",
    "          \"top_state\",\n",
    "          \"avg_interest_rates\"\n",
    "        ],\n",
    "        \"PartitionFolders\": [\n",
    "          \"D221212.full\"\n",
    "        ],\n",
    "        \"S3ExtractedUrls\": [\n",
    "          \"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_federal/D221212.full/top_federal.D221212.full.csv.gz\",\n",
    "          \"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_state/D221212.full/top_state.D221212.full.csv.gz\",\n",
    "          \"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/avg_interest_rates/D221212.full/avg_interest_rates.D221212.full.csv.gz\"\n",
    "        ]\n",
    "      }\n",
    "    }\n",
    "  },\n",
    "  \"batch_parms_s\": \"{\\\"GlueDatabaseName\\\": \\\"daab-lab-glue-fiscaldata\\\", \\\"S3OutputBucket\\\": \\\"daab-dtl-lab-datawork\\\", \\\"S3OutputFolder\\\": \\\"rmyers07/FISCALDATA/PARQUET\\\", \\\"StepFnArn\\\": \\\"arn:aws:states:us-east-2:442483223120:stateMachine:daab-lab-Extract_ZIP_to_Parquet\\\", \\\"BatchId\\\": \\\"FDMD.FISCALDATA.D221212.FULL.ZIP\\\", \\\"ExecName\\\": \\\"221222-205127-FDMD.FISCALDATA.D221212.FULL.ZIP\\\", \\\"ZipExtracted\\\": {\\\"GlueTableNames\\\": [\\\"top_federal\\\", \\\"top_state\\\", \\\"avg_interest_rates\\\"], \\\"PartitionFolders\\\": [\\\"D221212.full\\\"], \\\"S3ExtractedUrls\\\": [\\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_federal/D221212.full/top_federal.D221212.full.csv.gz\\\", \\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_state/D221212.full/top_state.D221212.full.csv.gz\\\", \\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/avg_interest_rates/D221212.full/avg_interest_rates.D221212.full.csv.gz\\\"]}}\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run a Glue ETL job\n",
    "#lambda_resp = json.loads(\"{\\\"S3BucketName\\\": \\\"daab-dtl-lab-datawork\\\", \\\"S3Key\\\": \\\"rmyers07/FISCALDATA/ARRIVED/FDMD.FISCALDATA.D221212.FULL.ZIP\\\", \\\"S3Url\\\": \\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/FDMD.FISCALDATA.D221212.FULL.ZIP\\\", \\\"S3ExtractedUrls\\\": [\\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_federal/D221212.full/top_federal.D221212.full.csv.gz\\\", \\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_state/D221212.full/top_state.D221212.full.csv.gz\\\", \\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/avg_interest_rates/D221212.full/avg_interest_rates.D221212.full.csv.gz\\\"]}\")\n",
    "\n",
    "#glue_jobname = 'convert_csv_to_parquet.py-copy'\n",
    "glue_jobname = 'just_get_args'\n",
    "\n",
    "import boto3\n",
    "glue_client = boto3.client('glue')\n",
    "\n",
    "response = glue_client.start_job_run(\n",
    "    JobName = glue_jobname ,\n",
    "    Arguments = {\n",
    "        '--Body' : input[\"batch_parms_s\"]\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'true' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn [1], line 11\u001b[0m\n\u001b[0;32m      1\u001b[0m first \u001b[39m=\u001b[39m {   \u001b[39m# using string arg\u001b[39;00m\n\u001b[0;32m      2\u001b[0m   \u001b[39m\"\u001b[39m\u001b[39mType\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mTask\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m      3\u001b[0m   \u001b[39m\"\u001b[39m\u001b[39mResource\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39marn:aws:states:::glue:startJobRun\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m      4\u001b[0m   \u001b[39m\"\u001b[39m\u001b[39mInputPath\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39m$.batch_parms_s\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m      5\u001b[0m   \u001b[39m\"\u001b[39m\u001b[39mParameters\u001b[39m\u001b[39m\"\u001b[39m: {\n\u001b[0;32m      6\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mJobName\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mjust_get_args\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m      7\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mArguments\u001b[39m\u001b[39m\"\u001b[39m: {\n\u001b[0;32m      8\u001b[0m       \u001b[39m\"\u001b[39m\u001b[39m--Body.$\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39m$\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m      9\u001b[0m     }\n\u001b[0;32m     10\u001b[0m   },\n\u001b[1;32m---> 11\u001b[0m   \u001b[39m\"\u001b[39m\u001b[39mEnd\u001b[39m\u001b[39m\"\u001b[39m: true\n\u001b[0;32m     12\u001b[0m }\n\u001b[0;32m     14\u001b[0m \u001b[39mnext\u001b[39m \u001b[39m=\u001b[39m {   \u001b[39m# converting json to string\u001b[39;00m\n\u001b[0;32m     15\u001b[0m   \u001b[39m\"\u001b[39m\u001b[39mType\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mTask\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m     16\u001b[0m   \u001b[39m\"\u001b[39m\u001b[39mResource\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39marn:aws:states:::glue:startJobRun\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     24\u001b[0m   \u001b[39m\"\u001b[39m\u001b[39mEnd\u001b[39m\u001b[39m\"\u001b[39m: true\n\u001b[0;32m     25\u001b[0m }\n\u001b[0;32m     27\u001b[0m InputPath \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m$.body.batch_parms\u001b[39m\u001b[39m\"\u001b[39m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'true' is not defined"
     ]
    }
   ],
   "source": [
    "first = {   # using string arg\n",
    "  \"Type\": \"Task\",\n",
    "  \"Resource\": \"arn:aws:states:::glue:startJobRun\",\n",
    "  \"InputPath\": \"$.batch_parms_s\",\n",
    "  \"Parameters\": {\n",
    "    \"JobName\": \"just_get_args\",\n",
    "    \"Arguments\": {\n",
    "      \"--Body.$\": \"$\"\n",
    "    }\n",
    "  },\n",
    "  \"End\": true\n",
    "}\n",
    "\n",
    "next = {   # converting json to string\n",
    "  \"Type\": \"Task\",\n",
    "  \"Resource\": \"arn:aws:states:::glue:startJobRun\",\n",
    "  \"InputPath\": \"$.body.batch_parms\",\n",
    "  \"Parameters\": {\n",
    "    \"JobName\": \"just_get_args\",\n",
    "    \"Arguments\": {\n",
    "      \"--Body.$\": \"States.JsonToString($)\"\n",
    "    }\n",
    "  },\n",
    "  \"End\": true\n",
    "}\n",
    "\n",
    "#InputPath = \"$.body.batch_parms\"\n",
    "\n",
    "{\n",
    "    \"JobName\": \"just_get_args\",\n",
    "    \"Arguments\": {\n",
    "      \"--Body.$\": \"States.JsonToString($)\"\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"source\": \"aws.s3\",\n",
      "  \"detail\": {\n",
      "    \"bucket\": {\n",
      "      \"name\": \"daab-dtl-lab-datawork\"\n",
      "    },\n",
      "    \"object\": {\n",
      "      \"key\": \"rmyers07/FISCALDATA/ARRIVED/FDMD.FISCALDATA.D221212.FULL.ZIP\"\n",
      "    }\n",
      "  },\n",
      "  \"batch_parms\": {\n",
      "    \"GlueDatabaseName\": \"daab-lab-glue-fiscaldata\",\n",
      "    \"S3OutputBucket\": \"daab-dtl-lab-datawork\",\n",
      "    \"S3OutputFolder\": \"rmyers07/FISCALDATA/PARQUET\",\n",
      "    \"StepFnArn\": \"arn:aws:states:us-east-2:442483223120:stateMachine:daab-lab-Extract_ZIP_to_Parquet\",\n",
      "    \"BatchId\": \"FDMD.FISCALDATA.D221212.FULL.ZIP\",\n",
      "    \"ExecName\": \"221222-191619-FDMD.FISCALDATA.D221212.FULL.ZIP\",\n",
      "    \"ZipExtracted\": {\n",
      "      \"GlueTableNames\": [\n",
      "        \"top_federal\",\n",
      "        \"top_state\",\n",
      "        \"avg_interest_rates\"\n",
      "      ],\n",
      "      \"PartitionFolders\": [\n",
      "        \"D221212.full\"\n",
      "      ],\n",
      "      \"S3ExtractedUrls\": [\n",
      "        \"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_federal/D221212.full/top_federal.D221212.full.csv.gz\",\n",
      "        \"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_state/D221212.full/top_state.D221212.full.csv.gz\",\n",
      "        \"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/avg_interest_rates/D221212.full/avg_interest_rates.D221212.full.csv.gz\"\n",
      "      ]\n",
      "    }\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "result = {\n",
    "  \"statusCode\": 200,\n",
    "  \"body\": \"{\\\"source\\\": \\\"aws.s3\\\", \\\"detail\\\": {\\\"bucket\\\": {\\\"name\\\": \\\"daab-dtl-lab-datawork\\\"}, \\\"object\\\": {\\\"key\\\": \\\"rmyers07/FISCALDATA/ARRIVED/FDMD.FISCALDATA.D221212.FULL.ZIP\\\"}}, \\\"batch_parms\\\": {\\\"GlueDatabaseName\\\": \\\"daab-lab-glue-fiscaldata\\\", \\\"S3OutputBucket\\\": \\\"daab-dtl-lab-datawork\\\", \\\"S3OutputFolder\\\": \\\"rmyers07/FISCALDATA/PARQUET\\\", \\\"StepFnArn\\\": \\\"arn:aws:states:us-east-2:442483223120:stateMachine:daab-lab-Extract_ZIP_to_Parquet\\\", \\\"BatchId\\\": \\\"FDMD.FISCALDATA.D221212.FULL.ZIP\\\", \\\"ExecName\\\": \\\"221222-191619-FDMD.FISCALDATA.D221212.FULL.ZIP\\\", \\\"ZipExtracted\\\": {\\\"GlueTableNames\\\": [\\\"top_federal\\\", \\\"top_state\\\", \\\"avg_interest_rates\\\"], \\\"PartitionFolders\\\": [\\\"D221212.full\\\"], \\\"S3ExtractedUrls\\\": [\\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_federal/D221212.full/top_federal.D221212.full.csv.gz\\\", \\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/top_state/D221212.full/top_state.D221212.full.csv.gz\\\", \\\"s3://daab-dtl-lab-datawork/rmyers07/FISCALDATA/ARRIVED/avg_interest_rates/D221212.full/avg_interest_rates.D221212.full.csv.gz\\\"]}}}\"\n",
    "}\n",
    "\n",
    "import json\n",
    "body = json.loads(result['body'])\n",
    "\n",
    "print(json.dumps(body, indent=2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "9650cb4e16cdd4a8e8e2d128bf38d875813998db22a3c986335f89e0cb4d7bb2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}